def test_publish__consume(self):
connection = BrokerConnection(transport=Transport)
channel = connection.channel()
producer = Producer(channel, self.exchange, routing_key="test_Redis")
consumer = Consumer(channel, self.queue)
producer.publish({"hello2": "world2"})
_received = []
def callback(message_data, message):
_received.append(message_data)
message.ack()
consumer.register_callback(callback)
consumer.consume()
self.assertIn(channel, channel.connection.cycle._channels)
try:
connection.drain_events(timeout=1)
self.assertTrue(_received)
self.assertRaises(socket.timeout,
connection.drain_events,
timeout=0.01)
finally:
channel.close()
def test_publish__consume(self):
connection = BrokerConnection(transport=Transport)
channel = connection.channel()
producer = Producer(channel, self.exchange, routing_key="test_Redis")
consumer = Consumer(channel, self.queue)
producer.publish({"hello2": "world2"})
_received = []
def callback(message_data, message):
_received.append(message_data)
message.ack()
consumer.register_callback(callback)
consumer.consume()
self.assertTrue(channel._poller._can_start())
try:
connection.drain_events(timeout=1)
self.assertTrue(_received)
self.assertFalse(channel._poller._can_start())
self.assertRaises(socket.timeout,
connection.drain_events, timeout=0.01)
finally:
channel.close()
class RabbitMQConnection:
def connection(self):
try:
logger.info("Connection to the AMQP broker '" + AMQP_IP + ":" + AMQP_PORT + "' ...")
connection = BrokerConnection(hostname=AMQP_IP,
port=AMQP_PORT,
userid=AMQP_USER,
password=AMQP_PASSWORD,
virtual_host="/",
transport=AMQP_TRANSPORT)
#channel = connection.channel()
logger.info("Connection to the AMQP broker established.")
return connection
except Exception as exep:
logger.warning("Could not connect to the AMQP broker '" + AMQP_IP + ":" + AMQP_PORT + "'. | " + str(exep))
def producer(self):
self.connection = BrokerConnection(hostname=AMQP_IP, port=AMQP_PORT,
userid=AMQP_USER, password=AMQP_PASSWORD,
virtual_host="/",
transport=AMQP_TRANSPORT)
self.channel = self.connection.channel()
# produce
self.media_exchange = Exchange("media", "direct", durable=True)
producer = Producer(self.channel, exchange=self.media_exchange, serializer="json")
producer.publish({"name": "/tmp/lolcat1.avi", "size": 1301013})
print self.connection.get_transport_cls()
def consumer(self):
self.connection = BrokerConnection(hostname=AMQP_IP, port=AMQP_PORT,
userid=AMQP_USER, password=AMQP_PASSWORD,
virtual_host="/")
self.channel = self.connection.channel()
# consume
self.media_exchange = Exchange("media", "direct", durable=True)
self.video_queue = Queue("video", exchange=self.media_exchange, key="video")
consumer = Consumer(self.channel, self.video_queue)
consumer.register_callback(self.process_media)
consumer.consume()
# Process messages on all channels
while True:
self.connection.drain_events()
def process_media(self, message_data, message):
feed_url = message_data["name"]
print ("Got feed import message for: %s" % feed_url)
# something importing this feed url
# import_feed(feed_url)
message.ack()
class AMQPWorker(Worker):
queues = [
{'routing_key': 'test',
'name': 'test',
'handler': 'handle_test'
}
]
_connection = None
def handle_test(self, body, message):
log.debug("Handle message: %s" % body)
message.ack()
def handle(self):
log.debug("Start consuming")
exchange = Exchange('amqp.topic', type='direct', durable=True)
self._connection = BrokerConnection(*CONNECTION)
channel = self._connection.channel()
for entry in self.queues:
log.debug("prepare to consume %s" % entry['routing_key'])
queue = Queue(entry['name'], exchange=exchange,
routing_key=entry['routing_key'])
consumer = Consumer(channel, queue)
consumer.register_callback(getattr(self, entry['handler']))
consumer.consume()
log.debug("start consuming...")
while True:
try:
self._connection.drain_events()
except socket.timeout:
log.debug("nothing to consume...")
break
self._connection.close()
def run(self):
while self.alive:
try:
self.handle()
except Exception:
self.alive = False
raise
def handle_quit(self, sig, frame):
if self._connection is not None:
self._connection.close()
self.alive = False
def handle_exit(self, sig, frame):
if self._connection is not None:
self._connection.close()
self.alive = False
sys.exit(0)
class AMQPWorker(Worker):
queues = [{
'routing_key': 'test',
'name': 'test',
'handler': 'handle_test'
}]
_connection = None
def handle_test(self, body, message):
log.debug("Handle message: %s" % body)
message.ack()
def handle(self):
log.debug("Start consuming")
exchange = Exchange('amqp.topic', type='direct', durable=True)
self._connection = BrokerConnection(*CONNECTION)
channel = self._connection.channel()
for entry in self.queues:
log.debug("prepare to consume %s" % entry['routing_key'])
queue = Queue(entry['name'],
exchange=exchange,
routing_key=entry['routing_key'])
consumer = Consumer(channel, queue)
consumer.register_callback(getattr(self, entry['handler']))
consumer.consume()
log.debug("start consuming...")
while True:
try:
self._connection.drain_events()
except socket.timeout:
log.debug("nothing to consume...")
break
self._connection.close()
def run(self):
while self.alive:
try:
self.handle()
except Exception:
self.alive = False
raise
def handle_quit(self, sig, frame):
if self._connection is not None:
self._connection.close()
self.alive = False
def handle_exit(self, sig, frame):
if self._connection is not None:
self._connection.close()
self.alive = False
sys.exit(0)
def test_establish_connection(self):
conn = BrokerConnection(port=5672, backend_cls=Backend)
conn.connect()
self.assertTrue(conn.connection.connected)
self.assertEqual(conn.host, "localhost:5672")
channel = conn.channel()
self.assertTrue(channel.open)
self.assertEqual(conn.drain_events(), "event")
_connection = conn.connection
conn.close()
self.assertFalse(_connection.connected)
self.assertIsInstance(conn.backend, Backend)
class Listener():
""" Simple class to wrap the operations needed for an AMQP listener """
def __init__(self, hostname="127.0.0.1", userid="guest", password="******",
virtual_host="/", port=5672):
""" Setup a connection to the AMQP server, get a channel
Create a topic exchange, attach a bonded queue to it
and register a consumer callback.
A specific service listener implementation overrides the name
and routing_key
"""
self.connection = BrokerConnection(hostname=hostname,
userid=userid, password=password,
virtual_host=virtual_host, port=port,
insist=False, ssl=False)
self.channel = self.connection.channel()
self.exchange = Exchange(name=self.name, type="topic", durable=True,
channel=self.channel)
self.queue = Queue(self.name, exchange=self.exchange,
routing_key=self.routing_key)
self.queue = self.queue(self.channel)
self.queue.declare()
self.queue.consume(consumer_tag="", callback=self.callback, no_ack=True)
self.connection.connect()
return
def callback(self, msg):
""" This callback is run when a message is recieved """
return
def consume(self):
""" Event loop """
while True:
self.connection.drain_events()
return
class test_MemoryTransport(unittest.TestCase):
def setUp(self):
self.c = BrokerConnection(transport="memory")
self.e = Exchange("test_transport_memory")
self.q = Queue("test_transport_memory",
exchange=self.e,
routing_key="test_transport_memory")
self.q2 = Queue("test_transport_memory2",
exchange=self.e,
routing_key="test_transport_memory2")
def test_produce_consume_noack(self):
channel = self.c.channel()
producer = Producer(channel, self.e)
consumer = Consumer(channel, self.q, no_ack=True)
for i in range(10):
producer.publish({"foo": i}, routing_key="test_transport_memory")
_received = []
def callback(message_data, message):
_received.append(message)
consumer.register_callback(callback)
consumer.consume()
while 1:
if len(_received) == 10:
break
self.c.drain_events()
self.assertEqual(len(_received), 10)
def test_produce_consume(self):
channel = self.c.channel()
producer = Producer(channel, self.e)
consumer1 = Consumer(channel, self.q)
consumer2 = Consumer(channel, self.q2)
self.q2(channel).declare()
for i in range(10):
producer.publish({"foo": i}, routing_key="test_transport_memory")
for i in range(10):
producer.publish({"foo": i}, routing_key="test_transport_memory2")
_received1 = []
_received2 = []
def callback1(message_data, message):
_received1.append(message)
message.ack()
def callback2(message_data, message):
_received2.append(message)
message.ack()
consumer1.register_callback(callback1)
consumer2.register_callback(callback2)
consumer1.consume()
consumer2.consume()
while 1:
if len(_received1) + len(_received2) == 20:
break
self.c.drain_events()
self.assertEqual(len(_received1) + len(_received2), 20)
# compression
producer.publish({"compressed": True},
routing_key="test_transport_memory",
compression="zlib")
m = self.q(channel).get()
self.assertDictEqual(m.payload, {"compressed": True})
# queue.delete
for i in range(10):
producer.publish({"foo": i}, routing_key="test_transport_memory")
self.assertTrue(self.q(channel).get())
self.q(channel).delete()
self.q(channel).declare()
self.assertIsNone(self.q(channel).get())
# queue.purge
for i in range(10):
producer.publish({"foo": i}, routing_key="test_transport_memory2")
self.assertTrue(self.q2(channel).get())
self.q2(channel).purge()
self.assertIsNone(self.q2(channel).get())
def test_drain_events(self):
self.assertRaises(ValueError, self.c.drain_events, timeout=0.1)
c1 = self.c.channel()
c2 = self.c.channel()
self.assertRaises(socket.timeout, self.c.drain_events, timeout=0.1)
del(c1) # so pyflakes doesn't complain.
del(c2)
def test_drain_events_unregistered_queue(self):
c1 = self.c.channel()
class Cycle(object):
def get(self):
return ("foo", "foo"), c1
self.c.transport.cycle = Cycle()
self.assertRaises(KeyError, self.c.drain_events)
connection = BrokerConnection(
hostname='localhost',
userid='gamelion',
password='******',
virtual_host='/'
)
channel = connection.channel()
consumer = Consumer(channel, server_queue)
consumer.register_callback(receive_message)
consumer.consume(no_ack=True)
while True:
try:
connection.drain_events(timeout=CONSUME_TIMEOUT)
except socket.timeout:
# flush the server buffer if we haven't gotten
# any messages in a while
if len(server_buffer) > 0:
logging.debug('QUEUE TIMED OUT, FLUSHING BUFFER')
process_servers()
elif display_waiting:
processing_time = datetime.now() -\
last_waiting_time -\
timedelta(seconds=CONSUME_TIMEOUT)
logging.debug(
'... WAITING ... TIME SINCE LAST: %s',
processing_time
)
self.consumer.register_callback(mq_callback)
c1 = AConsumer("test_1","test.1")
c2 = AConsumer("testing","test.ing")
# consumers can use simple pattern matching when defining a queue
c3 = AConsumer("test_all","test.*")
# 3. publish something to consume
# publishers always send to a specific route, the mq will route to the queues
producer = Producer(pub_chan, exchange=pub_exch, serializer=message_serializer)
producer.publish({"name": "Shane Caraveo", "username": "******"}, routing_key="test.1")
producer.publish({"name": "Micky Mouse", "username": "******"}, routing_key="test.ing")
producer.publish({"name": "Anonymous", "username": "******"}, routing_key="test.foobar")
def have_messages():
return sum([q.qsize() for q in cons_chan.queues.values()])
# 5. run the event loop
while have_messages():
try:
cons_conn.drain_events()
except KeyboardInterrupt:
print
print "quitting"
break
except Exception, e:
import traceback
print traceback.format_exc()
break