def main():
try:
connection = Connection('amqp://*****:*****@192.168.8.108:5672//')
except Exception:
raise
print 'connecting to amqp server succeed!'
channel = connection.channel()
_exchange = Exchange('media', type='direct', channel=channel)
video_queue = Queue('video',
exchange=_exchange,
routing_key='video',
channel=channel)
consumer = Consumer(channel,
queues=[video_queue],
callbacks=[process_data])
consumer.consume()
while True:
connection.drain_events(timeout=10)
consumer.cancel()
def test_publish__consume(self):
connection = Connection(transport=Transport)
channel = connection.channel()
producer = Producer(channel, self.exchange, routing_key='test_Redis')
consumer = Consumer(channel, self.queue)
producer.publish({'hello2': 'world2'})
_received = []
def callback(message_data, message):
_received.append(message_data)
message.ack()
consumer.register_callback(callback)
consumer.consume()
self.assertIn(channel, channel.connection.cycle._channels)
try:
connection.drain_events(timeout=1)
self.assertTrue(_received)
with self.assertRaises(socket.timeout):
connection.drain_events(timeout=0.01)
finally:
channel.close()
def test_publish__consume(self):
connection = Connection(transport=Transport)
channel = connection.channel()
producer = Producer(channel, self.exchange, routing_key='test_Redis')
consumer = Consumer(channel, self.queue)
producer.publish({'hello2': 'world2'})
_received = []
def callback(message_data, message):
_received.append(message_data)
message.ack()
consumer.register_callback(callback)
consumer.consume()
self.assertIn(channel, channel.connection.cycle._channels)
try:
connection.drain_events(timeout=1)
self.assertTrue(_received)
with self.assertRaises(socket.timeout):
connection.drain_events(timeout=0.01)
finally:
channel.close()
class RPCListener(threading.Thread):
"""
RPC Listener
"""
def __init__(self, host, port, user_id, password, virt_host, exchange_name,
routing_key, consumer_queue_name):
super(RPCListener, self).__init__()
self._exit = threading.Event()
self._exchange_name = exchange_name
self._routing_key = routing_key
self._consumer_queue_name = consumer_queue_name
self._exchange = Exchange(self._exchange_name,
type='topic',
durable=False)
self._connection = Connection(host, user_id, password, virt_host, port)
self._rpc_receive_queue = Queue(self._consumer_queue_name,
durable=True,
exchange=self._exchange,
routing_key=self._routing_key)
self._consumer = Consumer(self._connection, self._rpc_receive_queue)
self._consumer.register_callback(self._callback)
self._message_queue = selectable.ThreadQueue(consumer_queue_name)
self._message_filters_lock = threading.RLock()
self._message_filters = dict()
self._message_handlers = dict()
selobj.selobj_add_read_obj(self._message_queue.selobj,
self._dispatch_messages)
@coroutine
def _dispatch_messages(self):
"""
Dispatch messages from the message queue
"""
while True:
select_obj = (yield)
if select_obj == self._message_queue.selobj:
msg = self._message_queue.get_nowait()
if msg is not None:
msg_type = msg.get('type', None)
if msg_type is not None:
msg_handler = self._message_handlers.get(
msg_type, None)
if msg_handler is not None:
msg_handler(msg['data'])
def add_message_handler(self, msg_type, msg_filter, msg_handler):
"""
Add message handler
"""
self._message_filters_lock.acquire()
try:
self._message_filters[msg_type] = msg_filter
finally:
self._message_filters_lock.release()
self._message_handlers[msg_type] = msg_handler
def del_message_handler(self, msg_type):
"""
Delete message handler
"""
self._message_filters_lock.acquire()
try:
if msg_type in self._message_filters:
del self._message_filters[msg_type]
finally:
self._message_filters_lock.release()
if msg_type in self._message_handlers:
del self._message_handlers[msg_type]
def _callback(self, body, message):
"""
RPC Listener callback
"""
self._message_filters_lock.acquire()
try:
for msg_type, msg_filter in self._message_filters.iteritems():
msg_data = msg_filter(body)
if msg_data is not None:
msg = dict()
msg['type'] = msg_type
msg['data'] = msg_data
self._message_queue.put(msg)
else:
import pprint
pp_msg = pprint.pformat(body)
DLOG.verbose("Ignoring message: %s" % pp_msg)
finally:
self._message_filters_lock.release()
message.ack()
def run(self):
"""
RPC Listener main
"""
def _connection_error(exc, interval):
DLOG.info("Connection error, exception=%s" % exc)
while not self._exit.is_set():
try:
if not self._connection.connected:
DLOG.info(
"RPC-Listener not connected to exchange %s, queue=%s."
% (self._exchange_name, self._consumer_queue_name))
self._connection.ensure_connection(
errback=_connection_error,
interval_start=2,
interval_step=0,
interval_max=2)
self._consumer.revive(self._connection)
DLOG.info(
"RPC-Listener connected to exchange %s, queue=%s." %
(self._exchange_name, self._consumer_queue_name))
self._consumer.consume()
self._connection.drain_events(timeout=1)
except exceptions.TimeoutError as e:
DLOG.verbose("Timeout exception, exception=%s" % e)
except Exception as e:
DLOG.error("Exception received, exception=%s." % e)
time.sleep(2)
def stop(self):
"""
Stop RPC Listener
"""
self._exit.set()
class test_FilesystemTransport(TestCase):
def setUp(self):
try:
data_folder_in = tempfile.mkdtemp()
data_folder_out = tempfile.mkdtemp()
except Exception:
raise SkipTest('filesystem transport: cannot create tempfiles')
self.c = Connection(transport='filesystem',
transport_options={
'data_folder_in': data_folder_in,
'data_folder_out': data_folder_out,
})
self.p = Connection(transport='filesystem',
transport_options={
'data_folder_in': data_folder_out,
'data_folder_out': data_folder_in,
})
self.e = Exchange('test_transport_filesystem')
self.q = Queue('test_transport_filesystem',
exchange=self.e,
routing_key='test_transport_filesystem')
self.q2 = Queue('test_transport_filesystem2',
exchange=self.e,
routing_key='test_transport_filesystem2')
def test_produce_consume_noack(self):
producer = Producer(self.p.channel(), self.e)
consumer = Consumer(self.c.channel(), self.q, no_ack=True)
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem')
_received = []
def callback(message_data, message):
_received.append(message)
consumer.register_callback(callback)
consumer.consume()
while 1:
if len(_received) == 10:
break
self.c.drain_events()
self.assertEqual(len(_received), 10)
def test_produce_consume(self):
producer_channel = self.p.channel()
consumer_channel = self.c.channel()
producer = Producer(producer_channel, self.e)
consumer1 = Consumer(consumer_channel, self.q)
consumer2 = Consumer(consumer_channel, self.q2)
self.q2(consumer_channel).declare()
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem')
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem2')
_received1 = []
_received2 = []
def callback1(message_data, message):
_received1.append(message)
message.ack()
def callback2(message_data, message):
_received2.append(message)
message.ack()
consumer1.register_callback(callback1)
consumer2.register_callback(callback2)
consumer1.consume()
consumer2.consume()
while 1:
if len(_received1) + len(_received2) == 20:
break
self.c.drain_events()
self.assertEqual(len(_received1) + len(_received2), 20)
# compression
producer.publish({'compressed': True},
routing_key='test_transport_filesystem',
compression='zlib')
m = self.q(consumer_channel).get()
self.assertDictEqual(m.payload, {'compressed': True})
# queue.delete
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem')
self.assertTrue(self.q(consumer_channel).get())
self.q(consumer_channel).delete()
self.q(consumer_channel).declare()
self.assertIsNone(self.q(consumer_channel).get())
# queue.purge
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem2')
self.assertTrue(self.q2(consumer_channel).get())
self.q2(consumer_channel).purge()
self.assertIsNone(self.q2(consumer_channel).get())
class test_MemoryTransport(TestCase):
def setUp(self):
self.c = Connection(transport='memory')
self.e = Exchange('test_transport_memory')
self.q = Queue('test_transport_memory',
exchange=self.e,
routing_key='test_transport_memory')
self.q2 = Queue('test_transport_memory2',
exchange=self.e,
routing_key='test_transport_memory2')
def test_produce_consume_noack(self):
channel = self.c.channel()
producer = Producer(channel, self.e)
consumer = Consumer(channel, self.q, no_ack=True)
for i in range(10):
producer.publish({'foo': i}, routing_key='test_transport_memory')
_received = []
def callback(message_data, message):
_received.append(message)
consumer.register_callback(callback)
consumer.consume()
while 1:
if len(_received) == 10:
break
self.c.drain_events()
self.assertEqual(len(_received), 10)
def test_produce_consume(self):
channel = self.c.channel()
producer = Producer(channel, self.e)
consumer1 = Consumer(channel, self.q)
consumer2 = Consumer(channel, self.q2)
self.q2(channel).declare()
for i in range(10):
producer.publish({'foo': i}, routing_key='test_transport_memory')
for i in range(10):
producer.publish({'foo': i}, routing_key='test_transport_memory2')
_received1 = []
_received2 = []
def callback1(message_data, message):
_received1.append(message)
message.ack()
def callback2(message_data, message):
_received2.append(message)
message.ack()
consumer1.register_callback(callback1)
consumer2.register_callback(callback2)
consumer1.consume()
consumer2.consume()
while 1:
if len(_received1) + len(_received2) == 20:
break
self.c.drain_events()
self.assertEqual(len(_received1) + len(_received2), 20)
# compression
producer.publish({'compressed': True},
routing_key='test_transport_memory',
compression='zlib')
m = self.q(channel).get()
self.assertDictEqual(m.payload, {'compressed': True})
# queue.delete
for i in range(10):
producer.publish({'foo': i}, routing_key='test_transport_memory')
self.assertTrue(self.q(channel).get())
self.q(channel).delete()
self.q(channel).declare()
self.assertIsNone(self.q(channel).get())
# queue.purge
for i in range(10):
producer.publish({'foo': i}, routing_key='test_transport_memory2')
self.assertTrue(self.q2(channel).get())
self.q2(channel).purge()
self.assertIsNone(self.q2(channel).get())
def test_drain_events(self):
with self.assertRaises(socket.timeout):
self.c.drain_events(timeout=0.1)
c1 = self.c.channel()
c2 = self.c.channel()
with self.assertRaises(socket.timeout):
self.c.drain_events(timeout=0.1)
del(c1) # so pyflakes doesn't complain.
del(c2)
def test_drain_events_unregistered_queue(self):
c1 = self.c.channel()
class Cycle(object):
def get(self, timeout=None):
return ('foo', 'foo'), c1
self.c.transport.cycle = Cycle()
with self.assertRaises(KeyError):
self.c.drain_events()
def test_queue_for(self):
chan = self.c.channel()
chan.queues.clear()
x = chan._queue_for('foo')
self.assertTrue(x)
self.assertIs(chan._queue_for('foo'), x)
class test_FilesystemTransport(TestCase):
def setUp(self):
try:
data_folder_in = tempfile.mkdtemp()
data_folder_out = tempfile.mkdtemp()
except Exception:
raise SkipTest('filesystem transport: cannot create tempfiles')
self.c = Connection(transport='filesystem',
transport_options={
'data_folder_in': data_folder_in,
'data_folder_out': data_folder_out,
})
self.p = Connection(transport='filesystem',
transport_options={
'data_folder_in': data_folder_out,
'data_folder_out': data_folder_in,
})
self.e = Exchange('test_transport_filesystem')
self.q = Queue('test_transport_filesystem',
exchange=self.e,
routing_key='test_transport_filesystem')
self.q2 = Queue('test_transport_filesystem2',
exchange=self.e,
routing_key='test_transport_filesystem2')
def test_produce_consume_noack(self):
producer = Producer(self.p.channel(), self.e)
consumer = Consumer(self.c.channel(), self.q, no_ack=True)
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem')
_received = []
def callback(message_data, message):
_received.append(message)
consumer.register_callback(callback)
consumer.consume()
while 1:
if len(_received) == 10:
break
self.c.drain_events()
self.assertEqual(len(_received), 10)
def test_produce_consume(self):
producer_channel = self.p.channel()
consumer_channel = self.c.channel()
producer = Producer(producer_channel, self.e)
consumer1 = Consumer(consumer_channel, self.q)
consumer2 = Consumer(consumer_channel, self.q2)
self.q2(consumer_channel).declare()
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem')
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem2')
_received1 = []
_received2 = []
def callback1(message_data, message):
_received1.append(message)
message.ack()
def callback2(message_data, message):
_received2.append(message)
message.ack()
consumer1.register_callback(callback1)
consumer2.register_callback(callback2)
consumer1.consume()
consumer2.consume()
while 1:
if len(_received1) + len(_received2) == 20:
break
self.c.drain_events()
self.assertEqual(len(_received1) + len(_received2), 20)
# compression
producer.publish({'compressed': True},
routing_key='test_transport_filesystem',
compression='zlib')
m = self.q(consumer_channel).get()
self.assertDictEqual(m.payload, {'compressed': True})
# queue.delete
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem')
self.assertTrue(self.q(consumer_channel).get())
self.q(consumer_channel).delete()
self.q(consumer_channel).declare()
self.assertIsNone(self.q(consumer_channel).get())
# queue.purge
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem2')
self.assertTrue(self.q2(consumer_channel).get())
self.q2(consumer_channel).purge()
self.assertIsNone(self.q2(consumer_channel).get())