def main(): try: connection = Connection('amqp://*****:*****@192.168.8.108:5672//') except Exception: raise print 'connecting to amqp server succeed!' channel = connection.channel() _exchange = Exchange('media', type='direct', channel=channel) video_queue = Queue('video', exchange=_exchange, routing_key='video', channel=channel) consumer = Consumer(channel, queues=[video_queue], callbacks=[process_data]) consumer.consume() while True: connection.drain_events(timeout=10) consumer.cancel()
def test_publish__consume(self): connection = Connection(transport=Transport) channel = connection.channel() producer = Producer(channel, self.exchange, routing_key='test_Redis') consumer = Consumer(channel, self.queue) producer.publish({'hello2': 'world2'}) _received = [] def callback(message_data, message): _received.append(message_data) message.ack() consumer.register_callback(callback) consumer.consume() self.assertIn(channel, channel.connection.cycle._channels) try: connection.drain_events(timeout=1) self.assertTrue(_received) with self.assertRaises(socket.timeout): connection.drain_events(timeout=0.01) finally: channel.close()
class RPCListener(threading.Thread): """ RPC Listener """ def __init__(self, host, port, user_id, password, virt_host, exchange_name, routing_key, consumer_queue_name): super(RPCListener, self).__init__() self._exit = threading.Event() self._exchange_name = exchange_name self._routing_key = routing_key self._consumer_queue_name = consumer_queue_name self._exchange = Exchange(self._exchange_name, type='topic', durable=False) self._connection = Connection(host, user_id, password, virt_host, port) self._rpc_receive_queue = Queue(self._consumer_queue_name, durable=True, exchange=self._exchange, routing_key=self._routing_key) self._consumer = Consumer(self._connection, self._rpc_receive_queue) self._consumer.register_callback(self._callback) self._message_queue = selectable.ThreadQueue(consumer_queue_name) self._message_filters_lock = threading.RLock() self._message_filters = dict() self._message_handlers = dict() selobj.selobj_add_read_obj(self._message_queue.selobj, self._dispatch_messages) @coroutine def _dispatch_messages(self): """ Dispatch messages from the message queue """ while True: select_obj = (yield) if select_obj == self._message_queue.selobj: msg = self._message_queue.get_nowait() if msg is not None: msg_type = msg.get('type', None) if msg_type is not None: msg_handler = self._message_handlers.get( msg_type, None) if msg_handler is not None: msg_handler(msg['data']) def add_message_handler(self, msg_type, msg_filter, msg_handler): """ Add message handler """ self._message_filters_lock.acquire() try: self._message_filters[msg_type] = msg_filter finally: self._message_filters_lock.release() self._message_handlers[msg_type] = msg_handler def del_message_handler(self, msg_type): """ Delete message handler """ self._message_filters_lock.acquire() try: if msg_type in self._message_filters: del self._message_filters[msg_type] finally: self._message_filters_lock.release() if msg_type in self._message_handlers: del self._message_handlers[msg_type] def _callback(self, body, message): """ RPC Listener callback """ self._message_filters_lock.acquire() try: for msg_type, msg_filter in self._message_filters.iteritems(): msg_data = msg_filter(body) if msg_data is not None: msg = dict() msg['type'] = msg_type msg['data'] = msg_data self._message_queue.put(msg) else: import pprint pp_msg = pprint.pformat(body) DLOG.verbose("Ignoring message: %s" % pp_msg) finally: self._message_filters_lock.release() message.ack() def run(self): """ RPC Listener main """ def _connection_error(exc, interval): DLOG.info("Connection error, exception=%s" % exc) while not self._exit.is_set(): try: if not self._connection.connected: DLOG.info( "RPC-Listener not connected to exchange %s, queue=%s." % (self._exchange_name, self._consumer_queue_name)) self._connection.ensure_connection( errback=_connection_error, interval_start=2, interval_step=0, interval_max=2) self._consumer.revive(self._connection) DLOG.info( "RPC-Listener connected to exchange %s, queue=%s." % (self._exchange_name, self._consumer_queue_name)) self._consumer.consume() self._connection.drain_events(timeout=1) except exceptions.TimeoutError as e: DLOG.verbose("Timeout exception, exception=%s" % e) except Exception as e: DLOG.error("Exception received, exception=%s." % e) time.sleep(2) def stop(self): """ Stop RPC Listener """ self._exit.set()
class test_FilesystemTransport(TestCase): def setUp(self): try: data_folder_in = tempfile.mkdtemp() data_folder_out = tempfile.mkdtemp() except Exception: raise SkipTest('filesystem transport: cannot create tempfiles') self.c = Connection(transport='filesystem', transport_options={ 'data_folder_in': data_folder_in, 'data_folder_out': data_folder_out, }) self.p = Connection(transport='filesystem', transport_options={ 'data_folder_in': data_folder_out, 'data_folder_out': data_folder_in, }) self.e = Exchange('test_transport_filesystem') self.q = Queue('test_transport_filesystem', exchange=self.e, routing_key='test_transport_filesystem') self.q2 = Queue('test_transport_filesystem2', exchange=self.e, routing_key='test_transport_filesystem2') def test_produce_consume_noack(self): producer = Producer(self.p.channel(), self.e) consumer = Consumer(self.c.channel(), self.q, no_ack=True) for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') _received = [] def callback(message_data, message): _received.append(message) consumer.register_callback(callback) consumer.consume() while 1: if len(_received) == 10: break self.c.drain_events() self.assertEqual(len(_received), 10) def test_produce_consume(self): producer_channel = self.p.channel() consumer_channel = self.c.channel() producer = Producer(producer_channel, self.e) consumer1 = Consumer(consumer_channel, self.q) consumer2 = Consumer(consumer_channel, self.q2) self.q2(consumer_channel).declare() for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem2') _received1 = [] _received2 = [] def callback1(message_data, message): _received1.append(message) message.ack() def callback2(message_data, message): _received2.append(message) message.ack() consumer1.register_callback(callback1) consumer2.register_callback(callback2) consumer1.consume() consumer2.consume() while 1: if len(_received1) + len(_received2) == 20: break self.c.drain_events() self.assertEqual(len(_received1) + len(_received2), 20) # compression producer.publish({'compressed': True}, routing_key='test_transport_filesystem', compression='zlib') m = self.q(consumer_channel).get() self.assertDictEqual(m.payload, {'compressed': True}) # queue.delete for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') self.assertTrue(self.q(consumer_channel).get()) self.q(consumer_channel).delete() self.q(consumer_channel).declare() self.assertIsNone(self.q(consumer_channel).get()) # queue.purge for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem2') self.assertTrue(self.q2(consumer_channel).get()) self.q2(consumer_channel).purge() self.assertIsNone(self.q2(consumer_channel).get())
class test_MemoryTransport(TestCase): def setUp(self): self.c = Connection(transport='memory') self.e = Exchange('test_transport_memory') self.q = Queue('test_transport_memory', exchange=self.e, routing_key='test_transport_memory') self.q2 = Queue('test_transport_memory2', exchange=self.e, routing_key='test_transport_memory2') def test_produce_consume_noack(self): channel = self.c.channel() producer = Producer(channel, self.e) consumer = Consumer(channel, self.q, no_ack=True) for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory') _received = [] def callback(message_data, message): _received.append(message) consumer.register_callback(callback) consumer.consume() while 1: if len(_received) == 10: break self.c.drain_events() self.assertEqual(len(_received), 10) def test_produce_consume(self): channel = self.c.channel() producer = Producer(channel, self.e) consumer1 = Consumer(channel, self.q) consumer2 = Consumer(channel, self.q2) self.q2(channel).declare() for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory') for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory2') _received1 = [] _received2 = [] def callback1(message_data, message): _received1.append(message) message.ack() def callback2(message_data, message): _received2.append(message) message.ack() consumer1.register_callback(callback1) consumer2.register_callback(callback2) consumer1.consume() consumer2.consume() while 1: if len(_received1) + len(_received2) == 20: break self.c.drain_events() self.assertEqual(len(_received1) + len(_received2), 20) # compression producer.publish({'compressed': True}, routing_key='test_transport_memory', compression='zlib') m = self.q(channel).get() self.assertDictEqual(m.payload, {'compressed': True}) # queue.delete for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory') self.assertTrue(self.q(channel).get()) self.q(channel).delete() self.q(channel).declare() self.assertIsNone(self.q(channel).get()) # queue.purge for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory2') self.assertTrue(self.q2(channel).get()) self.q2(channel).purge() self.assertIsNone(self.q2(channel).get()) def test_drain_events(self): with self.assertRaises(socket.timeout): self.c.drain_events(timeout=0.1) c1 = self.c.channel() c2 = self.c.channel() with self.assertRaises(socket.timeout): self.c.drain_events(timeout=0.1) del(c1) # so pyflakes doesn't complain. del(c2) def test_drain_events_unregistered_queue(self): c1 = self.c.channel() class Cycle(object): def get(self, timeout=None): return ('foo', 'foo'), c1 self.c.transport.cycle = Cycle() with self.assertRaises(KeyError): self.c.drain_events() def test_queue_for(self): chan = self.c.channel() chan.queues.clear() x = chan._queue_for('foo') self.assertTrue(x) self.assertIs(chan._queue_for('foo'), x)