def test_produce_consume(self):
channel = self.c.channel()
producer = Producer(channel, self.e)
consumer1 = Consumer(channel, self.q)
consumer2 = Consumer(channel, self.q2)
self.q2(channel).declare()
for i in range(10):
producer.publish({"foo": i}, routing_key="test_transport_memory")
for i in range(10):
producer.publish({"foo": i}, routing_key="test_transport_memory2")
_received1 = []
_received2 = []
def callback1(message_data, message):
_received1.append(message)
message.ack()
def callback2(message_data, message):
_received2.append(message)
message.ack()
consumer1.register_callback(callback1)
consumer2.register_callback(callback2)
consumer1.consume()
consumer2.consume()
while 1:
if len(_received1) + len(_received2) == 20:
break
self.c.drain_events()
self.assertEqual(len(_received1) + len(_received2), 20)
# compression
producer.publish({"compressed": True},
routing_key="test_transport_memory",
compression="zlib")
m = self.q(channel).get()
self.assertDictEqual(m.payload, {"compressed": True})
# queue.delete
for i in range(10):
producer.publish({"foo": i}, routing_key="test_transport_memory")
self.assertTrue(self.q(channel).get())
self.q(channel).delete()
self.q(channel).declare()
self.assertIsNone(self.q(channel).get())
# queue.purge
for i in range(10):
producer.publish({"foo": i}, routing_key="test_transport_memory2")
self.assertTrue(self.q2(channel).get())
self.q2(channel).purge()
self.assertIsNone(self.q2(channel).get())
def get_consumers(self):
"""
Create two consumers for each message processor: one
for the outgoing message queue, and one for the incoming message
queue.
"""
consumers = {}
# import dynamically (use the import path in the settings file) all
# message processor then create one instance for each of them
mps = (import_class(mp) for mp in settings.MESSAGE_PROCESSORS)
self.message_processors = [mp() for mp in mps]
# Just a log loop to say that we do
mps = (mp.rsplit('.', 1)[1] for mp in settings.MESSAGE_PROCESSORS)
self.logger.info('Loading message processors: %s' % ', '.join(mps))
# Create the consumer for incoming messages and attach the callback
# of each message processor
queue = self.queues['incoming_messages']
c = consumers['incoming_messages'] = Consumer(self.channel, queue)
for mp in self.message_processors:
c.register_callback(mp.handle_incoming_message)
c.consume()
# Create the consumer for incoming messages and attach the callback
# of each message processor
# then attach a router callback that is going to relay the message
# to the proper transport queue
queue = self.queues['outgoing_messages']
c = consumers['outgoing_messages'] = Consumer(self.channel, queue)
for mp in self.message_processors:
c.register_callback(mp.handle_outgoing_message)
c.register_callback(self.relay_message_to_transport)
c.consume()
# Create the consumer for the log messages and attach a callback
# from the SMS router: all messages sent to this queue are going
# to be logged in the router log
consumers['logs'] = Consumer(self.channel, self.queues['logs'])
consumers['logs'].register_callback(self.handle_log)
consumers['logs'].consume()
# attach a fall back functions to handle message that kombu can't deliver
queue = self.queues['undelivered_kombu_message']
c = consumers['undeliverd_kombu_messages'] = Consumer(
self.channel, queue)
c.register_callback(self.handle_undelivered_kombu_message)
c.consume()
def worker(mq_url):
connection = Connection(mq_url)
channel = connection.channel()
consumer_json = Consumer(channel,
task_json_queue,
callbacks=[process_json],
accept=['json'])
consumer_json.consume()
consumer_pickle = Consumer(channel,
task_pickle_queue,
callbacks=[process_pickle],
accept=['pickle'])
consumer_pickle.consume()
while True:
connection.drain_events()
def _collect(self,
ticket,
limit=None,
timeout=1,
callback=None,
channel=None):
chan = channel or self.connection.channel()
queue = self.get_reply_queue(ticket)
consumer = Consumer(channel, [queue], no_ack=True)
responses = []
def on_message(body, message):
if callback:
callback(body)
responses.append(body)
try:
consumer.register_callback(on_message)
consumer.consume()
for i in limit and range(limit) or count():
try:
self.connection.drain_events(timeout=timeout)
except socket.timeout:
break
chan.after_reply_message_received(queue.name)
return responses
finally:
channel or chan.close()
def init_rabbit_mq(self):
"""
This function will attempt to connect to RabbitMQ Server and if successful
return 'True'. Returns 'False' otherwise.
"""
self.logger.info("Initializing RabbitMQ stuff")
try:
schedule_exchange = Exchange("airtime-media-monitor",
"direct",
durable=True,
auto_delete=True)
schedule_queue = Queue("media-monitor",
exchange=schedule_exchange,
key="filesystem")
self.connection = BrokerConnection(
self.config.cfg["rabbitmq_host"],
self.config.cfg["rabbitmq_user"],
self.config.cfg["rabbitmq_password"],
self.config.cfg["rabbitmq_vhost"])
channel = self.connection.channel()
consumer = Consumer(channel, schedule_queue)
consumer.register_callback(self.handle_message)
consumer.consume()
except Exception, e:
self.logger.error(e)
return False
def _connect(self, channel):
self._queue = Queue(channel=channel, **self._queue_kwargs)
self._queue.declare()
self._consumer = Consumer(channel, [self._queue],
callbacks=[self._callback])
self._consumer.consume()
def Connect(self):
self.connection = Connection(
hostname=self.host,
port=self.port,
userid=self.usr,
password=self.psw,
virtual_host=self.virtual_host,
transport='kombu.transport.pyamqp:Transport')
self.channel = self.connection.channel()
self.producer = Producer(self.channel)
self.task_count = 0
if self.needresult:
self.back_queue = Queue(name=self.result_queuename,
auto_delete=True,
exclusive=True,
durable=False)
self.consumer = Consumer(self.channel,
self.back_queue,
no_ack=True)
self.consumer.qos(prefetch_count=1)
self.consumer.register_callback(self.on_response)
self.callback_queue = self.consumer.queues[0].name
self.consumer.consume()
else:
self.callback_queue = None
def test_on_message_is_sending_to_reply_queue(self, conn):
ret_result = 'foooo'
class Foo(A):
class state:
def bar(self, my_bar):
return ret_result
a = Foo(conn)
ticket = uuid()
delivery_tag = uuid()
body, message = get_encoded_test_message('bar', {'my_bar': 'bar_arg'},
A.__class__.__name__,
reply_to=ticket,
delivery_tag=delivery_tag)
# Set up a reply queue to read from
# reply_q and reply_exchange should be set the sender
a.reply_exchange = a.reply_exchange.bind(a.connection.default_channel)
maybe_declare(a.reply_exchange)
reply_q = a.get_reply_queue(ticket)
reply_q(a.connection.default_channel).declare()
a._on_message(body, message)
a_con = Consumer(conn.channel(), reply_q)
self.assertNextMsgDataEqual(a_con, {'ok': ret_result})
def get_consumers(self, _, default_channel):
self._default_channel = default_channel
consumers = []
exchange = Exchange(self._exchange,
type=self._exchange_type,
durable=True,
auto_delete=False)
for _routing_key, handles in self.handler.handle_map.items():
if not isinstance(handles, list):
handles = [handles]
for handle in handles:
channel = default_channel.connection.channel()
self._channels[id(channel)] = channel
_queue = '{}.{}'.format(handle.__module__, handle.__name__)
consumers.append(
Consumer(channel=channel,
queues=[
Queue(_queue,
exchange,
routing_key=_routing_key,
durable=True,
auto_delete=False)
],
callbacks=[partial(self.on_message, handle)],
on_decode_error=self.on_decode_error))
return consumers
def test_publish__consume(self):
connection = BrokerConnection(transport=Transport)
channel = connection.channel()
producer = Producer(channel, self.exchange, routing_key="test_Redis")
consumer = Consumer(channel, self.queue)
producer.publish({"hello2": "world2"})
_received = []
def callback(message_data, message):
_received.append(message_data)
message.ack()
consumer.register_callback(callback)
consumer.consume()
self.assertIn(channel, channel.connection.cycle._channels)
try:
connection.drain_events(timeout=1)
self.assertTrue(_received)
self.assertRaises(socket.timeout,
connection.drain_events,
timeout=0.01)
finally:
channel.close()
def test_decode_error(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
consumer.channel.throw_decode_error = True
with self.assertRaises(ValueError):
consumer._receive_callback({'foo': 'bar'})
def get_consumers(self, _, channel):
consumer = Consumer(channel,
queues=[self.provider.queue],
accept=self.accept,
no_ack=False,
callbacks=[self.provider.handle_message])
consumer.qos(prefetch_count=self.prefetch_count)
return [consumer]
def test_set_callbacks(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
callbacks = [lambda x, y: x,
lambda x, y: x]
consumer = Consumer(channel, queue, auto_declare=True,
callbacks=callbacks)
self.assertEqual(consumer.callbacks, callbacks)
def test_decode_error(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
consumer.channel.throw_decode_error = True
with self.assertRaises(ValueError):
consumer._receive_callback({"foo": "bar"})
def consumer(self, wakeup=True):
"""Create event consumer."""
consumer = Consumer(self.connection, queues=[self.queue], no_ack=True)
consumer.register_callback(self._receive)
with consumer:
if wakeup:
self.wakeup_workers(channel=consumer.channel)
yield consumer
def test_consume__cancel(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.cancel()
self.assertIn("basic_cancel", channel)
self.assertFalse(consumer._active_tags)
def test_purge(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
b2 = Queue('qname2', self.exchange, 'rkey')
b3 = Queue('qname3', self.exchange, 'rkey')
b4 = Queue('qname4', self.exchange, 'rkey')
consumer = Consumer(channel, [b1, b2, b3, b4], auto_declare=True)
consumer.purge()
self.assertEqual(channel.called.count('queue_purge'), 4)
def test_revive(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
channel2 = self.connection.channel()
consumer.revive(channel2)
self.assertIs(consumer.channel, channel2)
self.assertIs(consumer.queues[0].channel, channel2)
self.assertIs(consumer.queues[0].exchange.channel, channel2)
def test_purge(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
b2 = Queue("qname2", self.exchange, "rkey")
b3 = Queue("qname3", self.exchange, "rkey")
b4 = Queue("qname4", self.exchange, "rkey")
consumer = Consumer(channel, [b1, b2, b3, b4], auto_declare=True)
consumer.purge()
self.assertEqual(channel.called.count("queue_purge"), 4)
def test___enter____exit__(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
context = consumer.__enter__()
self.assertIs(context, consumer)
self.assertTrue(consumer._active_tags)
consumer.__exit__()
self.assertIn("basic_cancel", channel)
self.assertFalse(consumer._active_tags)
def get_consumers(self, Consumer, channel):
consume_from = list()
for queue in settings.CONSUME_FROM:
consume_from.append(self.queues.get(queue))
consumer = [
Consumer(queues=consume_from,
on_message=self.on_message,
prefetch_count=settings.PREFETCH_COUNT)
]
return consumer
def test_basic_reject__requeue(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
consumer.register_callback(callback)
consumer._receive_callback({"foo": "bar"})
self.assertIn("basic_reject:requeue", channel)
def test_basic_reject__requeue(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
consumer.register_callback(callback)
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject:requeue', channel)
def test___enter____exit__(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
context = consumer.__enter__()
self.assertIs(context, consumer)
self.assertTrue(consumer._active_tags)
res = consumer.__exit__(None, None, None)
self.assertFalse(res)
self.assertIn('basic_cancel', channel)
self.assertFalse(consumer._active_tags)
def test_basic_ack_twice(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.ack()
message.ack()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({"foo": "bar"})
def test_basic_reject_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
message.reject()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject', channel)
def init(host, port, virtual_host, usr, psw, queue_name):
global connection, channel, producer, task_queue, consumer
connection = Connection(hostname=host,
port=port,
userid=usr,
password=psw,
virtual_host=virtual_host)
channel = connection.channel()
producer = Producer(channel)
task_queue = Queue(queue_name, durable=True)
consumer = Consumer(channel, task_queue, no_ack=False)
consumer.qos(prefetch_count=1)
consumer.register_callback(RequestCallBack)
def main():
cfg = {
'hostname': 'localhost',
'userid': 'guest',
'password': '******',
'virtual_host': '/',
'port': 5672
}
transport = 'pika'
#transport = 'librabbitmq'
connection = BrokerConnection(transport=transport, **cfg)
connection.connect()
cfg = {
'name': 'simple-test-1',
'auto_delete': True,
'durable': False,
'delivery_mode': 'transient'
}
channel = connection.channel()
exchange = Exchange(channel=channel, **cfg)
#exchange = exchange_def(channel)
routing_key = 'simple-test-1-route'
queue = Queue(exchange=exchange, routing_key=routing_key, **cfg)
channel = connection.channel()
producer = Producer(channel=channel,
exchange=exchange,
routing_key=routing_key)
channel = connection.channel()
consumer = Consumer(channel=channel, queues=[queue], callbacks=[receive])
consumer.consume()
def serve_forever():
while True:
#print 'drain'
#gevent.sleep(0.0001)
connection.drain_events(timeout=1)
def publish_forever():
while True:
producer.publish(loremIpsum)
gevent.sleep(0.0001)
#g1, g2 = gevent.spawn(publish_forever), gevent.spawn(serve_forever)
g2 = gevent.spawn(serve_forever)
g1 = gevent.spawn(publish_forever)
gevent.joinall([g1, g2])
def consumer(self):
"""Create event consumer.
.. warning::
This creates a new channel that needs to be closed
by calling `consumer.channel.close()`.
"""
consumer = Consumer(self.connection.channel(),
queues=[self.queue],
no_ack=True)
consumer.register_callback(self._receive)
return consumer
def consume(self, connection, queue_name):
self._consume_body = None
self._consume_message = None
with Consumer(channel=connection.default_channel,
queues=[Queue(name=queue_name)],
callbacks=[self._consume_callback],
no_ack=True,
auto_declare=False) as consumer:
try:
connection.drain_events(timeout=3)
yield self._consume_body, self._consume_message
except socket.timeout:
yield None, None
