def consume(self, queue, limit=None, callback=None, auto_declare=False):
"""consume messages in queue
queue - name of queue
limit - amount of messages to iterate through (default: no limit)
callback - method to call when a new message is received
must take two arguments: message_data, message
must send the acknowledgement: message.ack()
default: print message to stdout and send ack
auto_declare - automatically declare the queue (default: false)
"""
if not callback:
callback = _consume_callback
consumer = Consumer(connection=self.broker,
queue=queue,
auto_declare=auto_declare)
consumer.register_callback(callback)
for message in consumer.iterqueue(limit=limit, infinite=False):
consumer.receive(message.payload, message)
consumer.close()
def do_consume():
consumer = Consumer(ctx.broker, exchange='testx', queue='testq',
routing_key='testq', exclusive=True, durable=False)
consumer.register_callback(got_message_cb)
try:
consumer.wait(limit=1)
except StopIteration:
pass
except GreenletExit:
pass
finally:
consumer.close()
def get(self, exchange_name, queue_name, routing_key):
def msg_callback(message_data, message):
message.ack()
self.write(simplejson.dumps(message_data))
self.finish()
consumer = Consumer(connection=conn, queue=queue_name,
exchange=exchange_name, routing_key=routing_key)
consumer.register_callback(msg_callback)
try:
consumer.wait(1)
except StopIteration:
consumer.close()
def test_consumer_options(self):
opposite_defaults = {
"queue": "xyxyxyxy",
"exchange": "xyxyxyxy",
"routing_key": "xyxyxyxy",
"durable": False,
"exclusive": True,
"auto_delete": True,
"exchange_type": "topic",
}
consumer = Consumer(connection=self.conn, **opposite_defaults)
for opt_name, opt_value in opposite_defaults.items():
self.assertEquals(getattr(consumer, opt_name), opt_value)
consumer.close()
def test_consumer_options(self):
opposite_defaults = {
"queue": "xyxyxyxy",
"exchange": "xyxyxyxy",
"routing_key": "xyxyxyxy",
"durable": False,
"exclusive": True,
"auto_delete": True,
"exchange_type": "topic",
}
consumer = Consumer(connection=self.conn, **opposite_defaults)
for opt_name, opt_value in opposite_defaults.items():
self.assertEquals(getattr(consumer, opt_name), opt_value)
consumer.close()
def declare(self, exchange, exchange_type, binding="", queue=""):
"""declares the exchange, the queue and binds the queue to the exchange
exchange - exchange name
exchange_type - direct, topic, fanout
binding - binding to queue (optional)
queue - queue to bind to exchange using binding (optional)
"""
if (binding and not queue) or (queue and not binding):
if queue and not exchange_type == "fanout":
raise Error("binding and queue are not mutually exclusive")
consumer = Consumer(connection=self.broker,
exchange=exchange, exchange_type=exchange_type,
routing_key=binding, queue=queue)
consumer.declare()
consumer.close()
def declare(self, exchange, exchange_type, binding="", queue=""):
"""declares the exchange, the queue and binds the queue to the exchange
exchange - exchange name
exchange_type - direct, topic, fanout
binding - binding to queue (optional)
queue - queue to bind to exchange using binding (optional)
"""
if (binding and not queue) or (queue and not binding):
if queue and not exchange_type == "fanout":
raise Error("binding and queue are not mutually exclusive")
consumer = Consumer(connection=self.broker,
exchange=exchange,
exchange_type=exchange_type,
routing_key=binding,
queue=queue)
consumer.declare()
consumer.close()
def get_responses(logger=logging):
connection = DjangoBrokerConnection()
consumer = Consumer(connection=connection,
exchange="collector.response",
queue="responses",
routing_key="response")
for message in consumer.iterqueue():
responses = message.payload
for resp in responses:
logger.debug("resp=%s" % resp)
try:
tag = Tag.objects.get(name=resp['name'])
tag.save_with_history(resp['current_value'])
except Exception as ex:
logger.error(ex)
#print "Could have saved '%s' for tag '%s'" % (resp['current_value'], tag.id,)
message.ack()
consumer.close()
connection.close()
def main():
connection = BrokerConnection(
hostname = "localhost",
port = 5672,
userid = "test",
password = "******",
virtual_host = "test.com",
)
consumer = Consumer(
connection = connection,
queue = "messages",
exchange = "messages",
routing_key = "awesome",
)
consumer.register_callback(worker_callback)
try:
consumer.wait()
except KeyboardInterrupt:
consumer.close()
def get_responses(logger=logging):
connection = DjangoBrokerConnection()
consumer = Consumer(connection=connection,
exchange="collector.response",
queue="responses",
routing_key="response")
for message in consumer.iterqueue():
responses = message.payload
for resp in responses:
logger.debug("resp=%s" % resp )
try:
tag=Tag.objects.get(name=resp['name'])
tag.save_with_history(resp['current_value'])
except Exception as ex:
logger.error(ex)
#print "Could have saved '%s' for tag '%s'" % (resp['current_value'], tag.id,)
message.ack()
consumer.close()
connection.close()
def save_hits():
conn = DjangoAMQPConnection()
consumer = Consumer(connection=conn, queue='requestrequesthitqueue',
exchange='request',
routing_key='request.*', exchange_type='topic')
count = 0
messages = []
for message in consumer.iterqueue():
messages.append(message)
request = message.decode()
request.path = convert_unicode_to_string(request.path[:255])
request.data = convert_unicode_to_string(request.data[:255])
request.referer = convert_unicode_to_string(request.referer[:255])
request.user_agent =\
convert_unicode_to_string(request.user_agent[:255])
request.language = convert_unicode_to_string(request.language[:255])
request.save()
count += 1
logger.info("Saved {0} requests".format(count))
[m.ack() for m in messages]
logger.debug("Acknowledged all messages")
consumer.close()
conn.close()
def consume(self, queue, limit=None, callback=None, auto_declare=False):
"""consume messages in queue
queue - name of queue
limit - amount of messages to iterate through (default: no limit)
callback - method to call when a new message is received
must take two arguments: message_data, message
must send the acknowledgement: message.ack()
default: print message to stdout and send ack
auto_declare - automatically declare the queue (default: false)
"""
if not callback:
callback = _consume_callback
consumer = Consumer(connection=self.broker, queue=queue,
auto_declare=auto_declare)
consumer.register_callback(callback)
for message in consumer.iterqueue(limit=limit, infinite=False):
consumer.receive(message.payload, message)
consumer.close()
def test_consumer(self):
"""AMQP->RPC->AMQP
Send a AMQP message, and test RPCConsumer response.
Uses a mock RPC server that the consumer will call.
Check the consumer AMQP response with test consumer.
"""
class TestConsumer(IMessageBroker):
service_name = 'TestConsumer'
exchange_name = 'Test'
topic = 'test'
amqp_connection_settings = settings.AMQP_CONNECTION
def dispatch(self, message, request, qid):
""" AMQP -> RPC dispatcher.
"""
logging.info('dispatching AMQP -> RPC')
response = {}
if qid:
# call remote RPC
# this message has an qid so it expects a response
response['msg'] = test_rpc_service.push(request)
message.ack()
# return the response to AMQP, the caller should be listening ..
self.return_response(response,qid)
else:
# no qid, so do something stateless ..
print request
message.ack()
try:
consumer = TestConsumer()
self.assertEquals('test.request.*',consumer.binding_key)
self.assertEquals('test.response.%s',consumer.response_routing_key)
consumer.start()
test_rpc_service.start()
test_rpc_service.messages = []
# allow consumer to start
sleep(0.2)
self.assert_(not consumer.stopped)
self.assert_(consumer.isAlive())
# test variables
qid = str(randrange(0,999999))
jsondata = {'msg':'hello rpc'}
_publisher = Publisher(
connection=self.test_connection,
exchange='Test',
exchange_type="topic",
routing_key='test.request.'+qid,
)
# test channel
backend = self.test_connection.create_backend()
backend.queue_declare(
queue="test",
durable=False,
exclusive=False,
auto_delete=True,)
backend.queue_bind("test",'Test','test.response.'+qid)
_consumer = Consumer(
connection=self.test_connection,
exchange='Test',
exchange_type="topic",
queue="test",
)
_consumer.discard_all()
logging.debug('publishing JSON message to RPC')
data_on_the_wire = json.dumps({'q': jsondata, 'qid': qid})
_publisher.send(data_on_the_wire)
# allow data to pass the wire
sleep(0.2)
# retrieve dispatcher response
response = _consumer.fetch()
self.assert_(response, 'No response')
data = json.loads(response.payload)
self.assert_(len(data['msg']) > 0) # assert non-empty response
self.assertEquals('ok',data['msg'])
# check dispatcher RPC function
self.assert_(len(test_rpc_service.messages) > 0, 'Message did not arrive')
self.assertEquals(test_rpc_service.messages[0], jsondata)
finally:
try:
consumer.stop()
except:
pass
try:
_consumer.close()
except:
pass
test_rpc_service.stop()
self.assert_(consumer.stopped)
