def __init__(self,
hostname='localhost',
port=5672,
userid="collectoruser",
password="******",
**options):
conn = BrokerConnection(hostname=hostname,
port=port,
userid=userid,
password=password,
virtual_host='collectorvhost')
try:
self.publisher = Publisher(connection=conn,
exchange="collector.response",
exchange_type='direct',
routing_key="response",
serializer="json")
except Exception as ex:
raise Worker.ConnectionException(ex)
self.consumer = Consumer(connection=conn,
queue="feed",
exchange_type='topic',
exchange="collector",
routing_key="collector.*.*")
def consume(self, queue, limit=None, callback=None, auto_declare=False):
"""consume messages in queue
queue - name of queue
limit - amount of messages to iterate through (default: no limit)
callback - method to call when a new message is received
must take two arguments: message_data, message
must send the acknowledgement: message.ack()
default: print message to stdout and send ack
auto_declare - automatically declare the queue (default: false)
"""
if not callback:
callback = _consume_callback
consumer = Consumer(connection=self.broker,
queue=queue,
auto_declare=auto_declare)
consumer.register_callback(callback)
for message in consumer.iterqueue(limit=limit, infinite=False):
consumer.receive(message.payload, message)
consumer.close()
def get_consumer(config, queue_name, routing_key):
return Consumer(connection=get_carrot_connection(config),
queue=queue_name,
routing_key=routing_key,
exchange=config.get('ckan.site_id'),
exchange_type=EXCHANGE_TYPE,
durable=True, auto_delete=False)
def create_consumer(self, **options):
queue = "%s%s" % (self.queue, self.nextq())
return Consumer(connection=self.conn,
queue=queue,
exchange=self.exchange,
routing_key=self.routing_key,
**options)
def get_consumer(queue_name, routing_key):
return Consumer(connection=get_carrot_connection(),
queue=queue_name,
routing_key=routing_key,
exchange=EXCHANGE_NAME,
exchange_type=EXCHANGE_TYPE,
durable=True,
auto_delete=False)
def create_README_consumer(amqpconn):
from carrot.messaging import Consumer
consumer = Consumer(connection=amqpconn,
queue=README_QUEUE, exchange=README_EXCHANGE,
routing_key=README_ROUTING_KEY)
tcallbacks = CallbacksTestable()
consumer.register_callback(tcallbacks.import_feed)
consumer.register_callback(tcallbacks.dump_message)
return consumer, tcallbacks
def get_consumer(queue_name, routing_key):
connection = get_carrot_connection()
try:
return Consumer(connection=connection,
queue=queue_name,
routing_key=routing_key,
exchange=EXCHANGE_NAME,
exchange_type=EXCHANGE_TYPE,
durable=True,
auto_delete=False)
except socket.error, e:
log.error('Error connecting to RabbitMQ with settings: %r',
connection.__dict__)
raise
def test_consumer_options(self):
opposite_defaults = {
"queue": "xyxyxyxy",
"exchange": "xyxyxyxy",
"routing_key": "xyxyxyxy",
"durable": False,
"exclusive": True,
"auto_delete": True,
"exchange_type": "topic",
}
consumer = Consumer(connection=self.conn, **opposite_defaults)
for opt_name, opt_value in opposite_defaults.items():
self.assertEquals(getattr(consumer, opt_name), opt_value)
consumer.close()
def get_consumer_set(connection, queues=None, **options):
"""Get the :class:`carrot.messaging.ConsumerSet`` for a queue
configuration.
Defaults to the queues in :const:`CELERY_QUEUES`.
"""
queues = queues or conf.get_queues()
cset = ConsumerSet(connection)
for queue_name, queue_options in queues.items():
queue_options = dict(queue_options)
queue_options["routing_key"] = queue_options.pop("binding_key", None)
consumer = Consumer(connection,
queue=queue_name,
backend=cset.backend,
**queue_options)
cset.consumers.append(consumer)
return cset
def test_with_statement(self):
with BrokerConnection(**test_connection_args()) as conn:
self.assertFalse(conn._closed)
with Publisher(connection=conn, exchange="F", routing_key="G") \
as publisher:
self.assertFalse(publisher._closed)
self.assertTrue(conn._closed)
self.assertTrue(publisher._closed)
with BrokerConnection(**test_connection_args()) as conn:
self.assertFalse(conn._closed)
with Consumer(connection=conn,
queue="E",
exchange="F",
routing_key="G") as consumer:
self.assertFalse(consumer._closed)
self.assertTrue(conn._closed)
self.assertTrue(consumer._closed)
def declare(self, exchange, exchange_type, binding="", queue=""):
"""declares the exchange, the queue and binds the queue to the exchange
exchange - exchange name
exchange_type - direct, topic, fanout
binding - binding to queue (optional)
queue - queue to bind to exchange using binding (optional)
"""
if (binding and not queue) or (queue and not binding):
if queue and not exchange_type == "fanout":
raise Error("binding and queue are not mutually exclusive")
consumer = Consumer(connection=self.broker,
exchange=exchange,
exchange_type=exchange_type,
routing_key=binding,
queue=queue)
consumer.declare()
consumer.close()
def request_with_response():
"""Example of requesting over AMQP from another process, not necessarily Python."""
qid = str(randrange(0, 999))
data = {'qid': qid, 'q': 'what time is it?'}
broker = ExampleBroker()
publisher = Publisher(
connection=broker.amqp_connection,
exchange=broker.exchange_name,
exchange_type="topic",
routing_key=broker.binding_key,
)
# declare test queue for receiving response message
backend = broker.amqp_connection.create_backend()
backend.queue_declare(
queue="test",
durable=False,
exclusive=False,
auto_delete=True,
)
backend.queue_bind("test", broker.exchange_name,
broker.response_routing_key % qid)
consumer = Consumer(
connection=broker.amqp_connection,
exchange=broker.exchange_name,
exchange_type="topic",
queue="test",
)
consumer.discard_all()
# send message
publisher.send(json.dumps(data))
# allow data to pass the wire
sleep(0.2)
# retrieve response
response = consumer.fetch()
payload = json.loads(response.payload)
print "Response from AMQP: %s" % payload
def __init__(self, exchange, queue, routing_key=None, debug=None):
"""
Initialise signal, conection, exchange, queue and run consumer
@return: Django Signal like object.
@param exchange: name of exchange for this signal
@type exchange: string
@param queue: name queue of this signal
@type queue: string
@param routing_key: name of routing_key betwen exchange and queue of this signal
@type routing_key: string
@param debug: debug flag
@type debug: bool
Example:
>>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1")
>>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2")
>>> amqp_signal_1.queue_bind(['q2'])
>>> def amqp_handler(sender, **kwargs):
print "AMPQ handler:", sender, kwargs
>>> amqp_signal_2.connect(amqp_handler, sender=None)
>>> amqp_signal_1.send("Hello world!")
"""
super(SignalAMQP, self).__init__(providing_args=["message"])
self.exchange = exchange
self.queue = queue
self.routing_key = routing_key
self.debug = debug is None and settings.DEBUG or debug
self.conn = DjangoBrokerConnection()
self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\
routing_key=self.routing_key)
self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\
exchange=self.exchange, routing_key=self.routing_key)
self.consumer.register_callback(self.callback)
self.cl = self.listen()
def get_responses(logger=logging):
connection = DjangoBrokerConnection()
consumer = Consumer(connection=connection,
exchange="collector.response",
queue="responses",
routing_key="response")
for message in consumer.iterqueue():
responses = message.payload
for resp in responses:
logger.debug("resp=%s" % resp)
try:
tag = Tag.objects.get(name=resp['name'])
tag.save_with_history(resp['current_value'])
except Exception as ex:
logger.error(ex)
#print "Could have saved '%s' for tag '%s'" % (resp['current_value'], tag.id,)
message.ack()
consumer.close()
connection.close()
def disabled_publisher_mandatory_flag_regr16(self):
"""
Test that the publisher "mandatory" flag
raises exceptions at appropriate times.
"""
routing_key = 'black_hole'
assert self.conn.connection is not None
message = {'foo': 'mandatory'}
# sanity check cleanup from last test
assert not self.create_consumer().backend.queue_exists(routing_key)
publisher = self.create_publisher()
# this should just get discarded silently, it's not mandatory
publisher.send(message, routing_key=routing_key, mandatory=False)
# This raises an unspecified exception because there is no queue to
# deliver to
self.assertRaises(Exception,
publisher.send,
message,
routing_key=routing_key,
mandatory=True)
# now bind a queue to it
consumer = Consumer(connection=self.conn,
queue=routing_key,
exchange=self.exchange,
routing_key=routing_key,
durable=False,
exclusive=True)
# check that it exists
assert self.create_consumer().backend.queue_exists(routing_key)
# this should now get routed to our consumer with no exception
publisher.send(message, routing_key=routing_key, mandatory=True)
def test_consumer_interface(self):
to_send = ['No', 'soup', 'for', 'you!']
messages = []
def cb(message_data, message):
messages.append(message_data)
conn = BrokerConnection(backend_cls='memory')
consumer = Consumer(connection=conn,
queue="test",
exchange="test",
routing_key="test")
consumer.register_callback(cb)
publisher = Publisher(connection=conn,
exchange="test",
routing_key="test")
for i in to_send:
publisher.send(i)
it = consumer.iterconsume()
for i in range(len(to_send)):
it.next()
self.assertEqual(messages, to_send)
def main():
logging.basicConfig(level=logging.INFO)
args = parser.parse_args()
conn = BrokerConnection(
hostname=args.hostname,
virtual_host=args.vhost,
userid=args.user,
password=args.password,
)
consumer = Consumer(
auto_declare=False,
connection=conn,
exclusive=True,
auto_delete=True,
)
try:
logging.info("Creating exchange: %s" % args.exchange)
consumer.backend.exchange_declare(exchange=args.exchange,
type="topic",
durable=False,
auto_delete=True)
except Exception, e:
logging.warning("Failed to create exchange: %s" % e)
from carrot.connection import BrokerConnection
from carrot.messaging import Consumer
if __name__ == '__main__':
connection = BrokerConnection(hostname='localhost',
port=5672,
userid='guest',
password='******',
virtual_host=None)
consumer = Consumer(connection=connection,
queue="transactions",
exchange="transactions")
def print_message(message_data, message):
print(message_data)
message.ack()
consumer.register_callback(print_message)
consumer.wait()
#!/usr/bin/env python
from carrot.messaging import Consumer
from carrot.connection import BrokerConnection
conn = BrokerConnection(hostname='localhost',
port=5672,
userid='test',
password='******',
virtual_host='test')
consumer = Consumer(connection=conn,
queue="feed",
exchange_type='topic',
exchange="fst",
routing_key="easyip.*")
#publisher = Publisher(connection=conn, exchange="fst", exchange_type='topic',
# routing_key="easyip.incomming", serializer="json")
def get_using_callback(message_data, message):
print "%s" % message_data
message.ack()
consumer.register_callback(get_using_callback)
consumer.wait()
def create_consumer(self, **options):
return Consumer(connection=self.conn,
queue=self.queue,
exchange=self.exchange,
routing_key=self.routing_key,
**options)
def listen(self, callback=None):
# One can optionally provide a callback to listen (if it wasn't already)
if callback:
self.callback = callback
# Make suere there is an exchange given
if not self.exchange:
raise InvalidExchange(self.exchange)
# Make sure there is a topic given
if not self.topic:
raise InvalidTopic(self.topic)
# Make sure there is an applabel given
if self.durable and not self.applabel:
raise InvalidAppLabel('Durable consumers must have an applabel')
# Make sure there is a callback given
if not self.callback or not hasattr(self.callback, '__call__'):
raise InvalidCallback(self.callback)
# Connect to the broker if we haven't already
if not self.connection:
self.connect()
# Set up our broker consumer
self.consumer = Consumer(connection=self.connection,
queue=self.applabel,
exchange=self.exchange,
exchange_type="topic",
auto_declare=False,
routing_key=self.topic)
# We need to manually create / declare the queue
self.consumer.backend.queue_declare(
queue=self.applabel,
durable=self.durable,
exclusive=False,
auto_delete=not self.durable,
arguments=self.consumer.queue_arguments,
warn_if_exists=False)
# No need to manually create the exchange, as the producer creates it
# and we expect it to just be there
# We support multiple bindings if we were given an array for the topic
if not isinstance(self.topic, list):
self.topic = [self.topic]
# We need to bind the queue to the exchange with the specified keys
if self.consumer.queue:
for routing_key in self.topic:
self.consumer.backend.queue_bind(queue=self.consumer.queue,
exchange=self.exchange,
routing_key=routing_key)
if self.heartbeat:
self.consumer.backend.queue_bind(
queue=self.consumer.queue,
exchange='org.mozilla.exchange.pulse.test',
routing_key='heartbeat')
# Register the callback the user wants
self.consumer.register_callback(self.callback)
# This blocks, and then calls the user callback every time a message
# comes in
self.consumer.wait()
# Likely never get here but can't hurt
self.disconnect()
def get_consumer(self, broker, **kwargs):
return Consumer(broker, **kwargs)
