class Worker(object): def __init__(self, hostname="localhost", port=5672, userid="collectoruser", password="******", **options): conn = BrokerConnection( hostname=hostname, port=port, userid=userid, password=password, virtual_host="collectorvhost" ) try: self.publisher = Publisher( connection=conn, exchange="collector.response", exchange_type="direct", routing_key="response", serializer="json", ) except Exception as ex: raise Worker.ConnectionException(ex) self.consumer = Consumer( connection=conn, queue="feed", exchange_type="topic", exchange="collector", routing_key="collector.*.*" ) def run(self): self.consumer.register_callback(self._get_request_callback) print "Waiting..." self.consumer.wait() def _get_request_callback(self, message_data, message): # print "message_data=%s" % message_data drivers = [] drivers.append(message_data) DRIVER_KEY = "driver_routing_key" TAGS_KEY = "tags" for obj in drivers: driver = None if DRIVER_KEY in obj and TAGS_KEY in obj: module_name = obj["driver_routing_key"] try: __import__(module_name) module = sys.modules[module_name] driver = module.Driver() except: print "could not import module %s" % module_name values = [] for tag in obj["tags"]: if driver: value = driver.read_value(tag["device"], tag["address"]) else: value = "BAD_DRIVER" values.append({"name": tag["name"], "current_value": value, "read_at": "2009-01-01"}) self.publisher.send(values) else: print "Badly formated request %s" % obj message.ack() class WorkerException(Exception): pass class ConnectionException(WorkerException): pass
def create_README_consumer(amqpconn): from carrot.messaging import Consumer consumer = Consumer(connection=amqpconn, queue=README_QUEUE, exchange=README_EXCHANGE, routing_key=README_ROUTING_KEY) tcallbacks = CallbacksTestable() consumer.register_callback(tcallbacks.import_feed) consumer.register_callback(tcallbacks.dump_message) return consumer, tcallbacks
def consumer(self): conf = self.app.conf consumer = Consumer(self.connection, queue=conf.CELERY_EVENT_QUEUE, exchange=conf.CELERY_EVENT_EXCHANGE, exchange_type=conf.CELERY_EVENT_EXCHANGE_TYPE, routing_key=conf.CELERY_EVENT_ROUTING_KEY, no_ack=True) consumer.register_callback(self._receive) return consumer
def consume(self, queue, limit=None, callback=None, auto_declare=False): """consume messages in queue queue - name of queue limit - amount of messages to iterate through (default: no limit) callback - method to call when a new message is received must take two arguments: message_data, message must send the acknowledgement: message.ack() default: print message to stdout and send ack auto_declare - automatically declare the queue (default: false) """ if not callback: callback = _consume_callback consumer = Consumer(connection=self.broker, queue=queue, auto_declare=auto_declare) consumer.register_callback(callback) for message in consumer.iterqueue(limit=limit, infinite=False): consumer.receive(message.payload, message) consumer.close()
def _init_consumer(self): Consumer.__init__(self, connection=self.amqp_connection, exchange=self.exchange_name, exchange_type="topic", queue=self.req_queue_name, ) # register the callback method self.register_callback(self._request_filter) # calls "dispatch" self._stop = Event() self._stop.set() # set to stopped state
def test_consumer_options(self): opposite_defaults = { "queue": "xyxyxyxy", "exchange": "xyxyxyxy", "routing_key": "xyxyxyxy", "durable": False, "exclusive": True, "auto_delete": True, "exchange_type": "topic", } consumer = Consumer(connection=self.conn, **opposite_defaults) for opt_name, opt_value in opposite_defaults.items(): self.assertEquals(getattr(consumer, opt_name), opt_value) consumer.close()
def create_consumer(self, **options): queue = "%s%s" % (self.queue, self.nextq()) return Consumer(connection=self.conn, queue=queue, exchange=self.exchange, routing_key=self.routing_key, **options)
def get_consumer(config, queue_name, routing_key): return Consumer(connection=get_carrot_connection(config), queue=queue_name, routing_key=routing_key, exchange=config.get('ckan.site_id'), exchange_type=EXCHANGE_TYPE, durable=True, auto_delete=False)
def declare(self, exchange, exchange_type, binding="", queue=""): """declares the exchange, the queue and binds the queue to the exchange exchange - exchange name exchange_type - direct, topic, fanout binding - binding to queue (optional) queue - queue to bind to exchange using binding (optional) """ if (binding and not queue) or (queue and not binding): if queue and not exchange_type == "fanout": raise Error("binding and queue are not mutually exclusive") consumer = Consumer(connection=self.broker, exchange=exchange, exchange_type=exchange_type, routing_key=binding, queue=queue) consumer.declare() consumer.close()
def get_consumer(queue_name, routing_key): return Consumer(connection=get_carrot_connection(), queue=queue_name, routing_key=routing_key, exchange=EXCHANGE_NAME, exchange_type=EXCHANGE_TYPE, durable=True, auto_delete=False)
def test_consumer_interface(self): to_send = ['No', 'soup', 'for', 'you!'] messages = [] def cb(message_data, message): messages.append(message_data) conn = BrokerConnection(backend_cls='memory') consumer = Consumer(connection=conn, queue="test", exchange="test", routing_key="test") consumer.register_callback(cb) publisher = Publisher(connection=conn, exchange="test", routing_key="test") for i in to_send: publisher.send(i) it = consumer.iterconsume() for i in range(len(to_send)): it.next() self.assertEqual(messages, to_send)
def request_with_response(): """Example of requesting over AMQP from another process, not necessarily Python.""" qid = str(randrange(0,999)) data = {'qid': qid, 'q': 'what time is it?'} broker = ExampleBroker() publisher = Publisher( connection = broker.amqp_connection, exchange = broker.exchange_name, exchange_type = "topic", routing_key = broker.binding_key, ) # declare test queue for receiving response message backend = broker.amqp_connection.create_backend() backend.queue_declare( queue="test", durable=False, exclusive=False, auto_delete=True,) backend.queue_bind( "test", broker.exchange_name, broker.response_routing_key % qid) consumer = Consumer( connection=broker.amqp_connection, exchange=broker.exchange_name, exchange_type="topic", queue="test", ) consumer.discard_all() # send message publisher.send(json.dumps(data)) # allow data to pass the wire sleep(0.2) # retrieve response response = consumer.fetch() payload = json.loads(response.payload) print "Response from AMQP: %s" % payload
def request_with_response(): """Example of requesting over AMQP from another process, not necessarily Python.""" qid = str(randrange(0, 999)) data = {'qid': qid, 'q': 'what time is it?'} broker = ExampleBroker() publisher = Publisher( connection=broker.amqp_connection, exchange=broker.exchange_name, exchange_type="topic", routing_key=broker.binding_key, ) # declare test queue for receiving response message backend = broker.amqp_connection.create_backend() backend.queue_declare( queue="test", durable=False, exclusive=False, auto_delete=True, ) backend.queue_bind("test", broker.exchange_name, broker.response_routing_key % qid) consumer = Consumer( connection=broker.amqp_connection, exchange=broker.exchange_name, exchange_type="topic", queue="test", ) consumer.discard_all() # send message publisher.send(json.dumps(data)) # allow data to pass the wire sleep(0.2) # retrieve response response = consumer.fetch() payload = json.loads(response.payload) print "Response from AMQP: %s" % payload
def __init__(self, exchange, queue, routing_key=None, debug=None): """ Initialise signal, conection, exchange, queue and run consumer @return: Django Signal like object. @param exchange: name of exchange for this signal @type exchange: string @param queue: name queue of this signal @type queue: string @param routing_key: name of routing_key betwen exchange and queue of this signal @type routing_key: string @param debug: debug flag @type debug: bool Example: >>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1") >>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2") >>> amqp_signal_1.queue_bind(['q2']) >>> def amqp_handler(sender, **kwargs): print "AMPQ handler:", sender, kwargs >>> amqp_signal_2.connect(amqp_handler, sender=None) >>> amqp_signal_1.send("Hello world!") """ super(SignalAMQP, self).__init__(providing_args=["message"]) self.exchange = exchange self.queue = queue self.routing_key = routing_key self.debug = debug is None and settings.DEBUG or debug self.conn = DjangoBrokerConnection() self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\ routing_key=self.routing_key) self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\ exchange=self.exchange, routing_key=self.routing_key) self.consumer.register_callback(self.callback) self.cl = self.listen()
class AmqpMessageProcessor(object): purge_every = datetime.timedelta(minutes=10) conn_lifetime = datetime.timedelta(minutes=getattr(settings, "ACCESS_KEY_TIMEOUT_MINUTES", 2)) def __init__(self, connection): self.pygo_mp = PyGoWaveClientMessageProcessor() self.consumer = Consumer( connection, queue="wavelet_rpc_singlethread", exchange="wavelet.topic", routing_key="#.#.clientop", exchange_type="topic", serializer="json", auto_ack=True, ) self.consumer.register_callback(self.receive) self.publisher = Publisher( connection, exchange="wavelet.direct", exchange_type="direct", delivery_mode=1, serializer="json" ) self.pygo_mp.purge_connections() self.next_purge = datetime.datetime.now() + self.purge_every def wait(self, limit=None): self.consumer.wait(limit) def send(self, routing_key, message_data): self.publisher.send(message_data, routing_key=routing_key, delivery_mode=1) def receive(self, message_data, message): routing_key = message.amqp_message.routing_key msg_dict = self.pygo_mp.process(routing_key, message_data) for out_rkey, messages in msg_dict.iteritems(): self.send(out_rkey, messages) # Cleanup time? if datetime.datetime.now() > self.next_purge: self.pygo_mp.purge_connections() self.next_purge = datetime.datetime.now() + self.purge_every
def get_responses(logger=logging): connection = DjangoBrokerConnection() consumer = Consumer(connection=connection, exchange="collector.response", queue="responses", routing_key="response") for message in consumer.iterqueue(): responses = message.payload for resp in responses: logger.debug("resp=%s" % resp) try: tag = Tag.objects.get(name=resp['name']) tag.save_with_history(resp['current_value']) except Exception as ex: logger.error(ex) #print "Could have saved '%s' for tag '%s'" % (resp['current_value'], tag.id,) message.ack() consumer.close() connection.close()
def get_responses(logger=logging): connection = DjangoBrokerConnection() consumer = Consumer(connection=connection, exchange="collector.response", queue="responses", routing_key="response") for message in consumer.iterqueue(): responses = message.payload for resp in responses: logger.debug("resp=%s" % resp ) try: tag=Tag.objects.get(name=resp['name']) tag.save_with_history(resp['current_value']) except Exception as ex: logger.error(ex) #print "Could have saved '%s' for tag '%s'" % (resp['current_value'], tag.id,) message.ack() consumer.close() connection.close()
def do_consume(): consumer = Consumer(ctx.broker, exchange='testx', queue='testq', routing_key='testq', exclusive=True, durable=False) consumer.register_callback(got_message_cb) try: consumer.wait(limit=1) except StopIteration: pass except GreenletExit: pass finally: consumer.close()
def get_consumer(queue_name, routing_key): connection = get_carrot_connection() try: return Consumer(connection=connection, queue=queue_name, routing_key=routing_key, exchange=EXCHANGE_NAME, exchange_type=EXCHANGE_TYPE, durable=True, auto_delete=False) except socket.error, e: log.error('Error connecting to RabbitMQ with settings: %r', connection.__dict__) raise
def save_hits(): conn = DjangoAMQPConnection() consumer = Consumer(connection=conn, queue='requestrequesthitqueue', exchange='request', routing_key='request.*', exchange_type='topic') count = 0 messages = [] for message in consumer.iterqueue(): messages.append(message) request = message.decode() request.path = convert_unicode_to_string(request.path[:255]) request.data = convert_unicode_to_string(request.data[:255]) request.referer = convert_unicode_to_string(request.referer[:255]) request.user_agent =\ convert_unicode_to_string(request.user_agent[:255]) request.language = convert_unicode_to_string(request.language[:255]) request.save() count += 1 logger.info("Saved {0} requests".format(count)) [m.ack() for m in messages] logger.debug("Acknowledged all messages") consumer.close() conn.close()
def get(self, exchange_name, queue_name, routing_key): def msg_callback(message_data, message): message.ack() self.write(simplejson.dumps(message_data)) self.finish() consumer = Consumer(connection=conn, queue=queue_name, exchange=exchange_name, routing_key=routing_key) consumer.register_callback(msg_callback) try: consumer.wait(1) except StopIteration: consumer.close()
def get_consumer_set(connection, queues=None, **options): """Get the :class:`carrot.messaging.ConsumerSet`` for a queue configuration. Defaults to the queues in :const:`CELERY_QUEUES`. """ queues = queues or conf.get_queues() cset = ConsumerSet(connection) for queue_name, queue_options in queues.items(): queue_options = dict(queue_options) queue_options["routing_key"] = queue_options.pop("binding_key", None) consumer = Consumer(connection, queue=queue_name, backend=cset.backend, **queue_options) cset.consumers.append(consumer) return cset
def __init__(self, connection): self.pygo_mp = PyGoWaveClientMessageProcessor() self.consumer = Consumer( connection, queue="wavelet_rpc_singlethread", exchange="wavelet.topic", routing_key="#.#.clientop", exchange_type="topic", serializer="json", auto_ack=True, ) self.consumer.register_callback(self.receive) self.publisher = Publisher( connection, exchange="wavelet.direct", exchange_type="direct", delivery_mode=1, serializer="json" ) self.pygo_mp.purge_connections() self.next_purge = datetime.datetime.now() + self.purge_every
def test_with_statement(self): with BrokerConnection(**test_connection_args()) as conn: self.assertFalse(conn._closed) with Publisher(connection=conn, exchange="F", routing_key="G") \ as publisher: self.assertFalse(publisher._closed) self.assertTrue(conn._closed) self.assertTrue(publisher._closed) with BrokerConnection(**test_connection_args()) as conn: self.assertFalse(conn._closed) with Consumer(connection=conn, queue="E", exchange="F", routing_key="G") as consumer: self.assertFalse(consumer._closed) self.assertTrue(conn._closed) self.assertTrue(consumer._closed)
def __init__(self, hostname="localhost", port=5672, userid="collectoruser", password="******", **options): conn = BrokerConnection( hostname=hostname, port=port, userid=userid, password=password, virtual_host="collectorvhost" ) try: self.publisher = Publisher( connection=conn, exchange="collector.response", exchange_type="direct", routing_key="response", serializer="json", ) except Exception as ex: raise Worker.ConnectionException(ex) self.consumer = Consumer( connection=conn, queue="feed", exchange_type="topic", exchange="collector", routing_key="collector.*.*" )
def __init__(self, connection): self.consumer = Consumer( connection, queue="wavelet_rpc_singlethread", exchange="wavelet.topic", routing_key="#.#.clientop", exchange_type="topic", serializer="json", auto_ack=True, ) self.consumer.register_callback(self.receive) self.publisher = Publisher( connection, exchange="wavelet.direct", exchange_type="direct", delivery_mode=1, serializer="json", ) self.out_queue = {} self.purge_connections()
def disabled_publisher_mandatory_flag_regr16(self): """ Test that the publisher "mandatory" flag raises exceptions at appropriate times. """ routing_key = 'black_hole' assert self.conn.connection is not None message = {'foo': 'mandatory'} # sanity check cleanup from last test assert not self.create_consumer().backend.queue_exists(routing_key) publisher = self.create_publisher() # this should just get discarded silently, it's not mandatory publisher.send(message, routing_key=routing_key, mandatory=False) # This raises an unspecified exception because there is no queue to # deliver to self.assertRaises(Exception, publisher.send, message, routing_key=routing_key, mandatory=True) # now bind a queue to it consumer = Consumer(connection=self.conn, queue=routing_key, exchange=self.exchange, routing_key=routing_key, durable=False, exclusive=True) # check that it exists assert self.create_consumer().backend.queue_exists(routing_key) # this should now get routed to our consumer with no exception publisher.send(message, routing_key=routing_key, mandatory=True)
def main(): connection = BrokerConnection( hostname = "localhost", port = 5672, userid = "test", password = "******", virtual_host = "test.com", ) consumer = Consumer( connection = connection, queue = "messages", exchange = "messages", routing_key = "awesome", ) consumer.register_callback(worker_callback) try: consumer.wait() except KeyboardInterrupt: consumer.close()
def main(): logging.basicConfig(level=logging.INFO) args = parser.parse_args() conn = BrokerConnection( hostname=args.hostname, virtual_host=args.vhost, userid=args.user, password=args.password, ) consumer = Consumer( auto_declare=False, connection=conn, exclusive=True, auto_delete=True, ) try: logging.info("Creating exchange: %s" % args.exchange) consumer.backend.exchange_declare(exchange=args.exchange, type="topic", durable=False, auto_delete=True) except Exception, e: logging.warning("Failed to create exchange: %s" % e)
#!/usr/bin/env python from carrot.messaging import Consumer from carrot.connection import BrokerConnection conn = BrokerConnection(hostname='localhost', port=5672, userid='test', password='******', virtual_host='test') consumer = Consumer(connection=conn, queue="feed", exchange_type='topic', exchange="fst", routing_key="easyip.*") #publisher = Publisher(connection=conn, exchange="fst", exchange_type='topic', # routing_key="easyip.incomming", serializer="json") def get_using_callback(message_data, message): print "%s" % message_data message.ack() consumer.register_callback(get_using_callback) consumer.wait()
from carrot.connection import BrokerConnection from carrot.messaging import Consumer conn = BrokerConnection(hostname="localhost", port=61617, userid="admin", password="******", virtual_host="/") consumer = Consumer(connection=conn, queue="feed", exchange="feed", routing_key="importer") def import_feed_callback(message_data, message): feed_url = message_data["import_feed"] #print("Got feed import message for: %s" % feed_url) # something importing this feed url # import_feed(feed_url) message.ack() consumer.register_callback(import_feed_callback) consumer.wait() # Go into the consumer loop.
class SignalAMQP(Signal): def __init__(self, exchange, queue, routing_key=None, debug=None): """ Initialise signal, conection, exchange, queue and run consumer @return: Django Signal like object. @param exchange: name of exchange for this signal @type exchange: string @param queue: name queue of this signal @type queue: string @param routing_key: name of routing_key betwen exchange and queue of this signal @type routing_key: string @param debug: debug flag @type debug: bool Example: >>> amqp_signal_1 = SignalAMQP(exchange="test1", queue="q1") >>> amqp_signal_2 = SignalAMQP(exchange="test2", queue="q2") >>> amqp_signal_1.queue_bind(['q2']) >>> def amqp_handler(sender, **kwargs): print "AMPQ handler:", sender, kwargs >>> amqp_signal_2.connect(amqp_handler, sender=None) >>> amqp_signal_1.send("Hello world!") """ super(SignalAMQP, self).__init__(providing_args=["message"]) self.exchange = exchange self.queue = queue self.routing_key = routing_key self.debug = debug is None and settings.DEBUG or debug self.conn = DjangoBrokerConnection() self.publisher = Publisher(connection=self.conn, exchange=self.exchange, exchange_type="fanout",\ routing_key=self.routing_key) self.consumer = Consumer(connection=self.conn, queue=self.queue, exchange_type="fanout",\ exchange=self.exchange, routing_key=self.routing_key) self.consumer.register_callback(self.callback) self.cl = self.listen() def send(self, message, **kw): """ Transfer message to bus. Message can be any simple python type. """ self.publisher.send(message, **kw) def callback(self, message_data, message): """ Consumer callback function. Send Django singnal.""" try: sender = message_data['sender'] except: sender = None super(SignalAMQP, self).send(sender=sender, message_data=message_data, message=message) message.ack() if self.debug: print "AMPQ CALLBACK: sender=", sender, "messege=", message_data, message return True def get_backend(self): return self.conn.get_backend_cls()(DjangoBrokerConnection()) def queue_bind(self, queue_list): """ Bind another queues to current exchange """ routing_keys = [] backend = self.get_backend() for x in queue_list: backend.queue_bind(queue=x, exchange=self.exchange, \ routing_key='%s_%s' % (self.exchange, x)) routing_keys.append('%s_%s' % (self.exchange, x)) return routing_keys def listen(self): """ Run consumer loop thread """ cl = ConsumerLoop() cl.consumer = self.consumer cl.start() return cl def stop(self): """ Unactivate this signal """ self.conn.close()
def get_consumer(self, broker, **kwargs): return Consumer(broker, **kwargs)
class GenericConsumer(object): def __init__(self, config, exchange=None, connect=True, heartbeat=False, **kwargs): self.config = config self.exchange = exchange self.connection = None self.durable = False self.applabel = '' self.heartbeat = heartbeat for x in ['applabel', 'topic', 'callback', 'durable']: if x in kwargs: setattr(self, x, kwargs[x]) del kwargs[x] if connect: self.connect() # Sets vairables def configure(self, **kwargs): for x in kwargs: setattr(self, x, kwargs[x]) # Connect to the message broker def connect(self): if not self.connection: self.connection = BrokerConnection(hostname=self.config.host, port=self.config.port, userid=self.config.user, password=self.config.password, virtual_host=self.config.vhost) # Disconnect from the message broker def disconnect(self): if self.connection: self.connection.close() # Support purging messages that are already in the queue on the broker # TODO: I think this is only supported by the amqp backend def purge_existing_messages(self): # Make sure there is an applabel given if self.durable and not self.applabel: raise InvalidAppLabel('Durable consumers must have an applabel') # Purge the queue of existing messages self.connection.create_backend().queue_purge(self.applabel) # Blocks and calls the callback when a message comes into the queue # For info on one script listening to multiple channels, see # http://ask.github.com/carrot/changelog.html#id1 def listen(self, callback=None): # One can optionally provide a callback to listen (if it wasn't already) if callback: self.callback = callback # Make suere there is an exchange given if not self.exchange: raise InvalidExchange(self.exchange) # Make sure there is a topic given if not self.topic: raise InvalidTopic(self.topic) # Make sure there is an applabel given if self.durable and not self.applabel: raise InvalidAppLabel('Durable consumers must have an applabel') # Make sure there is a callback given if not self.callback or not hasattr(self.callback, '__call__'): raise InvalidCallback(self.callback) # Connect to the broker if we haven't already if not self.connection: self.connect() # Set up our broker consumer self.consumer = Consumer(connection=self.connection, queue=self.applabel, exchange=self.exchange, exchange_type="topic", auto_declare=False, routing_key=self.topic) # We need to manually create / declare the queue self.consumer.backend.queue_declare( queue=self.applabel, durable=self.durable, exclusive=False, auto_delete=not self.durable, arguments=self.consumer.queue_arguments, warn_if_exists=False) # No need to manually create the exchange, as the producer creates it # and we expect it to just be there # We support multiple bindings if we were given an array for the topic if not isinstance(self.topic, list): self.topic = [self.topic] # We need to bind the queue to the exchange with the specified keys if self.consumer.queue: for routing_key in self.topic: self.consumer.backend.queue_bind(queue=self.consumer.queue, exchange=self.exchange, routing_key=routing_key) if self.heartbeat: self.consumer.backend.queue_bind( queue=self.consumer.queue, exchange='org.mozilla.exchange.pulse.test', routing_key='heartbeat') # Register the callback the user wants self.consumer.register_callback(self.callback) # This blocks, and then calls the user callback every time a message # comes in self.consumer.wait() # Likely never get here but can't hurt self.disconnect()
from carrot.connection import BrokerConnection from carrot.messaging import Consumer if __name__ == '__main__': connection = BrokerConnection(hostname='localhost', port=5672, userid='guest', password='******', virtual_host=None) consumer = Consumer(connection=connection, queue="transactions", exchange="transactions") def print_message(message_data, message): print(message_data) message.ack() consumer.register_callback(print_message) consumer.wait()
def create_consumer(self, **options): return Consumer(connection=self.conn, queue=self.queue, exchange=self.exchange, routing_key=self.routing_key, **options)
def listen(self, callback=None): # One can optionally provide a callback to listen (if it wasn't already) if callback: self.callback = callback # Make suere there is an exchange given if not self.exchange: raise InvalidExchange(self.exchange) # Make sure there is a topic given if not self.topic: raise InvalidTopic(self.topic) # Make sure there is an applabel given if self.durable and not self.applabel: raise InvalidAppLabel('Durable consumers must have an applabel') # Make sure there is a callback given if not self.callback or not hasattr(self.callback, '__call__'): raise InvalidCallback(self.callback) # Connect to the broker if we haven't already if not self.connection: self.connect() # Set up our broker consumer self.consumer = Consumer(connection=self.connection, queue=self.applabel, exchange=self.exchange, exchange_type="topic", auto_declare=False, routing_key=self.topic) # We need to manually create / declare the queue self.consumer.backend.queue_declare( queue=self.applabel, durable=self.durable, exclusive=False, auto_delete=not self.durable, arguments=self.consumer.queue_arguments, warn_if_exists=False) # No need to manually create the exchange, as the producer creates it # and we expect it to just be there # We support multiple bindings if we were given an array for the topic if not isinstance(self.topic, list): self.topic = [self.topic] # We need to bind the queue to the exchange with the specified keys if self.consumer.queue: for routing_key in self.topic: self.consumer.backend.queue_bind(queue=self.consumer.queue, exchange=self.exchange, routing_key=routing_key) if self.heartbeat: self.consumer.backend.queue_bind( queue=self.consumer.queue, exchange='org.mozilla.exchange.pulse.test', routing_key='heartbeat') # Register the callback the user wants self.consumer.register_callback(self.callback) # This blocks, and then calls the user callback every time a message # comes in self.consumer.wait() # Likely never get here but can't hurt self.disconnect()
import jsonrpclib from carrot.connection import BrokerConnection from carrot.messaging import Consumer conn = BrokerConnection(hostname="localhost", port=5672, userid="guest", password="******", virtual_host="/") consumer = Consumer(connection=conn, queue="po_box", exchange="sorting_room", routing_key="jason") def amqp_callback(message_data, message): server = jsonrpclib.Server("http://localhost:8080") server.ping(message_data) print jsonrpclib.history.response message.ack() consumer.register_callback(amqp_callback) consumer.wait() # Go into the consumer loop. conn.close()
#!/usr/bin/env python from carrot.messaging import Consumer from carrot.connection import BrokerConnection conn=BrokerConnection(hostname='localhost', port=5672, userid='test', password='******', virtual_host='test') consumer = Consumer(connection=conn, queue="feed", exchange_type='topic', exchange="fst", routing_key="easyip.*") #publisher = Publisher(connection=conn, exchange="fst", exchange_type='topic', # routing_key="easyip.incomming", serializer="json") def get_using_callback(message_data, message): print "%s" % message_data message.ack() consumer.register_callback(get_using_callback) consumer.wait()
def test_consumer(self): """AMQP->RPC->AMQP Send a AMQP message, and test RPCConsumer response. Uses a mock RPC server that the consumer will call. Check the consumer AMQP response with test consumer. """ class TestConsumer(IMessageBroker): service_name = 'TestConsumer' exchange_name = 'Test' topic = 'test' amqp_connection_settings = settings.AMQP_CONNECTION def dispatch(self, message, request, qid): """ AMQP -> RPC dispatcher. """ logging.info('dispatching AMQP -> RPC') response = {} if qid: # call remote RPC # this message has an qid so it expects a response response['msg'] = test_rpc_service.push(request) message.ack() # return the response to AMQP, the caller should be listening .. self.return_response(response,qid) else: # no qid, so do something stateless .. print request message.ack() try: consumer = TestConsumer() self.assertEquals('test.request.*',consumer.binding_key) self.assertEquals('test.response.%s',consumer.response_routing_key) consumer.start() test_rpc_service.start() test_rpc_service.messages = [] # allow consumer to start sleep(0.2) self.assert_(not consumer.stopped) self.assert_(consumer.isAlive()) # test variables qid = str(randrange(0,999999)) jsondata = {'msg':'hello rpc'} _publisher = Publisher( connection=self.test_connection, exchange='Test', exchange_type="topic", routing_key='test.request.'+qid, ) # test channel backend = self.test_connection.create_backend() backend.queue_declare( queue="test", durable=False, exclusive=False, auto_delete=True,) backend.queue_bind("test",'Test','test.response.'+qid) _consumer = Consumer( connection=self.test_connection, exchange='Test', exchange_type="topic", queue="test", ) _consumer.discard_all() logging.debug('publishing JSON message to RPC') data_on_the_wire = json.dumps({'q': jsondata, 'qid': qid}) _publisher.send(data_on_the_wire) # allow data to pass the wire sleep(0.2) # retrieve dispatcher response response = _consumer.fetch() self.assert_(response, 'No response') data = json.loads(response.payload) self.assert_(len(data['msg']) > 0) # assert non-empty response self.assertEquals('ok',data['msg']) # check dispatcher RPC function self.assert_(len(test_rpc_service.messages) > 0, 'Message did not arrive') self.assertEquals(test_rpc_service.messages[0], jsondata) finally: try: consumer.stop() except: pass try: _consumer.close() except: pass test_rpc_service.stop() self.assert_(consumer.stopped)
class GenericConsumer(object): def __init__(self, config, exchange=None, connect=True, heartbeat=False, **kwargs): self.config = config self.exchange = exchange self.connection = None self.durable = False self.applabel = '' self.heartbeat = heartbeat for x in ['applabel','topic','callback','durable']: if x in kwargs: setattr(self, x, kwargs[x]) del kwargs[x] if connect: self.connect() # Sets vairables def configure(self, **kwargs): for x in kwargs: setattr(self, x, kwargs[x]) # Connect to the message broker def connect(self): if not self.connection: self.connection = BrokerConnection(hostname=self.config.host, port=self.config.port, userid=self.config.user, password=self.config.password, virtual_host=self.config.vhost) # Disconnect from the message broker def disconnect(self): if self.connection: self.connection.close() # Support purging messages that are already in the queue on the broker # TODO: I think this is only supported by the amqp backend def purge_existing_messages(self): # Make sure there is an applabel given if self.durable and not self.applabel: raise InvalidAppLabel('Durable consumers must have an applabel') # Purge the queue of existing messages self.connection.create_backend().queue_purge(self.applabel) # Blocks and calls the callback when a message comes into the queue # For info on one script listening to multiple channels, see # http://ask.github.com/carrot/changelog.html#id1 def listen(self, callback=None): # One can optionally provide a callback to listen (if it wasn't already) if callback: self.callback = callback # Make suere there is an exchange given if not self.exchange: raise InvalidExchange(self.exchange) # Make sure there is a topic given if not self.topic: raise InvalidTopic(self.topic) # Make sure there is an applabel given if self.durable and not self.applabel: raise InvalidAppLabel('Durable consumers must have an applabel') # Make sure there is a callback given if not self.callback or not hasattr(self.callback, '__call__'): raise InvalidCallback(self.callback) # Connect to the broker if we haven't already if not self.connection: self.connect() # Set up our broker consumer self.consumer = Consumer(connection=self.connection, queue=self.applabel, exchange=self.exchange, exchange_type="topic", auto_declare=False, routing_key=self.topic) # We need to manually create / declare the queue self.consumer.backend.queue_declare(queue=self.applabel, durable=self.durable, exclusive=False, auto_delete=not self.durable, arguments=self.consumer.queue_arguments, warn_if_exists=False) # No need to manually create the exchange, as the producer creates it # and we expect it to just be there # We support multiple bindings if we were given an array for the topic if not isinstance(self.topic, list): self.topic = [self.topic] # We need to bind the queue to the exchange with the specified keys if self.consumer.queue: for routing_key in self.topic: self.consumer.backend.queue_bind(queue=self.consumer.queue, exchange=self.exchange, routing_key=routing_key) if self.heartbeat: self.consumer.backend.queue_bind(queue=self.consumer.queue, exchange='org.mozilla.exchange.pulse.test', routing_key='heartbeat') # Register the callback the user wants self.consumer.register_callback(self.callback) # This blocks, and then calls the user callback every time a message # comes in self.consumer.wait() # Likely never get here but can't hurt self.disconnect()
def listen(self, callback=None): # One can optionally provide a callback to listen (if it wasn't already) if callback: self.callback = callback # Make suere there is an exchange given if not self.exchange: raise InvalidExchange(self.exchange) # Make sure there is a topic given if not self.topic: raise InvalidTopic(self.topic) # Make sure there is an applabel given if self.durable and not self.applabel: raise InvalidAppLabel('Durable consumers must have an applabel') # Make sure there is a callback given if not self.callback or not hasattr(self.callback, '__call__'): raise InvalidCallback(self.callback) # Connect to the broker if we haven't already if not self.connection: self.connect() # Set up our broker consumer self.consumer = Consumer(connection=self.connection, queue=self.applabel, exchange=self.exchange, exchange_type="topic", auto_declare=False, routing_key=self.topic) # We need to manually create / declare the queue self.consumer.backend.queue_declare(queue=self.applabel, durable=self.durable, exclusive=False, auto_delete=not self.durable, arguments=self.consumer.queue_arguments, warn_if_exists=False) # No need to manually create the exchange, as the producer creates it # and we expect it to just be there # We support multiple bindings if we were given an array for the topic if not isinstance(self.topic, list): self.topic = [self.topic] # We need to bind the queue to the exchange with the specified keys if self.consumer.queue: for routing_key in self.topic: self.consumer.backend.queue_bind(queue=self.consumer.queue, exchange=self.exchange, routing_key=routing_key) if self.heartbeat: self.consumer.backend.queue_bind(queue=self.consumer.queue, exchange='org.mozilla.exchange.pulse.test', routing_key='heartbeat') # Register the callback the user wants self.consumer.register_callback(self.callback) # This blocks, and then calls the user callback every time a message # comes in self.consumer.wait() # Likely never get here but can't hurt self.disconnect()
class PyGoWaveMessageProcessor(object): """ Handle all incoming messages. Routing key structure: <participant_conn_guid>.<wavelet_id>.[waveop|clientop] participant_conn_guid is created every time the user enters the wave viewer. Multiple instances of the wave viewer can be opened per user at the same time. A participant can "listen" to messages from multiple wavlets, by subscribing to multiple routing keys. In this scenario, a message may be sent to a queue, that the participant has not subscribed to. This is intended and it's RabbitMQ's job to drop the message. This keeps things simple for now and may be changed in the future. Message comsumers can be optimized in a multi-threading environment. If one consumer starts handling messages of a particular wavelet, it should block others from handling them. Some messages are handled synchronously (i.e. the client does not perform any actions and waits for the server's response). Those are in particular: WAVELET_ADD_PARTICIPANT """ purge_every = datetime.timedelta(minutes=10) conn_min_lifetime = datetime.timedelta(minutes=getattr(settings, "ACCESS_KEY_TIMEOUT_MINUTES", 2)) def __init__(self, connection): self.consumer = Consumer( connection, queue="wavelet_rpc_singlethread", exchange="wavelet.topic", routing_key="#.#.clientop", exchange_type="topic", serializer="json", auto_ack=True, ) self.consumer.register_callback(self.receive) self.publisher = Publisher( connection, exchange="wavelet.direct", exchange_type="direct", delivery_mode=1, serializer="json", ) self.out_queue = {} self.purge_connections() def broadcast(self, wavelet, type, property, except_connections=[]): """ Send messages to all participants. `wavelet` must be a Wavelet object. `except_connections` is a list of ParticipantConn objects to be excluded from the broadcast. """ msg_dict = { "type": type, "property": property } logger.debug("Broadcasting Message:\n" + repr(msg_dict)) for p in wavelet.participants.all(): for conn in p.connections.all(): if not conn in except_connections: if self.out_queue.has_key(conn.rx_key): self.out_queue[conn.rx_key].append(msg_dict) else: self.out_queue[conn.rx_key] = [msg_dict] def emit(self, to, type, property, except_connections=[]): """ Collect messages to be sent. `to` must be a ParticipantConn object. """ msg_dict = { "type": type, "property": property } logger.debug("Emiting Message to %s/%d:\n%s" % (to.participant.name, to.id, repr(msg_dict))) if self.out_queue.has_key(to.rx_key): self.out_queue[to.rx_key].append(msg_dict) else: self.out_queue[to.rx_key] = [msg_dict] def wait(self, limit=None): self.consumer.wait(limit) def send(self, message_data, routing_key): self.publisher.send(message_data, routing_key=routing_key, delivery_mode=1) def receive(self, message_data, message): rkey = message.amqp_message.routing_key participant_conn_key, wavelet_id, message_category = rkey.split(".") if message_category != "clientop": return logger.debug("Received Message from %s.%s.%s:\n%s" % (participant_conn_key, wavelet_id, message_category, repr(message_data))) # Get participant connection try: pconn = ParticipantConn.objects.get(tx_key=participant_conn_key) except ObjectDoesNotExist: logger.error("{%s} ParticipantConn not found" % (rkey)) return # Fail silently # Get wavelet try: wavelet = pconn.participant.wavelets.get(id=wavelet_id) except ObjectDoesNotExist: logger.error("{%s} Wavelet not found (or not participating)" % (rkey)) return # Fail silently # Handle message and reply to sender and/or broadcast an event self.out_queue = {} if isinstance(message_data, list): # multi-message? for sub_message in message_data: if not self.handle_participant_message(wavelet, pconn, sub_message): break else: self.handle_participant_message(wavelet, pconn, message_data) for receiver, messages in self.out_queue.iteritems(): self.send(messages, "%s.%s.waveop" % (receiver, wavelet_id)) self.out_queue = {} # Cleanup time? if datetime.datetime.now() > self.next_purge: self.purge_connections() def handle_participant_message(self, wavelet, pconn, message): """ Handle a participant's operation. If True is returned, go on with processing the next message. If False is returned, discard any following messages. """ participant = pconn.participant if message.has_key(u"type"): if message["type"] == "WAVELET_OPEN": logger.info("[%s/%d@%s] Opening wavelet" % (participant.name, pconn.id, wavelet.wave.id)) pconn.wavelets.add(wavelet) # I know this is neat :) self.emit(pconn, "WAVELET_OPEN", { "wavelet": wavelet.serialize(), "blips": wavelet.serialize_blips(), }) elif message["type"] == "PARTICIPANT_INFO": logger.info("[%s/%d@%s] Sending participant information" % (participant.name, pconn.id, wavelet.wave.id)) p_info = {} for p_id in message["property"]: try: p_info[p_id] = Participant.objects.get(id=p_id).serialize() except ObjectDoesNotExist: p_info[p_id] = None self.emit(pconn, "PARTICIPANT_INFO", p_info) elif message["type"] == "PARTICIPANT_SEARCH": if len(message["property"]) < getattr(settings, "PARTICIPANT_SEARCH_LENGTH", 0): self.emit(pconn, "PARTICIPANT_SEARCH", {"result": "TOO_SHORT", "data": getattr(settings, "PARTICIPANT_SEARCH_LENGTH", 0)}) logger.debug("[%s/%d@%s] Participant search query too short" % (participant.name, pconn.id, wavelet.wave.id)) else: logger.info("[%s/%d@%s] Performing participant search" % (participant.name, pconn.id, wavelet.wave.id)) lst = [] for p in Participant.objects.filter(name__icontains=message["property"]).exclude(id=participant.id): lst.append(p.id) self.emit(pconn, "PARTICIPANT_SEARCH", {"result": "OK", "data": lst}) elif message["type"] == "GADGET_LIST": all_gadgets = map(lambda g: {"id": g.id, "uploaded_by": g.by_user.participants.all()[0].name, "name": g.title, "descr": g.description, "url": g.url}, Gadget.objects.all()) logger.info("[%s/%d@%s] Sending Gadget list" % (participant.name, pconn.id, wavelet.wave.id)) self.emit(pconn, "GADGET_LIST", all_gadgets) elif message["type"] == "WAVELET_ADD_PARTICIPANT": # Find participant try: p = Participant.objects.get(id=message["property"]) except ObjectDoesNotExist: logger.error("[%s/%d@%s] Target participant '%s' not found" % (participant.name, pconn.id, wavelet.wave.id, message["property"])) return # Fail silently (TODO: report error to user) # Check if already participating if wavelet.participants.filter(id=message["property"]).count() > 0: logger.error("[%s/%d@%s] Target participant '%s' already there" % (participant.name, pconn.id, wavelet.wave.id, message["property"])) return # Fail silently (TODO: report error to user) wavelet.participants.add(p) logger.info("[%s/%d@%s] Added new participant '%s'" % (participant.name, pconn.id, wavelet.wave.id, message["property"])) self.broadcast(wavelet, "WAVELET_ADD_PARTICIPANT", message["property"]) elif message["type"] == "WAVELET_REMOVE_SELF": self.broadcast(wavelet, "WAVELET_REMOVE_PARTICIPANT", participant.id) wavelet.participants.remove(participant) # Bye bye pconn.wavelets.remove(wavelet) # Also for your connection logger.info("[%s/%d@%s] Participant removed himself" % (participant.name, pconn.id, wavelet.wave.id)) if wavelet.participants.count() == 0: # Oh my god, you killed the Wave! You bastard! logger.info("[%s/%d@%s] Wave got killed!" % (participant.name, pconn.id, wavelet.wave.id)) wavelet.wave.delete() return False elif message["type"] == "OPERATION_MESSAGE_BUNDLE": # Build OpManager newdelta = OpManager(wavelet.wave.id, wavelet.id) newdelta.unserialize(message["property"]["operations"]) version = message["property"]["version"] # Transform for delta in wavelet.deltas.filter(version__gt=version): for op in delta.getOpManager().operations: newdelta.transform(op) # Trash results (an existing delta cannot be changed) # Apply wavelet.applyOperations(newdelta.operations) # Raise version and store wavelet.version += 1 wavelet.save() Delta.createByOpManager(newdelta, wavelet.version).save() # Create tentative checksums blipsums = wavelet.blipsums() # Respond self.emit(pconn, "OPERATION_MESSAGE_BUNDLE_ACK", {"version": wavelet.version, "blipsums": blipsums}) self.broadcast(wavelet, "OPERATION_MESSAGE_BUNDLE", {"version": wavelet.version, "operations": newdelta.serialize(), "blipsums": blipsums}, [pconn]) logger.debug("[%s/%d@%s] Processed delta #%d -> v%d" % (participant.name, pconn.id, wavelet.wave.id, version, wavelet.version)) else: logger.error("[%s/%d@%s] Unknown message: %s" % (participant.name, pconn.id, wavelet.wave.id, message)) else: logger.error("[%s/%d@%s] Unknown message: %s" % (participant.name, pconn.id, wavelet.wave.id, message)) return True def purge_connections(self): """ This method will check all connections to the server and throw out all closed ones. This can be run asynchronously every 10 minutes or so. """ for conn in ParticipantConn.objects.all(): for wavelet in conn.wavelets.all(): if not self.queue_exists("%s.%s.waveop" % (conn.rx_key, wavelet.id)): wavelet.participant_conns.remove(conn) logger.info("[%s/%d@%s] Connection to wavelet closed" % (conn.participant.name, conn.id, wavelet.wave.id)) if conn.wavelets.count() == 0 and datetime.datetime.now() > conn.created + self.conn_min_lifetime: conn_id, conn_participant_name = conn.id, conn.participant.name conn.delete() logger.info("[%s/%d] Connection to server closed" % (conn_participant_name, conn_id)) self.next_purge = datetime.datetime.now() + self.purge_every def queue_exists(self, queue): """ Check if a queue exists, i.e. a user is connected to it (because auto-delete is always turned on). """ logger.debug("Checking queue %s" % (queue)) qex = self.publisher.backend.queue_exists(queue) # Re-open the channel if it was closed (this is a pyamqlib issue) if self.publisher.backend.channel.connection == None: self.publisher.backend.channel = self.publisher.backend.connection.connection.channel() return qex
def __init__(self, message_type, queue, context): Consumer.__init__(self, context.broker, routing_key=message_type, queue=queue)
from carrot.connection import BrokerConnection from carrot.messaging import Consumer import redis conn = BrokerConnection(hostname="localhost", port=5672, userid="guest", password="******", virtual_host="/") consumer = Consumer(connection=conn, queue="feed", exchange="feed", routing_key="importer") db = redis.Redis() def message_callback(message_data, message): db.incr(message_data["test_name"]) message.ack() consumer.register_callback(message_callback) consumer.wait()
def __init__(self, channel, context): queue = 'eb_%s' % uuid1().hex Consumer.__init__(self, context.broker, exchange=_exchange_for_channel(channel), queue=queue)
class Worker(object): def __init__(self, hostname='localhost', port=5672, userid="collectoruser", password="******", **options): conn = BrokerConnection(hostname=hostname, port=port, userid=userid, password=password, virtual_host='collectorvhost') try: self.publisher = Publisher(connection=conn, exchange="collector.response", exchange_type='direct', routing_key="response", serializer="json") except Exception as ex: raise Worker.ConnectionException(ex) self.consumer = Consumer(connection=conn, queue="feed", exchange_type='topic', exchange="collector", routing_key="collector.*.*") def run(self): self.consumer.register_callback(self._get_request_callback) print "Waiting..." self.consumer.wait() def _get_request_callback(self, message_data, message): #print "message_data=%s" % message_data drivers = [] drivers.append(message_data) DRIVER_KEY = "driver_routing_key" TAGS_KEY = "tags" for obj in drivers: driver = None if DRIVER_KEY in obj and TAGS_KEY in obj: module_name = obj["driver_routing_key"] try: __import__(module_name) module = sys.modules[module_name] driver = module.Driver() except: print "could not import module %s" % module_name values = [] for tag in obj['tags']: if driver: value = driver.read_value(tag['device'], tag['address']) else: value = "BAD_DRIVER" values.append({ 'name': tag['name'], 'current_value': value, 'read_at': '2009-01-01' }) self.publisher.send(values) else: print "Badly formated request %s" % obj message.ack() class WorkerException(Exception): pass class ConnectionException(WorkerException): pass