def send_messages(ui, repo, node, **kwargs): # Read the config get_configuration(ui) # Let the user know what is going on print "Sending messages to %s" % CONF["BROKER_HOST"] print "Please do not interrupt..." try: # Get the push data # TODO: Support pulling from the pushlog db pushdata = get_push_data(ui, repo, node) # If they don't want AMQP, switch the backend if CONF["BROKER_PROTOCOL"] == "STOMP": backend = "carrot.backends.pystomp.Backend" else: backend = None # Connect once for all the messages we will send connection = BrokerConnection( hostname=CONF["BROKER_HOST"], port=CONF["BROKER_PORT"], userid=CONF["BROKER_USER"], password=CONF["BROKER_PASS"], backend_cls=backend, ) # Send the overall push message. Most consumers likely # only care about this message print " Sending the %s message..." % CONF["LABEL_PUSH"] send_push_message(connection, pushdata) # Also send messages for each changeset as consumers may be # interested in those as well print " Sending individual %s messages..." % CONF["LABEL_CHANGESET"] for changedata in pushdata[CONF["LABEL_CHANGESETS"]]: changedata["repository"] = pushdata["repository"] send_changeset_message(connection, changedata) try: connection.close() except: # We don't care about connection close failures pass except Exception, e: # Something went wrong... print "ERROR: Hook returned an exception: %s" % e if CONF["HG_FAIL_ON_MSG_FAIL"]: # Admin wants the hook to fail on the message send failure print "Please try pushing again." return 1 else: # Admin wants the hook to succeed on the message send failure print "Ignoring and continuing the push..." return 0
def send_msg(routing_key, data): conn = BrokerConnection( hostname=settings.BROKER_HOST, port=settings.BROKER_PORT, userid=settings.BROKER_USER, password=settings.BROKER_PASSWORD, virtual_host=settings.BROKER_VHOST) publisher = Publisher(connection=conn, exchange="django_send", routing_key=routing_key, exchange_type="topic", ) publisher.send(data) publisher.close() conn.close()
class IMessageBroker(Consumer, Thread): """Interface for communicating JSON over AMQP. Consumer thread interface that can also publish a response for each request with unique id. Consumer and publisher use the same AMQP connection. Subclass this to implement a worker thread. See the example in the tests. """ # implemented in subclass service_name = None amqp_connection = None amqp_connection_settings = None exchange_name = None topic = None # dynamic instance variables binding_key = None response_routing_key = None req_queue_name = None def __init__(self, *args, **kwargs): logging.debug("%s starting up" % self.__unicode__()) # set instance variables self.binding_key = '.'.join([self.topic, 'request', '*']) self.response_routing_key = '.'.join([self.topic, 'response', '%s']) self.req_queue_name = '%s_req' % self.service_name # initialize the thread Thread.__init__(self, *args, **kwargs) # wait for AMQP connection, declare exchange and request queue # and bind the consumer self._wait_connection() self._declare_channel() self._init_consumer() def __exit__(self, e_type, e_value, e_trace): logging.debug('%s exiting' % self.__unicode__()) if e_type: logging.error(e_type(e_value)) def __unicode__(self): return self.service_name def _declare_channel(self): """Declares exchange and request queue.""" backend = self.amqp_connection.create_backend() backend.exchange_declare( exchange=self.exchange_name, type="topic", durable=True, auto_delete=False,) backend.queue_declare( queue=self.req_queue_name, durable=True, exclusive=False, auto_delete=False,) backend.queue_bind(self.req_queue_name,self.exchange_name,self.binding_key) logging.debug("%s queue %s bound to %s" % ( self.exchange_name, self.req_queue_name, self.binding_key)) def _wait_connection(self,time_to_wait=60): """Waits for RabbitMQ connection. """ _connected = False while(not _connected): try: self.amqp_connection = BrokerConnection(**self.amqp_connection_settings) if self.amqp_connection.connection: _connected = True logging.debug('%s connected to AMQP @ %s:%i' % ( self.service_name, self.amqp_connection_settings['hostname'], self.amqp_connection_settings['port'])) except socket_error: logging.warn("%s waiting %i sec for RabbitMQ connection..." % ( self.service_name, time_to_wait)) sleep(time_to_wait) def _init_consumer(self): Consumer.__init__(self, connection=self.amqp_connection, exchange=self.exchange_name, exchange_type="topic", queue=self.req_queue_name, ) # register the callback method self.register_callback(self._request_filter) # calls "dispatch" self._stop = Event() self._stop.set() # set to stopped state def _request_filter(self, message_data, message): """Filter for incoming AMQP requests. Expects JSON input as message payload. It has a metadata wrapper with keys "q" and "qid", for actual message data and an identifier, respectively. Example: {"qid": 293742, "q": {"hello my friend . we speak over the RPC now . "}} """ logging.debug("consumer received message: \n%s" % message.delivery_info) #routing_key = message.delivery_info['routing_key'] qid = None try: data = json.loads(message_data) qid = data.get('qid') request = data.get('q') self.dispatch(message, request, qid) except: logging.error(sys.exc_info()[1]) return def dispatch(self, message, request, qid): """AMQP "request" callback handler. - message: carrot Message - request: JSON data - qid: unique request identifier Do whatever your worker is supposed to do here. You can return a response by calling self.return_response(json_response, qid). """ raise NotImplementedError, "Write your own subclass" def return_response(self, response, qid): """AMQP "response" handler. Publishes a response message to a temporary queue. - response is json, qid is string """ message = json.dumps(response) routing_key = self.response_routing_key % qid logging.debug("response to %s with routing_key: %s, message: \n%s" % (self.exchange_name, routing_key, message)) try: publisher = Publisher( connection=self.amqp_connection, exchange=self.exchange_name, exchange_type="topic", routing_key=routing_key, ) publisher.send(message) publisher.close() except: """Trying to send with broken connection. Handle gracefully by waiting for connection and publish again.""" logging.error('%s AMQP error: %s' % (self.service_name, sys.exc_info()[1])) self._wait_connection(5) self.return_response(response, qid) @property def stopped(self): return self._stop.isSet() def run(self): self._stop.clear() # started logging.info('%s entering AMQP consumer loop' % self.__unicode__()) it = self.iterconsume() while not self.stopped: try: it.next() except KeyboardInterrupt: logging.warn("KB interrupt") except (AMQPConnectionException, IOError, AttributeError): """AMQPConnectionException or IOError if the AMQ socket closes. AttributeError if channel has been closed. Handle gracefully by waiting for connection and re-enter consumer loop, unless the service stop flag is set. """ if self.stopped: break # else reconnect logging.error('%s AMQP error: %s' % (self.service_name, sys.exc_info()[1])) self._wait_connection() self._init_consumer() self._stop.clear() # continue in started mode it = self.iterconsume() continue except StopIteration: """Happens at cancel() and on lost connection, if IOError exception handler could not restore consumption loop.""" logging.debug('%s %s' % (self.__unicode__(), sys.exc_info()[1])) logging.info('%s break out of consumer loop' % self.__unicode__()) self._stop.set() break except: logging.error("BUG") logging.error('%s %s' % (self.__unicode__(), sys.exc_info()[1])) traceback.print_tb(sys.exc_info()[2],10) try: self.amqp_connection.close() except: logging.debug('%s %s' % (self.__unicode__(), sys.exc_info()[1])) logging.info('%s thread exiting' % self.__unicode__()) exit() def stop(self,*args): """Stop consumer""" logging.debug('%s stop' % self.__unicode__()) try: try: # set the Event self._stop.set() # release the consumer from iterconsume self.cancel() except: pass #logging.debug('%s %s' % (self.__unicode__(), sys.exc_info()[1])) finally: try: self.amqp_connection.close() except IOError: logging.debug(sys.exc_info()[1]) if self.amqp_connection.pool: self.amqp_connection.release()
class GenericConsumer(object): def __init__(self, config, exchange=None, connect=True, heartbeat=False, **kwargs): self.config = config self.exchange = exchange self.connection = None self.durable = False self.applabel = '' self.heartbeat = heartbeat for x in ['applabel','topic','callback','durable']: if x in kwargs: setattr(self, x, kwargs[x]) del kwargs[x] if connect: self.connect() # Sets vairables def configure(self, **kwargs): for x in kwargs: setattr(self, x, kwargs[x]) # Connect to the message broker def connect(self): if not self.connection: self.connection = BrokerConnection(hostname=self.config.host, port=self.config.port, userid=self.config.user, password=self.config.password, virtual_host=self.config.vhost) # Disconnect from the message broker def disconnect(self): if self.connection: self.connection.close() # Support purging messages that are already in the queue on the broker # TODO: I think this is only supported by the amqp backend def purge_existing_messages(self): # Make sure there is an applabel given if self.durable and not self.applabel: raise InvalidAppLabel('Durable consumers must have an applabel') # Purge the queue of existing messages self.connection.create_backend().queue_purge(self.applabel) # Blocks and calls the callback when a message comes into the queue # For info on one script listening to multiple channels, see # http://ask.github.com/carrot/changelog.html#id1 def listen(self, callback=None): # One can optionally provide a callback to listen (if it wasn't already) if callback: self.callback = callback # Make suere there is an exchange given if not self.exchange: raise InvalidExchange(self.exchange) # Make sure there is a topic given if not self.topic: raise InvalidTopic(self.topic) # Make sure there is an applabel given if self.durable and not self.applabel: raise InvalidAppLabel('Durable consumers must have an applabel') # Make sure there is a callback given if not self.callback or not hasattr(self.callback, '__call__'): raise InvalidCallback(self.callback) # Connect to the broker if we haven't already if not self.connection: self.connect() # Set up our broker consumer self.consumer = Consumer(connection=self.connection, queue=self.applabel, exchange=self.exchange, exchange_type="topic", auto_declare=False, routing_key=self.topic) # We need to manually create / declare the queue self.consumer.backend.queue_declare(queue=self.applabel, durable=self.durable, exclusive=False, auto_delete=not self.durable, arguments=self.consumer.queue_arguments, warn_if_exists=False) # No need to manually create the exchange, as the producer creates it # and we expect it to just be there # We support multiple bindings if we were given an array for the topic if not isinstance(self.topic, list): self.topic = [self.topic] # We need to bind the queue to the exchange with the specified keys if self.consumer.queue: for routing_key in self.topic: self.consumer.backend.queue_bind(queue=self.consumer.queue, exchange=self.exchange, routing_key=routing_key) if self.heartbeat: self.consumer.backend.queue_bind(queue=self.consumer.queue, exchange='org.mozilla.exchange.pulse.test', routing_key='heartbeat') # Register the callback the user wants self.consumer.register_callback(self.callback) # This blocks, and then calls the user callback every time a message # comes in self.consumer.wait() # Likely never get here but can't hurt self.disconnect()
class Connection: def __init__(self, hostname, port, vhost, userid, password): """connects to broker and provides convenience methods""" self.broker = BrokerConnection(hostname=hostname, port=port, userid=userid, password=password, virtual_host=vhost) def __del__(self): self.broker.close() def declare(self, exchange, exchange_type, binding="", queue=""): """declares the exchange, the queue and binds the queue to the exchange exchange - exchange name exchange_type - direct, topic, fanout binding - binding to queue (optional) queue - queue to bind to exchange using binding (optional) """ if (binding and not queue) or (queue and not binding): if queue and not exchange_type == "fanout": raise Error("binding and queue are not mutually exclusive") consumer = Consumer(connection=self.broker, exchange=exchange, exchange_type=exchange_type, routing_key=binding, queue=queue) consumer.declare() consumer.close() def consume(self, queue, limit=None, callback=None, auto_declare=False): """consume messages in queue queue - name of queue limit - amount of messages to iterate through (default: no limit) callback - method to call when a new message is received must take two arguments: message_data, message must send the acknowledgement: message.ack() default: print message to stdout and send ack auto_declare - automatically declare the queue (default: false) """ if not callback: callback = _consume_callback consumer = Consumer(connection=self.broker, queue=queue, auto_declare=auto_declare) consumer.register_callback(callback) for message in consumer.iterqueue(limit=limit, infinite=False): consumer.receive(message.payload, message) consumer.close() def publish(self, exchange, routing_key, message, auto_declare=False, persistent=True): """publish a message to exchange using routing_key exchange - name of exchange routing_key - interpretation of routing key depends on exchange type message - message content to send auto_declare - automatically declare the exchange (default: false) persistent - store message on disk as well as memory (default: True) """ delivery_mode = 2 if not persistent: delivery_mode = 1 publisher = Publisher(connection=self.broker, exchange=exchange, routing_key=routing_key, auto_declare=auto_declare) publisher.send(message, delivery_mode=delivery_mode) publisher.close()
import jsonrpclib from carrot.connection import BrokerConnection from carrot.messaging import Consumer conn = BrokerConnection(hostname="localhost", port=5672, userid="guest", password="******", virtual_host="/") consumer = Consumer(connection=conn, queue="po_box", exchange="sorting_room", routing_key="jason") def amqp_callback(message_data, message): server = jsonrpclib.Server("http://localhost:8080") server.ping(message_data) print jsonrpclib.history.response message.ack() consumer.register_callback(amqp_callback) consumer.wait() # Go into the consumer loop. conn.close()
class AmqpbusTests(unittest.TestCase): """Tests the AMQP communication. """ test_connection = None def setUp(self): # single global RPC server thread, ok in test framework global test_rpc_service if not test_rpc_service: test_rpc_service = MockRPCServiceProvider() if not self.test_connection: self.test_connection = BrokerConnection(**settings.AMQP_CONNECTION) def tearDown(self): test_rpc_service.stop() self.test_connection.close() def test_consumer(self): """AMQP->RPC->AMQP Send a AMQP message, and test RPCConsumer response. Uses a mock RPC server that the consumer will call. Check the consumer AMQP response with test consumer. """ class TestConsumer(IMessageBroker): service_name = 'TestConsumer' exchange_name = 'Test' topic = 'test' amqp_connection_settings = settings.AMQP_CONNECTION def dispatch(self, message, request, qid): """ AMQP -> RPC dispatcher. """ logging.info('dispatching AMQP -> RPC') response = {} if qid: # call remote RPC # this message has an qid so it expects a response response['msg'] = test_rpc_service.push(request) message.ack() # return the response to AMQP, the caller should be listening .. self.return_response(response,qid) else: # no qid, so do something stateless .. print request message.ack() try: consumer = TestConsumer() self.assertEquals('test.request.*',consumer.binding_key) self.assertEquals('test.response.%s',consumer.response_routing_key) consumer.start() test_rpc_service.start() test_rpc_service.messages = [] # allow consumer to start sleep(0.2) self.assert_(not consumer.stopped) self.assert_(consumer.isAlive()) # test variables qid = str(randrange(0,999999)) jsondata = {'msg':'hello rpc'} _publisher = Publisher( connection=self.test_connection, exchange='Test', exchange_type="topic", routing_key='test.request.'+qid, ) # test channel backend = self.test_connection.create_backend() backend.queue_declare( queue="test", durable=False, exclusive=False, auto_delete=True,) backend.queue_bind("test",'Test','test.response.'+qid) _consumer = Consumer( connection=self.test_connection, exchange='Test', exchange_type="topic", queue="test", ) _consumer.discard_all() logging.debug('publishing JSON message to RPC') data_on_the_wire = json.dumps({'q': jsondata, 'qid': qid}) _publisher.send(data_on_the_wire) # allow data to pass the wire sleep(0.2) # retrieve dispatcher response response = _consumer.fetch() self.assert_(response, 'No response') data = json.loads(response.payload) self.assert_(len(data['msg']) > 0) # assert non-empty response self.assertEquals('ok',data['msg']) # check dispatcher RPC function self.assert_(len(test_rpc_service.messages) > 0, 'Message did not arrive') self.assertEquals(test_rpc_service.messages[0], jsondata) finally: try: consumer.stop() except: pass try: _consumer.close() except: pass test_rpc_service.stop() self.assert_(consumer.stopped)
class GenericPublisher(object): def __init__(self, config, exchange=None, connect=True): self.config = config self.exchange = exchange self.connection = None if connect: self.connect() # Connect to the message broker def connect(self): if not self.connection: self.connection = BrokerConnection(hostname=self.config.host, port=self.config.port, userid=self.config.user, password=self.config.password, virtual_host=self.config.vhost) # Disconnect from the message broker def disconnect(self): if self.connection: self.connection.close() self.connection = None # Used to publish a pulse message to the proper exchange def publish(self, message): # Make suere there is an exchange given if not self.exchange: raise InvalidExchange(self.exchange) # Make sure there is a message given if not message: raise MalformedMessage(message) # Have the message prepare and validate itself message._prepare() # Connect to the broker if we haven't already if not self.connection: self.connect() # Set up our broker publisher self.publisher = Publisher(connection=self.connection, exchange=self.exchange, exchange_type="topic", routing_key=message.routing_key) # The message is actually a simple envelope format with a payload and # some metadata final_data = {} final_data['payload'] = message.data final_data['_meta'] = message.metadata.copy() final_data['_meta'].update({ 'exchange': self.exchange, 'routing_key': message.routing_key, 'serializer': self.config.serializer, 'sent': time_to_string(datetime.now(timezone(self.config.broker_timezone))) }) # Send the message self.publisher.send(final_data, serializer=self.config.serializer) # Close the publishing connection self.publisher.close()
class GenericConsumer(object): def __init__(self, config, exchange=None, connect=True, heartbeat=False, **kwargs): self.config = config self.exchange = exchange self.connection = None self.durable = False self.applabel = '' self.heartbeat = heartbeat for x in ['applabel', 'topic', 'callback', 'durable']: if x in kwargs: setattr(self, x, kwargs[x]) del kwargs[x] if connect: self.connect() # Sets vairables def configure(self, **kwargs): for x in kwargs: setattr(self, x, kwargs[x]) # Connect to the message broker def connect(self): if not self.connection: self.connection = BrokerConnection(hostname=self.config.host, port=self.config.port, userid=self.config.user, password=self.config.password, virtual_host=self.config.vhost) # Disconnect from the message broker def disconnect(self): if self.connection: self.connection.close() # Support purging messages that are already in the queue on the broker # TODO: I think this is only supported by the amqp backend def purge_existing_messages(self): # Make sure there is an applabel given if self.durable and not self.applabel: raise InvalidAppLabel('Durable consumers must have an applabel') # Purge the queue of existing messages self.connection.create_backend().queue_purge(self.applabel) # Blocks and calls the callback when a message comes into the queue # For info on one script listening to multiple channels, see # http://ask.github.com/carrot/changelog.html#id1 def listen(self, callback=None): # One can optionally provide a callback to listen (if it wasn't already) if callback: self.callback = callback # Make suere there is an exchange given if not self.exchange: raise InvalidExchange(self.exchange) # Make sure there is a topic given if not self.topic: raise InvalidTopic(self.topic) # Make sure there is an applabel given if self.durable and not self.applabel: raise InvalidAppLabel('Durable consumers must have an applabel') # Make sure there is a callback given if not self.callback or not hasattr(self.callback, '__call__'): raise InvalidCallback(self.callback) # Connect to the broker if we haven't already if not self.connection: self.connect() # Set up our broker consumer self.consumer = Consumer(connection=self.connection, queue=self.applabel, exchange=self.exchange, exchange_type="topic", auto_declare=False, routing_key=self.topic) # We need to manually create / declare the queue self.consumer.backend.queue_declare( queue=self.applabel, durable=self.durable, exclusive=False, auto_delete=not self.durable, arguments=self.consumer.queue_arguments, warn_if_exists=False) # No need to manually create the exchange, as the producer creates it # and we expect it to just be there # We support multiple bindings if we were given an array for the topic if not isinstance(self.topic, list): self.topic = [self.topic] # We need to bind the queue to the exchange with the specified keys if self.consumer.queue: for routing_key in self.topic: self.consumer.backend.queue_bind(queue=self.consumer.queue, exchange=self.exchange, routing_key=routing_key) if self.heartbeat: self.consumer.backend.queue_bind( queue=self.consumer.queue, exchange='org.mozilla.exchange.pulse.test', routing_key='heartbeat') # Register the callback the user wants self.consumer.register_callback(self.callback) # This blocks, and then calls the user callback every time a message # comes in self.consumer.wait() # Likely never get here but can't hurt self.disconnect()
class Connection: def __init__(self, hostname, port, vhost, userid, password): """connects to broker and provides convenience methods""" self.broker = BrokerConnection(hostname=hostname, port=port, userid=userid, password=password, virtual_host=vhost) def __del__(self): self.broker.close() def declare(self, exchange, exchange_type, binding="", queue=""): """declares the exchange, the queue and binds the queue to the exchange exchange - exchange name exchange_type - direct, topic, fanout binding - binding to queue (optional) queue - queue to bind to exchange using binding (optional) """ if (binding and not queue) or (queue and not binding): if queue and not exchange_type == "fanout": raise Error("binding and queue are not mutually exclusive") consumer = Consumer(connection=self.broker, exchange=exchange, exchange_type=exchange_type, routing_key=binding, queue=queue) consumer.declare() consumer.close() def consume(self, queue, limit=None, callback=None, auto_declare=False): """consume messages in queue queue - name of queue limit - amount of messages to iterate through (default: no limit) callback - method to call when a new message is received must take two arguments: message_data, message must send the acknowledgement: message.ack() default: print message to stdout and send ack auto_declare - automatically declare the queue (default: false) """ if not callback: callback = _consume_callback consumer = Consumer(connection=self.broker, queue=queue, auto_declare=auto_declare) consumer.register_callback(callback) for message in consumer.iterqueue(limit=limit, infinite=False): consumer.receive(message.payload, message) consumer.close() def publish(self, exchange, routing_key, message, auto_declare=False, persistent=True): """publish a message to exchange using routing_key exchange - name of exchange routing_key - interpretation of routing key depends on exchange type message - message content to send auto_declare - automatically declare the exchange (default: false) persistent - store message on disk as well as memory (default: True) """ delivery_mode = 2 if not persistent: delivery_mode = 1 publisher = Publisher(connection=self.broker, exchange=exchange, routing_key=routing_key, auto_declare=auto_declare) publisher.send(message, delivery_mode=delivery_mode) publisher.close()
class GenericPublisher(object): def __init__(self, config, exchange=None, connect=True): self.config = config self.exchange = exchange self.connection = None if connect: self.connect() # Connect to the message broker def connect(self): if not self.connection: self.connection = BrokerConnection(hostname=self.config.host, port=self.config.port, userid=self.config.user, password=self.config.password, virtual_host=self.config.vhost) # Disconnect from the message broker def disconnect(self): if self.connection: self.connection.close() self.connection = None # Used to publish a pulse message to the proper exchange def publish(self, message): # Make suere there is an exchange given if not self.exchange: raise InvalidExchange(self.exchange) # Make sure there is a message given if not message: raise MalformedMessage(message) # Have the message prepare and validate itself message._prepare() # Connect to the broker if we haven't already if not self.connection: self.connect() # Set up our broker publisher self.publisher = Publisher(connection=self.connection, exchange=self.exchange, exchange_type="topic", routing_key=message.routing_key) # The message is actually a simple envelope format with a payload and # some metadata final_data = {} final_data['payload'] = message.data final_data['_meta'] = message.metadata.copy() final_data['_meta'].update({ 'exchange': self.exchange, 'routing_key': message.routing_key, 'serializer': self.config.serializer, 'sent': time_to_string(datetime.now(timezone(self.config.broker_timezone))) }) # Send the message self.publisher.send(final_data, serializer=self.config.serializer) # Close the publishing connection self.publisher.close()