Beispiel #1
0
def send_messages(ui, repo, node, **kwargs):

    # Read the config
    get_configuration(ui)

    # Let the user know what is going on
    print "Sending messages to %s" % CONF["BROKER_HOST"]
    print "Please do not interrupt..."

    try:
        # Get the push data
        # TODO: Support pulling from the pushlog db
        pushdata = get_push_data(ui, repo, node)

        # If they don't want AMQP, switch the backend
        if CONF["BROKER_PROTOCOL"] == "STOMP":
            backend = "carrot.backends.pystomp.Backend"
        else:
            backend = None

        # Connect once for all the messages we will send
        connection = BrokerConnection(
            hostname=CONF["BROKER_HOST"],
            port=CONF["BROKER_PORT"],
            userid=CONF["BROKER_USER"],
            password=CONF["BROKER_PASS"],
            backend_cls=backend,
        )

        # Send the overall push message. Most consumers likely
        # only care about this message
        print "    Sending the %s message..." % CONF["LABEL_PUSH"]
        send_push_message(connection, pushdata)

        # Also send messages for each changeset as consumers may be
        # interested in those as well
        print "    Sending individual %s messages..." % CONF["LABEL_CHANGESET"]
        for changedata in pushdata[CONF["LABEL_CHANGESETS"]]:
            changedata["repository"] = pushdata["repository"]
            send_changeset_message(connection, changedata)

        try:
            connection.close()
        except:
            # We don't care about connection close failures
            pass

    except Exception, e:
        # Something went wrong...
        print "ERROR: Hook returned an exception: %s" % e

        if CONF["HG_FAIL_ON_MSG_FAIL"]:
            # Admin wants the hook to fail on the message send failure
            print "Please try pushing again."
            return 1
        else:
            # Admin wants the hook to succeed on the message send failure
            print "Ignoring and continuing the push..."
            return 0
Beispiel #2
0
def send_msg(routing_key, data):
    conn = BrokerConnection(
                hostname=settings.BROKER_HOST,
                port=settings.BROKER_PORT,
                userid=settings.BROKER_USER,
                password=settings.BROKER_PASSWORD,
                virtual_host=settings.BROKER_VHOST)
    publisher = Publisher(connection=conn,
                          exchange="django_send",
                          routing_key=routing_key,
                          exchange_type="topic",
                          )
    publisher.send(data)
    publisher.close()
    conn.close()
Beispiel #3
0
class IMessageBroker(Consumer, Thread):
    """Interface for communicating JSON over AMQP.
    
    Consumer thread interface that can also publish a response for each request with unique id.
    Consumer and publisher use the same AMQP connection.

    Subclass this to implement a worker thread. See the example in the tests.
    """
    # implemented in subclass
    service_name = None
    amqp_connection = None
    amqp_connection_settings = None
    exchange_name = None
    topic = None

    # dynamic instance variables
    binding_key = None
    response_routing_key = None
    req_queue_name = None

    def __init__(self, *args, **kwargs):
        logging.debug("%s starting up" % self.__unicode__())

        # set instance variables
        self.binding_key = '.'.join([self.topic, 'request', '*'])
        self.response_routing_key = '.'.join([self.topic, 'response', '%s'])
        self.req_queue_name = '%s_req' % self.service_name

        # initialize the thread
        Thread.__init__(self, *args, **kwargs)
        # wait for AMQP connection, declare exchange and request queue
        # and bind the consumer
        self._wait_connection()
        self._declare_channel()
        self._init_consumer()

    def __exit__(self, e_type, e_value, e_trace):
        logging.debug('%s exiting' % self.__unicode__())
        if e_type:
            logging.error(e_type(e_value))

    def __unicode__(self):
        return self.service_name

    def _declare_channel(self):
        """Declares exchange and request queue."""
        backend = self.amqp_connection.create_backend()
        backend.exchange_declare(
            exchange=self.exchange_name,
            type="topic",
            durable=True,
            auto_delete=False,)
        backend.queue_declare(
            queue=self.req_queue_name,
            durable=True,
            exclusive=False,
            auto_delete=False,)
        backend.queue_bind(self.req_queue_name,self.exchange_name,self.binding_key)
        logging.debug("%s queue %s bound to %s" % (
            self.exchange_name,
            self.req_queue_name,
            self.binding_key))

    def _wait_connection(self,time_to_wait=60):
        """Waits for RabbitMQ connection. """
        _connected = False
        while(not _connected):
            try:
                self.amqp_connection = BrokerConnection(**self.amqp_connection_settings)
                if self.amqp_connection.connection:
                    _connected = True
                    logging.debug('%s connected to AMQP @ %s:%i' % (
                        self.service_name,
                        self.amqp_connection_settings['hostname'],
                        self.amqp_connection_settings['port']))
            except socket_error:
                logging.warn("%s waiting %i sec for RabbitMQ connection..." % (
                    self.service_name, time_to_wait))
                sleep(time_to_wait)

    def _init_consumer(self):
        Consumer.__init__(self,
            connection=self.amqp_connection,
            exchange=self.exchange_name,
            exchange_type="topic",
            queue=self.req_queue_name,
            )
        # register the callback method
        self.register_callback(self._request_filter) # calls "dispatch"
        self._stop = Event()
        self._stop.set() # set to stopped state

    def _request_filter(self, message_data, message):
        """Filter for incoming AMQP requests.
        
        Expects JSON input as message payload. It has a metadata wrapper with keys
        "q" and "qid", for actual message data and an identifier, respectively.
        Example:
        
        {"qid": 293742, "q": {"hello my friend . we speak over the RPC now . "}}
        """
        logging.debug("consumer received message: \n%s" % message.delivery_info)
        #routing_key = message.delivery_info['routing_key']
        qid = None
        try:
            data = json.loads(message_data)
            qid = data.get('qid')
            request = data.get('q')
            self.dispatch(message, request, qid)
        except:
            logging.error(sys.exc_info()[1])
            return

    def dispatch(self, message, request, qid):
        """AMQP "request" callback handler.

        - message: carrot Message
        - request: JSON data
        - qid: unique request identifier
        
        Do whatever your worker is supposed to do here.
        You can return a response by calling self.return_response(json_response, qid).
        """
        raise NotImplementedError, "Write your own subclass"

    def return_response(self, response, qid):
        """AMQP "response" handler.
        
        Publishes a response message to a temporary queue.
        - response is json, qid is string
        """
        message = json.dumps(response)
        routing_key = self.response_routing_key % qid
        logging.debug("response to %s with routing_key: %s, message: \n%s" % (self.exchange_name, routing_key, message))
        try:
            publisher = Publisher(
                connection=self.amqp_connection,
                exchange=self.exchange_name,
                exchange_type="topic",
                routing_key=routing_key,
                )
            publisher.send(message)
            publisher.close()
        except:
            """Trying to send with broken connection.

            Handle gracefully by waiting for connection and publish again."""
            logging.error('%s AMQP error: %s' % (self.service_name, sys.exc_info()[1]))
            self._wait_connection(5)
            self.return_response(response, qid)

    @property
    def stopped(self):
        return self._stop.isSet()

    def run(self):
        self._stop.clear() # started
        logging.info('%s entering AMQP consumer loop' % self.__unicode__())
        it = self.iterconsume()
        while not self.stopped:
            try:
                it.next()
            except KeyboardInterrupt:
                logging.warn("KB interrupt")
            except (AMQPConnectionException, IOError, AttributeError):
                """AMQPConnectionException or IOError if the AMQ socket closes.
                AttributeError if channel has been closed.

                Handle gracefully by waiting for connection and re-enter consumer loop,
                unless the service stop flag is set.
                """
                if self.stopped:
                    break

                # else reconnect
                logging.error('%s AMQP error: %s' % (self.service_name, sys.exc_info()[1]))
                self._wait_connection()
                self._init_consumer()
                self._stop.clear() # continue in started mode
                it = self.iterconsume()
                continue
            except StopIteration:
                """Happens at cancel() and on lost connection, 
                if IOError exception handler could not restore consumption loop."""
                logging.debug('%s %s' % (self.__unicode__(), sys.exc_info()[1]))
                logging.info('%s break out of consumer loop' % self.__unicode__())
                self._stop.set()
                break
            except:
                logging.error("BUG")
                logging.error('%s %s' % (self.__unicode__(), sys.exc_info()[1]))
                traceback.print_tb(sys.exc_info()[2],10)

        try:
            self.amqp_connection.close()
        except:
            logging.debug('%s %s' % (self.__unicode__(), sys.exc_info()[1]))

        logging.info('%s thread exiting' % self.__unicode__())
        exit()

    def stop(self,*args):
        """Stop consumer"""
        logging.debug('%s stop' % self.__unicode__())
        try:
            try:
                # set the Event
                self._stop.set()
                # release the consumer from iterconsume
                self.cancel()
            except:
                pass
                #logging.debug('%s %s' % (self.__unicode__(), sys.exc_info()[1]))
        finally:
            try:
                self.amqp_connection.close()
            except IOError:
                logging.debug(sys.exc_info()[1])
            if self.amqp_connection.pool:
                self.amqp_connection.release()
class GenericConsumer(object):

    def __init__(self, config, exchange=None, connect=True, heartbeat=False,
                 **kwargs):
        self.config     = config
        self.exchange   = exchange
        self.connection = None
        self.durable    = False
        self.applabel   = ''
        self.heartbeat  = heartbeat
        for x in ['applabel','topic','callback','durable']:
            if x in kwargs:
                setattr(self, x, kwargs[x])
                del kwargs[x]

        if connect:
            self.connect()

    # Sets vairables
    def configure(self, **kwargs):
        for x in kwargs:
            setattr(self, x, kwargs[x])

    # Connect to the message broker
    def connect(self):
        if not self.connection:
            self.connection = BrokerConnection(hostname=self.config.host,
                                               port=self.config.port,
                                               userid=self.config.user,
                                               password=self.config.password,
                                               virtual_host=self.config.vhost)

    # Disconnect from the message broker
    def disconnect(self):
        if self.connection:
            self.connection.close()

    # Support purging messages that are already in the queue on the broker
    # TODO: I think this is only supported by the amqp backend
    def purge_existing_messages(self):

        # Make sure there is an applabel given
        if self.durable and not self.applabel:
            raise InvalidAppLabel('Durable consumers must have an applabel')
        
        # Purge the queue of existing messages
        self.connection.create_backend().queue_purge(self.applabel)

    # Blocks and calls the callback when a message comes into the queue
    # For info on one script listening to multiple channels, see
    # http://ask.github.com/carrot/changelog.html#id1
    def listen(self, callback=None):

        # One can optionally provide a callback to listen (if it wasn't already)
        if callback:
            self.callback = callback

        # Make suere there is an exchange given
        if not self.exchange:
            raise InvalidExchange(self.exchange)

        # Make sure there is a topic given
        if not self.topic:
            raise InvalidTopic(self.topic)

        # Make sure there is an applabel given
        if self.durable and not self.applabel:
            raise InvalidAppLabel('Durable consumers must have an applabel')

        # Make sure there is a callback given
        if not self.callback or not hasattr(self.callback, '__call__'):
            raise InvalidCallback(self.callback)

        # Connect to the broker if we haven't already
        if not self.connection:
            self.connect()

        # Set up our broker consumer
        self.consumer = Consumer(connection=self.connection,
                                   queue=self.applabel,
                                   exchange=self.exchange,
                                   exchange_type="topic",
                                   auto_declare=False,
                                   routing_key=self.topic)

        # We need to manually create / declare the queue
        self.consumer.backend.queue_declare(queue=self.applabel,
                                            durable=self.durable,
                                            exclusive=False,
                                            auto_delete=not self.durable,
                                            arguments=self.consumer.queue_arguments,
                                            warn_if_exists=False)

        # No need to manually create the exchange, as the producer creates it
        # and we expect it to just be there

        # We support multiple bindings if we were given an array for the topic
        if not isinstance(self.topic, list):
            self.topic = [self.topic]

        # We need to bind the queue to the exchange with the specified keys
        if self.consumer.queue:
            for routing_key in self.topic:
                self.consumer.backend.queue_bind(queue=self.consumer.queue,
                                                 exchange=self.exchange,
                                                 routing_key=routing_key)
            if self.heartbeat:
                self.consumer.backend.queue_bind(queue=self.consumer.queue,
                                                 exchange='org.mozilla.exchange.pulse.test',
                                                 routing_key='heartbeat')

        # Register the callback the user wants
        self.consumer.register_callback(self.callback)

        # This blocks, and then calls the user callback every time a message 
        # comes in
        self.consumer.wait()

        # Likely never get here but can't hurt
        self.disconnect()
Beispiel #5
0
class Connection:
    def __init__(self, hostname, port, vhost, userid, password):
        """connects to broker and provides convenience methods"""
        self.broker = BrokerConnection(hostname=hostname,
                                       port=port,
                                       userid=userid,
                                       password=password,
                                       virtual_host=vhost)

    def __del__(self):
        self.broker.close()

    def declare(self, exchange, exchange_type, binding="", queue=""):
        """declares the exchange, the queue and binds the queue to the exchange
        
        exchange        - exchange name
        exchange_type   - direct, topic, fanout
        binding         - binding to queue (optional)
        queue           - queue to bind to exchange using binding (optional)
        """
        if (binding and not queue) or (queue and not binding):
            if queue and not exchange_type == "fanout":
                raise Error("binding and queue are not mutually exclusive")

        consumer = Consumer(connection=self.broker,
                            exchange=exchange,
                            exchange_type=exchange_type,
                            routing_key=binding,
                            queue=queue)
        consumer.declare()
        consumer.close()

    def consume(self, queue, limit=None, callback=None, auto_declare=False):
        """consume messages in queue
        
        queue           - name of queue
        limit           - amount of messages to iterate through (default: no limit)

        callback        - method to call when a new message is received
                          must take two arguments: message_data, message
                          must send the acknowledgement: message.ack()
                          default: print message to stdout and send ack

        auto_declare    - automatically declare the queue (default: false)
        """
        if not callback:
            callback = _consume_callback

        consumer = Consumer(connection=self.broker,
                            queue=queue,
                            auto_declare=auto_declare)

        consumer.register_callback(callback)
        for message in consumer.iterqueue(limit=limit, infinite=False):
            consumer.receive(message.payload, message)

        consumer.close()

    def publish(self,
                exchange,
                routing_key,
                message,
                auto_declare=False,
                persistent=True):
        """publish a message to exchange using routing_key
        
        exchange        - name of exchange
        routing_key     - interpretation of routing key depends on exchange type
        message         - message content to send
        auto_declare    - automatically declare the exchange (default: false)
        persistent      - store message on disk as well as memory (default: True)
        """
        delivery_mode = 2
        if not persistent:
            delivery_mode = 1

        publisher = Publisher(connection=self.broker,
                              exchange=exchange,
                              routing_key=routing_key,
                              auto_declare=auto_declare)

        publisher.send(message, delivery_mode=delivery_mode)
        publisher.close()
import jsonrpclib
from carrot.connection import BrokerConnection
from carrot.messaging import Consumer

conn = BrokerConnection(hostname="localhost", port=5672, userid="guest", password="******", virtual_host="/")

consumer = Consumer(connection=conn, queue="po_box", exchange="sorting_room", routing_key="jason")


def amqp_callback(message_data, message):
    server = jsonrpclib.Server("http://localhost:8080")
    server.ping(message_data)
    print jsonrpclib.history.response
    message.ack()


consumer.register_callback(amqp_callback)
consumer.wait()  # Go into the consumer loop.
conn.close()
class AmqpbusTests(unittest.TestCase):
    """Tests the AMQP communication.
    """
    test_connection = None

    def setUp(self):
        # single global RPC server thread, ok in test framework
        global test_rpc_service
        if not test_rpc_service:
            test_rpc_service = MockRPCServiceProvider()
        if not self.test_connection:
            self.test_connection = BrokerConnection(**settings.AMQP_CONNECTION)

    def tearDown(self):
        test_rpc_service.stop()
        self.test_connection.close()

    def test_consumer(self):
        """AMQP->RPC->AMQP
        
        Send a AMQP message, and test RPCConsumer response.
        Uses a mock RPC server that the consumer will call.
        Check the consumer AMQP response with test consumer.
        """
        class TestConsumer(IMessageBroker):
            service_name = 'TestConsumer'
            exchange_name = 'Test'
            topic = 'test'
            amqp_connection_settings = settings.AMQP_CONNECTION

            def dispatch(self, message, request, qid):
                """ AMQP -> RPC dispatcher.
                """
                logging.info('dispatching AMQP -> RPC')
                response = {}
                if qid:
                    # call remote RPC
                    # this message has an qid so it expects a response
                    response['msg'] = test_rpc_service.push(request)
                    message.ack()
                    # return the response to AMQP, the caller should be listening ..
                    self.return_response(response,qid)
                else:
                    # no qid, so do something stateless .. 
                    print request
                    message.ack()

        try:
            consumer = TestConsumer()
            self.assertEquals('test.request.*',consumer.binding_key)
            self.assertEquals('test.response.%s',consumer.response_routing_key)

            consumer.start()
            test_rpc_service.start()
            test_rpc_service.messages = []
            # allow consumer to start
            sleep(0.2)
            self.assert_(not consumer.stopped)
            self.assert_(consumer.isAlive())

            # test variables
            qid = str(randrange(0,999999))
            jsondata = {'msg':'hello rpc'}

            _publisher = Publisher(
                connection=self.test_connection,
                exchange='Test',
                exchange_type="topic",
                routing_key='test.request.'+qid,
                )

            # test channel
            backend = self.test_connection.create_backend()
            backend.queue_declare(
                queue="test",
                durable=False,
                exclusive=False,
                auto_delete=True,)
            backend.queue_bind("test",'Test','test.response.'+qid)
            _consumer = Consumer(
                connection=self.test_connection,
                exchange='Test',
                exchange_type="topic",
                queue="test",
                )
            _consumer.discard_all()

            logging.debug('publishing JSON message to RPC')
            data_on_the_wire = json.dumps({'q': jsondata, 'qid': qid})
            _publisher.send(data_on_the_wire)

            # allow data to pass the wire
            sleep(0.2)

            # retrieve dispatcher response
            response = _consumer.fetch()
            self.assert_(response, 'No response')
            data = json.loads(response.payload)
            self.assert_(len(data['msg']) > 0) # assert non-empty response
            self.assertEquals('ok',data['msg'])

            # check dispatcher RPC function
            self.assert_(len(test_rpc_service.messages) > 0, 'Message did not arrive')
            self.assertEquals(test_rpc_service.messages[0], jsondata)
        finally:
            try:
                consumer.stop()
            except:
                pass
            try:
                _consumer.close()
            except:
                pass
            test_rpc_service.stop()
            self.assert_(consumer.stopped)
class GenericPublisher(object):

    def __init__(self, config, exchange=None, connect=True):
        self.config = config
        self.exchange = exchange
        self.connection = None
        if connect:
            self.connect()

    # Connect to the message broker
    def connect(self):
        if not self.connection:
            self.connection = BrokerConnection(hostname=self.config.host,
                                               port=self.config.port,
                                               userid=self.config.user,
                                               password=self.config.password,
                                               virtual_host=self.config.vhost)

    # Disconnect from the message broker
    def disconnect(self):
        if self.connection:
            self.connection.close()
            self.connection = None

    # Used to publish a pulse message to the proper exchange
    def publish(self, message):

        # Make suere there is an exchange given
        if not self.exchange:
            raise InvalidExchange(self.exchange)

        # Make sure there is a message given
        if not message:
            raise MalformedMessage(message)

        # Have the message prepare and validate itself
        message._prepare()

        # Connect to the broker if we haven't already
        if not self.connection:
            self.connect()

        # Set up our broker publisher
        self.publisher = Publisher(connection=self.connection,
                                   exchange=self.exchange,
                                   exchange_type="topic",
                                   routing_key=message.routing_key)

        # The message is actually a simple envelope format with a payload and
        # some metadata
        final_data = {}
        final_data['payload'] = message.data
        final_data['_meta'] = message.metadata.copy()
        final_data['_meta'].update({
            'exchange': self.exchange,
            'routing_key': message.routing_key,
            'serializer': self.config.serializer,
            'sent': time_to_string(datetime.now(timezone(self.config.broker_timezone)))
        })

        # Send the message
        self.publisher.send(final_data, serializer=self.config.serializer)

        # Close the publishing connection
        self.publisher.close()
Beispiel #9
0
class GenericConsumer(object):
    def __init__(self,
                 config,
                 exchange=None,
                 connect=True,
                 heartbeat=False,
                 **kwargs):
        self.config = config
        self.exchange = exchange
        self.connection = None
        self.durable = False
        self.applabel = ''
        self.heartbeat = heartbeat
        for x in ['applabel', 'topic', 'callback', 'durable']:
            if x in kwargs:
                setattr(self, x, kwargs[x])
                del kwargs[x]

        if connect:
            self.connect()

    # Sets vairables
    def configure(self, **kwargs):
        for x in kwargs:
            setattr(self, x, kwargs[x])

    # Connect to the message broker
    def connect(self):
        if not self.connection:
            self.connection = BrokerConnection(hostname=self.config.host,
                                               port=self.config.port,
                                               userid=self.config.user,
                                               password=self.config.password,
                                               virtual_host=self.config.vhost)

    # Disconnect from the message broker
    def disconnect(self):
        if self.connection:
            self.connection.close()

    # Support purging messages that are already in the queue on the broker
    # TODO: I think this is only supported by the amqp backend
    def purge_existing_messages(self):

        # Make sure there is an applabel given
        if self.durable and not self.applabel:
            raise InvalidAppLabel('Durable consumers must have an applabel')

        # Purge the queue of existing messages
        self.connection.create_backend().queue_purge(self.applabel)

    # Blocks and calls the callback when a message comes into the queue
    # For info on one script listening to multiple channels, see
    # http://ask.github.com/carrot/changelog.html#id1
    def listen(self, callback=None):

        # One can optionally provide a callback to listen (if it wasn't already)
        if callback:
            self.callback = callback

        # Make suere there is an exchange given
        if not self.exchange:
            raise InvalidExchange(self.exchange)

        # Make sure there is a topic given
        if not self.topic:
            raise InvalidTopic(self.topic)

        # Make sure there is an applabel given
        if self.durable and not self.applabel:
            raise InvalidAppLabel('Durable consumers must have an applabel')

        # Make sure there is a callback given
        if not self.callback or not hasattr(self.callback, '__call__'):
            raise InvalidCallback(self.callback)

        # Connect to the broker if we haven't already
        if not self.connection:
            self.connect()

        # Set up our broker consumer
        self.consumer = Consumer(connection=self.connection,
                                 queue=self.applabel,
                                 exchange=self.exchange,
                                 exchange_type="topic",
                                 auto_declare=False,
                                 routing_key=self.topic)

        # We need to manually create / declare the queue
        self.consumer.backend.queue_declare(
            queue=self.applabel,
            durable=self.durable,
            exclusive=False,
            auto_delete=not self.durable,
            arguments=self.consumer.queue_arguments,
            warn_if_exists=False)

        # No need to manually create the exchange, as the producer creates it
        # and we expect it to just be there

        # We support multiple bindings if we were given an array for the topic
        if not isinstance(self.topic, list):
            self.topic = [self.topic]

        # We need to bind the queue to the exchange with the specified keys
        if self.consumer.queue:
            for routing_key in self.topic:
                self.consumer.backend.queue_bind(queue=self.consumer.queue,
                                                 exchange=self.exchange,
                                                 routing_key=routing_key)
            if self.heartbeat:
                self.consumer.backend.queue_bind(
                    queue=self.consumer.queue,
                    exchange='org.mozilla.exchange.pulse.test',
                    routing_key='heartbeat')

        # Register the callback the user wants
        self.consumer.register_callback(self.callback)

        # This blocks, and then calls the user callback every time a message
        # comes in
        self.consumer.wait()

        # Likely never get here but can't hurt
        self.disconnect()
Beispiel #10
0
class Connection:
    def __init__(self, hostname, port, vhost, userid, password):
        """connects to broker and provides convenience methods"""
        self.broker = BrokerConnection(hostname=hostname, port=port,
                                       userid=userid, password=password,
                                       virtual_host=vhost)
    
    def __del__(self):
        self.broker.close()

    def declare(self, exchange, exchange_type, binding="", queue=""):
        """declares the exchange, the queue and binds the queue to the exchange
        
        exchange        - exchange name
        exchange_type   - direct, topic, fanout
        binding         - binding to queue (optional)
        queue           - queue to bind to exchange using binding (optional)
        """
        if (binding and not queue) or (queue and not binding):
            if queue and not exchange_type == "fanout":
                raise Error("binding and queue are not mutually exclusive")

        consumer = Consumer(connection=self.broker,
                            exchange=exchange, exchange_type=exchange_type,
                            routing_key=binding, queue=queue)
        consumer.declare()
        consumer.close()

    def consume(self, queue, limit=None, callback=None, auto_declare=False):
        """consume messages in queue
        
        queue           - name of queue
        limit           - amount of messages to iterate through (default: no limit)

        callback        - method to call when a new message is received
                          must take two arguments: message_data, message
                          must send the acknowledgement: message.ack()
                          default: print message to stdout and send ack

        auto_declare    - automatically declare the queue (default: false)
        """
        if not callback:
            callback = _consume_callback

        consumer = Consumer(connection=self.broker, queue=queue,
                            auto_declare=auto_declare)

        consumer.register_callback(callback)
        for message in consumer.iterqueue(limit=limit, infinite=False):
            consumer.receive(message.payload, message)

        consumer.close()

    def publish(self, exchange, routing_key, message,
                auto_declare=False, persistent=True):
        """publish a message to exchange using routing_key
        
        exchange        - name of exchange
        routing_key     - interpretation of routing key depends on exchange type
        message         - message content to send
        auto_declare    - automatically declare the exchange (default: false)
        persistent      - store message on disk as well as memory (default: True)
        """
        delivery_mode = 2
        if not persistent:
            delivery_mode = 1

        publisher = Publisher(connection=self.broker,
                              exchange=exchange, routing_key=routing_key,
                              auto_declare=auto_declare)

        publisher.send(message, delivery_mode=delivery_mode)
        publisher.close()
Beispiel #11
0
class GenericPublisher(object):
    def __init__(self, config, exchange=None, connect=True):
        self.config = config
        self.exchange = exchange
        self.connection = None
        if connect:
            self.connect()

    # Connect to the message broker
    def connect(self):
        if not self.connection:
            self.connection = BrokerConnection(hostname=self.config.host,
                                               port=self.config.port,
                                               userid=self.config.user,
                                               password=self.config.password,
                                               virtual_host=self.config.vhost)

    # Disconnect from the message broker
    def disconnect(self):
        if self.connection:
            self.connection.close()
            self.connection = None

    # Used to publish a pulse message to the proper exchange
    def publish(self, message):

        # Make suere there is an exchange given
        if not self.exchange:
            raise InvalidExchange(self.exchange)

        # Make sure there is a message given
        if not message:
            raise MalformedMessage(message)

        # Have the message prepare and validate itself
        message._prepare()

        # Connect to the broker if we haven't already
        if not self.connection:
            self.connect()

        # Set up our broker publisher
        self.publisher = Publisher(connection=self.connection,
                                   exchange=self.exchange,
                                   exchange_type="topic",
                                   routing_key=message.routing_key)

        # The message is actually a simple envelope format with a payload and
        # some metadata
        final_data = {}
        final_data['payload'] = message.data
        final_data['_meta'] = message.metadata.copy()
        final_data['_meta'].update({
            'exchange':
            self.exchange,
            'routing_key':
            message.routing_key,
            'serializer':
            self.config.serializer,
            'sent':
            time_to_string(datetime.now(timezone(self.config.broker_timezone)))
        })

        # Send the message
        self.publisher.send(final_data, serializer=self.config.serializer)

        # Close the publishing connection
        self.publisher.close()