Ejemplo n.º 1
0
 def test_multiple_urls_hostname(self):
     conn = Connection(['example.com;amqp://example.com'])
     assert conn.as_uri() == 'amqp://*****:*****@example.com:5672//'
     conn = Connection(['example.com', 'amqp://example.com'])
     assert conn.as_uri() == 'amqp://*****:*****@example.com:5672//'
     conn = Connection('example.com;example.com;')
     assert conn.as_uri() == 'amqp://*****:*****@example.com:5672//'
Ejemplo n.º 2
0
 def test_parse_generated_as_uri(self):
     conn = Connection(self.url)
     info = conn.info()
     for k, v in self.expected.items():
         assert info[k] == v
     # by default almost the same- no password
     assert conn.as_uri() == self.nopass
     assert conn.as_uri(include_password=True) == self.url
Ejemplo n.º 3
0
 def test_parse_generated_as_uri(self):
     conn = Connection(self.url)
     info = conn.info()
     for k, v in self.expected.items():
         self.assertEqual(info[k], v)
     # by default almost the same- no password
     self.assertEqual(conn.as_uri(), self.nopass)
     self.assertEqual(conn.as_uri(include_password=True), self.url)
Ejemplo n.º 4
0
 def test_parse_generated_as_uri(self):
     conn = Connection(self.url)
     info = conn.info()
     for k, v in self.expected.items():
         self.assertEqual(info[k], v)
     # by default almost the same- no password
     self.assertEqual(conn.as_uri(), self.nopass)
     self.assertEqual(conn.as_uri(include_password=True), self.url)
Ejemplo n.º 5
0
 def test_parse_generated_as_uri(self):
     conn = Connection(self.url)
     info = conn.info()
     for k, v in self.expected.items():
         assert info[k] == v
     # by default almost the same- no password
     assert conn.as_uri() == self.nopass
     assert conn.as_uri(include_password=True) == self.url
Ejemplo n.º 6
0
 def test_uri_passthrough(self):
     transport = Mock(name='transport')
     with patch('kombu.connection.get_transport_cls') as gtc:
         gtc.return_value = transport
         transport.can_parse_url = True
         with patch('kombu.connection.parse_url') as parse_url:
             c = Connection('foo+mysql://some_host')
             assert c.transport_cls == 'foo'
             parse_url.assert_not_called()
             assert c.hostname == 'mysql://some_host'
             assert c.as_uri().startswith('foo+')
         with patch('kombu.connection.parse_url') as parse_url:
             c = Connection('mysql://some_host', transport='foo')
             assert c.transport_cls == 'foo'
             parse_url.assert_not_called()
             assert c.hostname == 'mysql://some_host'
     c = Connection('pyamqp+sqlite://some_host')
     assert c.as_uri().startswith('pyamqp+')
Ejemplo n.º 7
0
 def test_uri_passthrough(self):
     transport = Mock(name='transport')
     with patch('kombu.connection.get_transport_cls') as gtc:
         gtc.return_value = transport
         transport.can_parse_url = True
         with patch('kombu.connection.parse_url') as parse_url:
             c = Connection('foo+mysql://some_host')
             assert c.transport_cls == 'foo'
             parse_url.assert_not_called()
             assert c.hostname == 'mysql://some_host'
             assert c.as_uri().startswith('foo+')
         with patch('kombu.connection.parse_url') as parse_url:
             c = Connection('mysql://some_host', transport='foo')
             assert c.transport_cls == 'foo'
             parse_url.assert_not_called()
             assert c.hostname == 'mysql://some_host'
     c = Connection('pyamqp+sqlite://some_host')
     assert c.as_uri().startswith('pyamqp+')
Ejemplo n.º 8
0
 def test_uri_passthrough(self):
     transport = Mock(name="transport")
     with patch("kombu.connection.get_transport_cls") as gtc:
         gtc.return_value = transport
         transport.can_parse_url = True
         with patch("kombu.connection.parse_url") as parse_url:
             c = Connection("foo+mysql://some_host")
             self.assertEqual(c.transport_cls, "foo")
             self.assertFalse(parse_url.called)
             self.assertEqual(c.hostname, "mysql://some_host")
             self.assertTrue(c.as_uri().startswith("foo+"))
         with patch("kombu.connection.parse_url") as parse_url:
             c = Connection("mysql://some_host", transport="foo")
             self.assertEqual(c.transport_cls, "foo")
             self.assertFalse(parse_url.called)
             self.assertEqual(c.hostname, "mysql://some_host")
     c = Connection("pyamqp+sqlite://some_host")
     self.assertTrue(c.as_uri().startswith("pyamqp+"))
Ejemplo n.º 9
0
 def test_uri_passthrough(self):
     transport = Mock(name='transport')
     with patch('kombu.connection.get_transport_cls') as gtc:
         gtc.return_value = transport
         transport.can_parse_url = True
         with patch('kombu.connection.parse_url') as parse_url:
             c = Connection('foo+mysql://some_host')
             self.assertEqual(c.transport_cls, 'foo')
             self.assertFalse(parse_url.called)
             self.assertEqual(c.hostname, 'mysql://some_host')
             self.assertTrue(c.as_uri().startswith('foo+'))
         with patch('kombu.connection.parse_url') as parse_url:
             c = Connection('mysql://some_host', transport='foo')
             self.assertEqual(c.transport_cls, 'foo')
             self.assertFalse(parse_url.called)
             self.assertEqual(c.hostname, 'mysql://some_host')
     c = Connection('pyamqp+sqlite://some_host')
     self.assertTrue(c.as_uri().startswith('pyamqp+'))
Ejemplo n.º 10
0
 def test_uri_passthrough(self):
     transport = Mock(name='transport')
     with patch('kombu.connection.get_transport_cls') as gtc:
         gtc.return_value = transport
         transport.can_parse_url = True
         with patch('kombu.connection.parse_url') as parse_url:
             c = Connection('foo+mysql://some_host')
             self.assertEqual(c.transport_cls, 'foo')
             self.assertFalse(parse_url.called)
             self.assertEqual(c.hostname, 'mysql://some_host')
             self.assertTrue(c.as_uri().startswith('foo+'))
         with patch('kombu.connection.parse_url') as parse_url:
             c = Connection('mysql://some_host', transport='foo')
             self.assertEqual(c.transport_cls, 'foo')
             self.assertFalse(parse_url.called)
             self.assertEqual(c.hostname, 'mysql://some_host')
     c = Connection('pyamqp+sqlite://some_host')
     self.assertTrue(c.as_uri().startswith('pyamqp+'))
Ejemplo n.º 11
0
 def test_uri_passthrough(self):
     from kombu import connection as mod
     prev, mod.URI_PASSTHROUGH = mod.URI_PASSTHROUGH, set(['foo'])
     try:
         with patch('kombu.connection.parse_url') as parse_url:
             c = Connection('foo+mysql://some_host')
             self.assertEqual(c.transport_cls, 'foo')
             self.assertFalse(parse_url.called)
             self.assertEqual(c.hostname, 'mysql://some_host')
             self.assertTrue(c.as_uri().startswith('foo+'))
         with patch('kombu.connection.parse_url') as parse_url:
             c = Connection('mysql://some_host', transport='foo')
             self.assertEqual(c.transport_cls, 'foo')
             self.assertFalse(parse_url.called)
             self.assertEqual(c.hostname, 'mysql://some_host')
     finally:
         mod.URI_PASSTHROUGH = prev
     c = Connection('amqp+sqlite://some_host')
     self.assertTrue(c.as_uri().startswith('amqp+'))
Ejemplo n.º 12
0
 def test_uri_passthrough(self):
     from kombu import connection as mod
     prev, mod.URI_PASSTHROUGH = mod.URI_PASSTHROUGH, set(['foo'])
     try:
         with patch('kombu.connection.parse_url') as parse_url:
             c = Connection('foo+mysql://some_host')
             self.assertEqual(c.transport_cls, 'foo')
             self.assertFalse(parse_url.called)
             self.assertEqual(c.hostname, 'mysql://some_host')
             self.assertTrue(c.as_uri().startswith('foo+'))
         with patch('kombu.connection.parse_url') as parse_url:
             c = Connection('mysql://some_host', transport='foo')
             self.assertEqual(c.transport_cls, 'foo')
             self.assertFalse(parse_url.called)
             self.assertEqual(c.hostname, 'mysql://some_host')
     finally:
         mod.URI_PASSTHROUGH = prev
     c = Connection('amqp+sqlite://some_host')
     self.assertTrue(c.as_uri().startswith('amqp+'))
Ejemplo n.º 13
0
class Publisher(KombuConfReader):

    def __init__(self, config):
        self._log = log.getLogger()
        KombuConfReader.__init__(self, config)

        self.connection = Connection(self.broker_url)
        try:
            self._init_amqp()
        except Exception as exc:
            self._log.error('Publisher fail in init connection: %s' % exc)
            raise

    def _init_amqp(self):
        """Init AMQP objects after connection"""
        self.producer = self.connection.Producer()
        self.exchange = Exchange(
            self.exchange_name,
            channel=self.connection.channel(),
            type=self.exchange_type,
            durable=self.exchange_is_durable)

        self.queue = Queue(
            self.queue_name,
            self.exchange,
            channel=self.connection.channel(),
            durable=self.queue_is_durable,
            routing_key=self.routing_key,
            queue_arguments=self.queue_args)

        # We declare object to broker. this way only we can
        # ensure to publish to an existing queue and routing_key
        # AMQP work this way, not a library principle
        self.exchange.declare()
        self.queue.declare()

    def switch_connection(self):
        """Switch AMQP connection from url to backup_url and vice versa"""
        self._log.warn('Switching AMQP connection from %s' %
                       self.connection.as_uri())
        if (self.connection.hostname in self.broker_url
                and self.broker_backup_url):
            self.connection = Connection(self.broker_backup_url)
        elif self.connection.hostname in self.broker_backup_url:
            self.connection = Connection(self.broker_url)
        else:
            raise URLError('Invalid current URI to switch connection : %s' %
                           self.connection.as_uri())
        self._init_amqp()

    def _publish(self, msg):
        """Publish message ensuring connection is available"""
        publish = self.connection.ensure(
            self.producer, self.producer.publish, max_retries=3)

        publish(msg, exchange=self.exchange,
                routing_key=self.routing_key,
                serializer=self.serializer,
                compression=self.compression)

        return True

    def publish(self, msg):
        """
        must return True/False if message is well publish or not
        """
        try:
            return self._publish(msg)
        except Exception as exc:
            try:
                self.switch_connection()
                return self._publish(msg)
            except Exception as exc:
                self._log.error('Publish fail when switching connection: %s' %
                                exc)
            return False
Ejemplo n.º 14
0
 def test_as_uri_when_mongodb(self):
     x = Connection('mongodb://localhost')
     self.assertTrue(x.as_uri())
Ejemplo n.º 15
0
 def test_as_uri_when_prefix(self):
     conn = Connection('memory://')
     conn.uri_prefix = 'foo'
     self.assertTrue(conn.as_uri().startswith('foo+memory://'))
Ejemplo n.º 16
0
class Publisher(KombuConfReader):
    def __init__(self, config):
        self._log = log.getLogger()
        KombuConfReader.__init__(self, config)

        self.connection = Connection(self.broker_url)
        try:
            self._init_amqp()
        except Exception as exc:
            self._log.error('Publisher fail in init connection: %s' % exc)
            raise

    def _init_amqp(self):
        """Init AMQP objects after connection"""
        self.producer = self.connection.Producer()
        self.exchange = Exchange(self.exchange_name,
                                 channel=self.connection.channel(),
                                 type=self.exchange_type,
                                 durable=self.exchange_is_durable)

        self.queue = Queue(self.queue_name,
                           self.exchange,
                           channel=self.connection.channel(),
                           durable=self.queue_is_durable,
                           routing_key=self.routing_key,
                           queue_arguments=self.queue_args)

        # We declare object to broker. this way only we can
        # ensure to publish to an existing queue and routing_key
        # AMQP work this way, not a library principle
        self.exchange.declare()
        self.queue.declare()

    def switch_connection(self):
        """Switch AMQP connection from url to backup_url and vice versa"""
        self._log.warn('Switching AMQP connection from %s' %
                       self.connection.as_uri())
        if (self.connection.hostname in self.broker_url
                and self.broker_backup_url):
            self.connection = Connection(self.broker_backup_url)
        elif self.connection.hostname in self.broker_backup_url:
            self.connection = Connection(self.broker_url)
        else:
            raise URLError('Invalid current URI to switch connection : %s' %
                           self.connection.as_uri())
        self._init_amqp()

    def _publish(self, msg):
        """Publish message ensuring connection is available"""
        publish = self.connection.ensure(self.producer,
                                         self.producer.publish,
                                         max_retries=3)

        publish(msg,
                exchange=self.exchange,
                routing_key=self.routing_key,
                serializer=self.serializer,
                compression=self.compression)

        return True

    def publish(self, msg):
        """
        must return True/False if message is well publish or not
        """
        try:
            return self._publish(msg)
        except Exception as exc:
            try:
                self.switch_connection()
                return self._publish(msg)
            except Exception as exc:
                self._log.error('Publish fail when switching connection: %s' %
                                exc)
            return False
Ejemplo n.º 17
0
 def test_as_uri_when_mongodb(self):
     pytest.importorskip('pymongo')
     x = Connection('mongodb://localhost')
     assert x.as_uri()
Ejemplo n.º 18
0
 def test_as_uri_when_prefix(self):
     conn = Connection('redis+socket:///var/spool/x/y/z/redis.sock')
     assert conn.as_uri() == 'redis+socket:///var/spool/x/y/z/redis.sock'
Ejemplo n.º 19
0
 def test_as_uri_when_prefix(self):
     conn = Connection('redis+socket:///var/spool/x/y/z/redis.sock')
     assert conn.as_uri() == 'redis+socket:///var/spool/x/y/z/redis.sock'
Ejemplo n.º 20
0
 def test_as_uri_when_prefix(self):
     conn = Connection("memory://")
     conn.uri_prefix = "foo"
     self.assertTrue(conn.as_uri().startswith("foo+memory://"))
Ejemplo n.º 21
0
 def test_parse_generated_as_uri_pg(self):
     conn = Connection(self.pg_url)
     assert conn.as_uri() == self.pg_nopass
     assert conn.as_uri(include_password=True) == self.pg_url
class AMQPPublisher:
    """Simple publisher for connecting and publishing messages to a AMQP broker exchange.
    An optional formatter instance can be passed in order to format the message before sending it,
    (it must implement a method named `format`).
    Connections are lazy and stored in a global pool so that AMQPPublisher instances with the same uri
    in the same process will use the same connection object.
    The additional kwargs `options` are passed as additional parameters to kombu.Producer.publish
    please refer to: https://kombu.readthedocs.io/en/latest/userguide/producers.html#reference
    """
    def __init__(self,
                 uri,
                 exchange_name,
                 exchange_type='topic',
                 routing_key=None,
                 formatter=None,
                 **options):
        self.connection = Connection(uri)
        self.exchange = Exchange(exchange_name, type=exchange_type)
        self.routing_key = routing_key
        self.formatter = formatter
        self.options = {**OPTIONS, **options}

    class AMQPProducer:
        def __init__(self, producer, exchange, routing_key, formatter,
                     options):
            self.producer = producer
            self.exchange = exchange
            self.routing_key = routing_key
            self.formatter = formatter
            self.options = options

        def publish(self, msg, routing_key=None):
            body = self.formatter.format(
                msg) if self.formatter is not None else msg
            key = routing_key if routing_key else self.routing_key
            if not key:
                raise ValueError(
                    'Invalid routing_key: {} (Producer routing_key is: {})'.
                    format(routing_key, self.routing_key))
            self.producer.publish(body,
                                  exchange=self.exchange,
                                  routing_key=key,
                                  declare=(self.exchange, ),
                                  **self.options)
            logger.debug(
                'Published AMQP message. Exchange: %s, Routing Key: %s, Body: %s',
                self.exchange, key, str(body))

    @contextmanager
    def acquire_producer(self, block=True):
        """Acquire a producer from the global pool for holding a connection/channel while publishing.
        Useful for removing the overhead of acquiring a producer from the global pool when
        publishing several messages in a row.
        """
        with producers[self.connection].acquire(block=block) as producer:
            logger.debug('Acquired producer for connection %s',
                         self.connection.as_uri())
            yield self.AMQPProducer(producer, self.exchange, self.routing_key,
                                    self.formatter, self.options)
        logger.debug('Released producer for connection %s',
                     self.connection.as_uri())

    def publish(self, msg, routing_key=None):
        """Acquire a producer from the global pool and publish a single message.
        An optional `routing_key` can be passed. If no `routing_key` is passed
        the instance one is used (in that case, if no `routing_key` has been set in the instance)
        a ValueError is raised.
        """
        with self.acquire_producer() as producer:
            producer.publish(msg, routing_key)
Ejemplo n.º 23
0
 def test_as_uri_when_prefix(self):
     conn = Connection('memory://')
     conn.uri_prefix = 'foo'
     self.assertTrue(conn.as_uri().startswith('foo+memory://'))
Ejemplo n.º 24
0
 def test_as_uri_when_prefix(self):
     conn = Connection('redis+socket:///var/spool/x/y/z/redis.sock')
     self.assertEqual(
         conn.as_uri(), 'redis+socket:///var/spool/x/y/z/redis.sock',
     )
Ejemplo n.º 25
0
 def test_as_uri_when_mongodb(self):
     x = Connection('mongodb://localhost')
     self.assertTrue(x.as_uri())
Ejemplo n.º 26
0
 def test_parse_generated_as_uri_pg(self):
     pytest.importorskip('sqlalchemy')
     conn = Connection(self.pg_url)
     assert conn.as_uri() == self.pg_nopass
     assert conn.as_uri(include_password=True) == self.pg_url