def test_multiple_urls_hostname(self):
conn = Connection(['example.com;amqp://example.com'])
assert conn.as_uri() == 'amqp://*****:*****@example.com:5672//'
conn = Connection(['example.com', 'amqp://example.com'])
assert conn.as_uri() == 'amqp://*****:*****@example.com:5672//'
conn = Connection('example.com;example.com;')
assert conn.as_uri() == 'amqp://*****:*****@example.com:5672//'
def test_parse_generated_as_uri(self):
conn = Connection(self.url)
info = conn.info()
for k, v in self.expected.items():
assert info[k] == v
# by default almost the same- no password
assert conn.as_uri() == self.nopass
assert conn.as_uri(include_password=True) == self.url
def test_parse_generated_as_uri(self):
conn = Connection(self.url)
info = conn.info()
for k, v in self.expected.items():
self.assertEqual(info[k], v)
# by default almost the same- no password
self.assertEqual(conn.as_uri(), self.nopass)
self.assertEqual(conn.as_uri(include_password=True), self.url)
def test_parse_generated_as_uri(self):
conn = Connection(self.url)
info = conn.info()
for k, v in self.expected.items():
self.assertEqual(info[k], v)
# by default almost the same- no password
self.assertEqual(conn.as_uri(), self.nopass)
self.assertEqual(conn.as_uri(include_password=True), self.url)
def test_parse_generated_as_uri(self):
conn = Connection(self.url)
info = conn.info()
for k, v in self.expected.items():
assert info[k] == v
# by default almost the same- no password
assert conn.as_uri() == self.nopass
assert conn.as_uri(include_password=True) == self.url
def test_uri_passthrough(self):
transport = Mock(name='transport')
with patch('kombu.connection.get_transport_cls') as gtc:
gtc.return_value = transport
transport.can_parse_url = True
with patch('kombu.connection.parse_url') as parse_url:
c = Connection('foo+mysql://some_host')
assert c.transport_cls == 'foo'
parse_url.assert_not_called()
assert c.hostname == 'mysql://some_host'
assert c.as_uri().startswith('foo+')
with patch('kombu.connection.parse_url') as parse_url:
c = Connection('mysql://some_host', transport='foo')
assert c.transport_cls == 'foo'
parse_url.assert_not_called()
assert c.hostname == 'mysql://some_host'
c = Connection('pyamqp+sqlite://some_host')
assert c.as_uri().startswith('pyamqp+')
def test_uri_passthrough(self):
transport = Mock(name='transport')
with patch('kombu.connection.get_transport_cls') as gtc:
gtc.return_value = transport
transport.can_parse_url = True
with patch('kombu.connection.parse_url') as parse_url:
c = Connection('foo+mysql://some_host')
assert c.transport_cls == 'foo'
parse_url.assert_not_called()
assert c.hostname == 'mysql://some_host'
assert c.as_uri().startswith('foo+')
with patch('kombu.connection.parse_url') as parse_url:
c = Connection('mysql://some_host', transport='foo')
assert c.transport_cls == 'foo'
parse_url.assert_not_called()
assert c.hostname == 'mysql://some_host'
c = Connection('pyamqp+sqlite://some_host')
assert c.as_uri().startswith('pyamqp+')
def test_uri_passthrough(self):
transport = Mock(name="transport")
with patch("kombu.connection.get_transport_cls") as gtc:
gtc.return_value = transport
transport.can_parse_url = True
with patch("kombu.connection.parse_url") as parse_url:
c = Connection("foo+mysql://some_host")
self.assertEqual(c.transport_cls, "foo")
self.assertFalse(parse_url.called)
self.assertEqual(c.hostname, "mysql://some_host")
self.assertTrue(c.as_uri().startswith("foo+"))
with patch("kombu.connection.parse_url") as parse_url:
c = Connection("mysql://some_host", transport="foo")
self.assertEqual(c.transport_cls, "foo")
self.assertFalse(parse_url.called)
self.assertEqual(c.hostname, "mysql://some_host")
c = Connection("pyamqp+sqlite://some_host")
self.assertTrue(c.as_uri().startswith("pyamqp+"))
def test_uri_passthrough(self):
transport = Mock(name='transport')
with patch('kombu.connection.get_transport_cls') as gtc:
gtc.return_value = transport
transport.can_parse_url = True
with patch('kombu.connection.parse_url') as parse_url:
c = Connection('foo+mysql://some_host')
self.assertEqual(c.transport_cls, 'foo')
self.assertFalse(parse_url.called)
self.assertEqual(c.hostname, 'mysql://some_host')
self.assertTrue(c.as_uri().startswith('foo+'))
with patch('kombu.connection.parse_url') as parse_url:
c = Connection('mysql://some_host', transport='foo')
self.assertEqual(c.transport_cls, 'foo')
self.assertFalse(parse_url.called)
self.assertEqual(c.hostname, 'mysql://some_host')
c = Connection('pyamqp+sqlite://some_host')
self.assertTrue(c.as_uri().startswith('pyamqp+'))
def test_uri_passthrough(self):
transport = Mock(name='transport')
with patch('kombu.connection.get_transport_cls') as gtc:
gtc.return_value = transport
transport.can_parse_url = True
with patch('kombu.connection.parse_url') as parse_url:
c = Connection('foo+mysql://some_host')
self.assertEqual(c.transport_cls, 'foo')
self.assertFalse(parse_url.called)
self.assertEqual(c.hostname, 'mysql://some_host')
self.assertTrue(c.as_uri().startswith('foo+'))
with patch('kombu.connection.parse_url') as parse_url:
c = Connection('mysql://some_host', transport='foo')
self.assertEqual(c.transport_cls, 'foo')
self.assertFalse(parse_url.called)
self.assertEqual(c.hostname, 'mysql://some_host')
c = Connection('pyamqp+sqlite://some_host')
self.assertTrue(c.as_uri().startswith('pyamqp+'))
def test_uri_passthrough(self):
from kombu import connection as mod
prev, mod.URI_PASSTHROUGH = mod.URI_PASSTHROUGH, set(['foo'])
try:
with patch('kombu.connection.parse_url') as parse_url:
c = Connection('foo+mysql://some_host')
self.assertEqual(c.transport_cls, 'foo')
self.assertFalse(parse_url.called)
self.assertEqual(c.hostname, 'mysql://some_host')
self.assertTrue(c.as_uri().startswith('foo+'))
with patch('kombu.connection.parse_url') as parse_url:
c = Connection('mysql://some_host', transport='foo')
self.assertEqual(c.transport_cls, 'foo')
self.assertFalse(parse_url.called)
self.assertEqual(c.hostname, 'mysql://some_host')
finally:
mod.URI_PASSTHROUGH = prev
c = Connection('amqp+sqlite://some_host')
self.assertTrue(c.as_uri().startswith('amqp+'))
def test_uri_passthrough(self):
from kombu import connection as mod
prev, mod.URI_PASSTHROUGH = mod.URI_PASSTHROUGH, set(['foo'])
try:
with patch('kombu.connection.parse_url') as parse_url:
c = Connection('foo+mysql://some_host')
self.assertEqual(c.transport_cls, 'foo')
self.assertFalse(parse_url.called)
self.assertEqual(c.hostname, 'mysql://some_host')
self.assertTrue(c.as_uri().startswith('foo+'))
with patch('kombu.connection.parse_url') as parse_url:
c = Connection('mysql://some_host', transport='foo')
self.assertEqual(c.transport_cls, 'foo')
self.assertFalse(parse_url.called)
self.assertEqual(c.hostname, 'mysql://some_host')
finally:
mod.URI_PASSTHROUGH = prev
c = Connection('amqp+sqlite://some_host')
self.assertTrue(c.as_uri().startswith('amqp+'))
class Publisher(KombuConfReader):
def __init__(self, config):
self._log = log.getLogger()
KombuConfReader.__init__(self, config)
self.connection = Connection(self.broker_url)
try:
self._init_amqp()
except Exception as exc:
self._log.error('Publisher fail in init connection: %s' % exc)
raise
def _init_amqp(self):
"""Init AMQP objects after connection"""
self.producer = self.connection.Producer()
self.exchange = Exchange(
self.exchange_name,
channel=self.connection.channel(),
type=self.exchange_type,
durable=self.exchange_is_durable)
self.queue = Queue(
self.queue_name,
self.exchange,
channel=self.connection.channel(),
durable=self.queue_is_durable,
routing_key=self.routing_key,
queue_arguments=self.queue_args)
# We declare object to broker. this way only we can
# ensure to publish to an existing queue and routing_key
# AMQP work this way, not a library principle
self.exchange.declare()
self.queue.declare()
def switch_connection(self):
"""Switch AMQP connection from url to backup_url and vice versa"""
self._log.warn('Switching AMQP connection from %s' %
self.connection.as_uri())
if (self.connection.hostname in self.broker_url
and self.broker_backup_url):
self.connection = Connection(self.broker_backup_url)
elif self.connection.hostname in self.broker_backup_url:
self.connection = Connection(self.broker_url)
else:
raise URLError('Invalid current URI to switch connection : %s' %
self.connection.as_uri())
self._init_amqp()
def _publish(self, msg):
"""Publish message ensuring connection is available"""
publish = self.connection.ensure(
self.producer, self.producer.publish, max_retries=3)
publish(msg, exchange=self.exchange,
routing_key=self.routing_key,
serializer=self.serializer,
compression=self.compression)
return True
def publish(self, msg):
"""
must return True/False if message is well publish or not
"""
try:
return self._publish(msg)
except Exception as exc:
try:
self.switch_connection()
return self._publish(msg)
except Exception as exc:
self._log.error('Publish fail when switching connection: %s' %
exc)
return False
def test_as_uri_when_mongodb(self):
x = Connection('mongodb://localhost')
self.assertTrue(x.as_uri())
def test_as_uri_when_prefix(self):
conn = Connection('memory://')
conn.uri_prefix = 'foo'
self.assertTrue(conn.as_uri().startswith('foo+memory://'))
class Publisher(KombuConfReader):
def __init__(self, config):
self._log = log.getLogger()
KombuConfReader.__init__(self, config)
self.connection = Connection(self.broker_url)
try:
self._init_amqp()
except Exception as exc:
self._log.error('Publisher fail in init connection: %s' % exc)
raise
def _init_amqp(self):
"""Init AMQP objects after connection"""
self.producer = self.connection.Producer()
self.exchange = Exchange(self.exchange_name,
channel=self.connection.channel(),
type=self.exchange_type,
durable=self.exchange_is_durable)
self.queue = Queue(self.queue_name,
self.exchange,
channel=self.connection.channel(),
durable=self.queue_is_durable,
routing_key=self.routing_key,
queue_arguments=self.queue_args)
# We declare object to broker. this way only we can
# ensure to publish to an existing queue and routing_key
# AMQP work this way, not a library principle
self.exchange.declare()
self.queue.declare()
def switch_connection(self):
"""Switch AMQP connection from url to backup_url and vice versa"""
self._log.warn('Switching AMQP connection from %s' %
self.connection.as_uri())
if (self.connection.hostname in self.broker_url
and self.broker_backup_url):
self.connection = Connection(self.broker_backup_url)
elif self.connection.hostname in self.broker_backup_url:
self.connection = Connection(self.broker_url)
else:
raise URLError('Invalid current URI to switch connection : %s' %
self.connection.as_uri())
self._init_amqp()
def _publish(self, msg):
"""Publish message ensuring connection is available"""
publish = self.connection.ensure(self.producer,
self.producer.publish,
max_retries=3)
publish(msg,
exchange=self.exchange,
routing_key=self.routing_key,
serializer=self.serializer,
compression=self.compression)
return True
def publish(self, msg):
"""
must return True/False if message is well publish or not
"""
try:
return self._publish(msg)
except Exception as exc:
try:
self.switch_connection()
return self._publish(msg)
except Exception as exc:
self._log.error('Publish fail when switching connection: %s' %
exc)
return False
def test_as_uri_when_mongodb(self):
pytest.importorskip('pymongo')
x = Connection('mongodb://localhost')
assert x.as_uri()
def test_as_uri_when_prefix(self):
conn = Connection('redis+socket:///var/spool/x/y/z/redis.sock')
assert conn.as_uri() == 'redis+socket:///var/spool/x/y/z/redis.sock'
def test_as_uri_when_prefix(self):
conn = Connection('redis+socket:///var/spool/x/y/z/redis.sock')
assert conn.as_uri() == 'redis+socket:///var/spool/x/y/z/redis.sock'
def test_as_uri_when_prefix(self):
conn = Connection("memory://")
conn.uri_prefix = "foo"
self.assertTrue(conn.as_uri().startswith("foo+memory://"))
def test_parse_generated_as_uri_pg(self):
conn = Connection(self.pg_url)
assert conn.as_uri() == self.pg_nopass
assert conn.as_uri(include_password=True) == self.pg_url
class AMQPPublisher:
"""Simple publisher for connecting and publishing messages to a AMQP broker exchange.
An optional formatter instance can be passed in order to format the message before sending it,
(it must implement a method named `format`).
Connections are lazy and stored in a global pool so that AMQPPublisher instances with the same uri
in the same process will use the same connection object.
The additional kwargs `options` are passed as additional parameters to kombu.Producer.publish
please refer to: https://kombu.readthedocs.io/en/latest/userguide/producers.html#reference
"""
def __init__(self,
uri,
exchange_name,
exchange_type='topic',
routing_key=None,
formatter=None,
**options):
self.connection = Connection(uri)
self.exchange = Exchange(exchange_name, type=exchange_type)
self.routing_key = routing_key
self.formatter = formatter
self.options = {**OPTIONS, **options}
class AMQPProducer:
def __init__(self, producer, exchange, routing_key, formatter,
options):
self.producer = producer
self.exchange = exchange
self.routing_key = routing_key
self.formatter = formatter
self.options = options
def publish(self, msg, routing_key=None):
body = self.formatter.format(
msg) if self.formatter is not None else msg
key = routing_key if routing_key else self.routing_key
if not key:
raise ValueError(
'Invalid routing_key: {} (Producer routing_key is: {})'.
format(routing_key, self.routing_key))
self.producer.publish(body,
exchange=self.exchange,
routing_key=key,
declare=(self.exchange, ),
**self.options)
logger.debug(
'Published AMQP message. Exchange: %s, Routing Key: %s, Body: %s',
self.exchange, key, str(body))
@contextmanager
def acquire_producer(self, block=True):
"""Acquire a producer from the global pool for holding a connection/channel while publishing.
Useful for removing the overhead of acquiring a producer from the global pool when
publishing several messages in a row.
"""
with producers[self.connection].acquire(block=block) as producer:
logger.debug('Acquired producer for connection %s',
self.connection.as_uri())
yield self.AMQPProducer(producer, self.exchange, self.routing_key,
self.formatter, self.options)
logger.debug('Released producer for connection %s',
self.connection.as_uri())
def publish(self, msg, routing_key=None):
"""Acquire a producer from the global pool and publish a single message.
An optional `routing_key` can be passed. If no `routing_key` is passed
the instance one is used (in that case, if no `routing_key` has been set in the instance)
a ValueError is raised.
"""
with self.acquire_producer() as producer:
producer.publish(msg, routing_key)
def test_as_uri_when_prefix(self):
conn = Connection('memory://')
conn.uri_prefix = 'foo'
self.assertTrue(conn.as_uri().startswith('foo+memory://'))
def test_as_uri_when_prefix(self):
conn = Connection('redis+socket:///var/spool/x/y/z/redis.sock')
self.assertEqual(
conn.as_uri(), 'redis+socket:///var/spool/x/y/z/redis.sock',
)
def test_as_uri_when_mongodb(self):
x = Connection('mongodb://localhost')
self.assertTrue(x.as_uri())
def test_parse_generated_as_uri_pg(self):
pytest.importorskip('sqlalchemy')
conn = Connection(self.pg_url)
assert conn.as_uri() == self.pg_nopass
assert conn.as_uri(include_password=True) == self.pg_url