def test_redis_info_raises(self):
pool = Mock(name='pool')
pool_at_init = [pool]
client = Mock(name='client')
class XChannel(Channel):
def __init__(self, *args, **kwargs):
self._pool = pool_at_init[0]
super(XChannel, self).__init__(*args, **kwargs)
def _get_client(self):
return lambda *_, **__: client
class XTransport(Transport):
Channel = XChannel
conn = Connection(transport=XTransport)
client.info.side_effect = RuntimeError()
with self.assertRaises(RuntimeError):
conn.channel()
pool.disconnect.assert_called_with()
pool.disconnect.reset_mock()
pool_at_init = [None]
with self.assertRaises(RuntimeError):
conn.channel()
self.assertFalse(pool.disconnect.called)
def test_region(self):
import boto3
_environ = dict(os.environ)
# when the region is unspecified
connection = Connection(transport=SQS.Transport)
channel = connection.channel()
assert channel.transport_options.get('region') is None
# the default region is us-east-1
assert channel.region == 'us-east-1'
# when boto3 picks a region
os.environ['AWS_DEFAULT_REGION'] = 'us-east-2'
assert boto3.Session().region_name == 'us-east-2'
# the default region should match
connection = Connection(transport=SQS.Transport)
channel = connection.channel()
assert channel.region == 'us-east-2'
# when transport_options are provided
connection = Connection(transport=SQS.Transport, transport_options={
'region': 'us-west-2'
})
channel = connection.channel()
assert channel.transport_options.get('region') == 'us-west-2'
# the specified region should be used
assert connection.channel().region == 'us-west-2'
os.environ.clear()
os.environ.update(_environ)
def test_can_create_connection(self):
from redis.exceptions import ConnectionError
with self.assertRaises(ConnectionError):
connection = Connection('sentinel://localhost:65534/', transport_options={
'master_name': 'not_important'
})
connection.channel()
def test_method_called(self):
from kombu.transport.redis import SentinelChannel
with patch.object(SentinelChannel, '_sentinel_managed_pool') as patched:
connection = Connection('sentinel://localhost:65534/', transport_options={
'master_name': 'not_important'
})
connection.channel()
self.assertTrue(patched.called)
def test_getting_master_from_sentinel(self):
from redis.sentinel import Sentinel
with patch.object(Sentinel, '__new__') as patched:
connection = Connection('sentinel://localhost:65534/', transport_options={
'master_name': 'not_important'
})
connection.channel()
self.assertTrue(patched)
sentinel_obj = patched.return_value
self.assertTrue(sentinel_obj.master_for.called, 'master_for was not called')
sentinel_obj.master_for.assert_called_with('not_important', ANY)
self.assertTrue(sentinel_obj.master_for().connection_pool.get_connection.called, 'get_connection on redis connection pool was not called')
def test_getting_master_from_sentinel(self):
with patch('redis.sentinel.Sentinel') as patched:
connection = Connection(
'sentinel://localhost:65534/',
transport_options={
'master_name': 'not_important',
},
)
connection.channel()
assert patched
master_for = patched.return_value.master_for
master_for.assert_called()
master_for.assert_called_with('not_important', ANY)
master_for().connection_pool.get_connection.assert_called()
class AMQPConnection(object):
"""
A class representing a connection to an amqp broker.
"""
def __init__(self, configuration):
self.logger = logging.getLogger('narc.amqp.AMQPConnection')
self.url = str.format("amqp://{hostname}:{port}", **dict(configuration['AMQP'].items()))
if 'virtual host' in configuration['AMQP'] and configuration['AMQP']['virtual host'] != '':
self.url = str.format("{}/{}", self.url, configuration['AMQP']['virtual host'])
self.logger.debug("AMQPConnection configured with url {}", self.url)
self.exchange = Exchange(configuration['AMQP'].get('exchange', "amqp.topic"), type='topic')
self.logger.debug("AMQPConnection is using exchange {}", self.exchange)
if 'username' in configuration['AMQP'] and 'password' in configuration['AMQP']:
username = configuration['AMQP']['username']
password = configuration['AMQP']['password']
self.logger.debug("Using username {} and password {} to connect to AMQP Broker", username, password)
self.connection = Connection(self.url, userid=username, password=password)
else:
self.connection = Connection(self.url)
self.channels = []
self.consumers = []
self.default_channel = self.connection.default_channel
def add_channel(self):
self.logger.debug("Adding channel number {} to the list of channels", len(self.channels) + 1)
self.channels.append(self.connection.channel())
return self.channels[-1]
def add_consumer(self, consumer):
assert(isinstance(consumer, Consumer))
self.logger.debug("Adding consumer number {} to the list of consumers", len(self.consumers) + 1)
self.consumers.append(consumer)
class test_StdChannel:
def setup(self):
self.conn = Connection('memory://')
self.channel = self.conn.channel()
self.channel.queues.clear()
self.conn.connection.state.clear()
def test_Consumer(self):
q = Queue('foo', Exchange('foo'))
print(self.channel.queues)
cons = self.channel.Consumer(q)
assert isinstance(cons, Consumer)
assert cons.channel is self.channel
def test_Producer(self):
prod = self.channel.Producer()
assert isinstance(prod, Producer)
assert prod.channel is self.channel
def test_interface_get_bindings(self):
with pytest.raises(NotImplementedError):
StdChannel().get_bindings()
def test_interface_after_reply_message_received(self):
assert StdChannel().after_reply_message_received(Queue('foo')) is None
class QueuePush(object):
def __init__(self,Queue_Server,Queue_Port,Queue_User,Queue_PassWord,Queue_Path):
self.usr=Queue_User
self.psw=Queue_PassWord
self.server=Queue_Server
self.port=Queue_Port
self.path=Queue_Path
self.connection=None
self.smsExchange=None
def _InitConnect(self):
if self.connection is None or self.connection.connected==False:
self.connection = Connection(hostname=self.server,port=self.port,userid=self.usr,password=self.psw,virtual_host=self.path)
self.channel = self.connection.channel()
self.producer=Producer(self.channel)
self.smsExchange=Exchange("sys.sms",type='topic',channel=self.channel,durable=True,delivery_mode=2)
self.smsCodeProduce=Producer(self.channel,self.smsExchange,routing_key='sms.code')
def Push(self,queueid,connectid,body):
self.rawPush(queueid,{'connid':connectid},body)
def Close(self,queueid,connectid):
self.rawPush(queueid,{'connid':connectid,'close_connect':'1'},'close')
def rawPush(self,routing_key,headers,body):
self._InitConnect()
self.producer.publish(body=body,delivery_mode=2,headers=headers,
routing_key=routing_key,retry=True,compression='gzip')
def sendCode(self,phone,code):
self._InitConnect()
json_str=json.dumps({'phone':str(phone),"content":u"您的莱信验证码为:%s，请在5分钟内输入完成验证。【莱福思】"%str(code)},ensure_ascii=False)
self.smsCodeProduce.publish(body=json_str,retry=True,compression='gzip')
class test_StdChannel(TestCase):
def setUp(self):
self.conn = Connection('memory://')
self.channel = self.conn.channel()
self.channel.queues.clear()
self.conn.connection.state.clear()
def test_Consumer(self):
q = Queue('foo')
print(self.channel.queues)
cons = self.channel.Consumer(q)
self.assertIsInstance(cons, Consumer)
self.assertIs(cons.channel, self.channel)
def test_Producer(self):
prod = self.channel.Producer()
self.assertIsInstance(prod, Producer)
self.assertIs(prod.channel, self.channel)
def test_interface_get_bindings(self):
with self.assertRaises(NotImplementedError):
StdChannel().get_bindings()
def test_interface_after_reply_message_received(self):
self.assertIsNone(
StdChannel().after_reply_message_received(Queue('foo')),
)
class AMQPConnection(object):
conf = None
connection = None
producer = None
needs_reconnect = False
def __init__(self, conf, exchange):
self.conf = conf
self.exchange = exchange
self.reconnect()
def reconnect(self):
self.needs_reconnect = False
LOG.info("Opening new AMQP connection to amqp://%s@%s:%s%s (%s)" % (
self.conf.amqp_userid, self.conf.amqp_hostname, self.conf.amqp_port, self.conf.amqp_virtual_host, self.exchange.name))
if self.connection:
self.connection.release()
try:
self.connection = Connection(
hostname=self.conf.amqp_hostname,
userid=self.conf.amqp_userid,
password=self.conf.amqp_password,
virtual_host=self.conf.amqp_virtual_host,
port=self.conf.amqp_port)
channel = self.connection.channel() # get a new channel
self.producer = Producer(channel, self.exchange, auto_declare=False)
except Exception, e:
LOG.error("Error opening AMQP connection: %s" % e)
self.needs_reconnect = True
def _do_test():
conn = Connection(transport=Transport)
chan = conn.channel()
self.assertTrue(chan.Client)
self.assertTrue(chan.ResponseError)
self.assertTrue(conn.transport.connection_errors)
self.assertTrue(conn.transport.channel_errors)
def test_check_at_least_we_try_to_connect_and_fail(self):
import redis
connection = Connection('redis://localhost:65534/')
with pytest.raises(redis.exceptions.ConnectionError):
chan = connection.channel()
chan._size('some_queue')
class test_ConsumerSet(TestCase):
def setUp(self):
self.connection = Connection(transport=Transport)
@patch('kombu.compat._iterconsume')
def test_iterconsume(self, _iterconsume, n='test_iterconsume'):
c = compat.Consumer(self.connection, queue=n, exchange=n)
cs = compat.ConsumerSet(self.connection, consumers=[c])
cs.iterconsume(limit=10, no_ack=True)
_iterconsume.assert_called_with(c.connection, cs, True, 10)
def test_revive(self, n='test_revive'):
c = compat.Consumer(self.connection, queue=n, exchange=n)
cs = compat.ConsumerSet(self.connection, consumers=[c])
with self.connection.channel() as c2:
cs.revive(c2)
self.assertIs(cs.backend, c2)
def test_constructor(self, prefix='0daf8h21'):
dcon = {'%s.xyx' % prefix: {'exchange': '%s.xyx' % prefix,
'routing_key': 'xyx'},
'%s.xyz' % prefix: {'exchange': '%s.xyz' % prefix,
'routing_key': 'xyz'}}
consumers = [compat.Consumer(self.connection, queue=prefix + str(i),
exchange=prefix + str(i))
for i in range(3)]
c = compat.ConsumerSet(self.connection, consumers=consumers)
c2 = compat.ConsumerSet(self.connection, from_dict=dcon)
self.assertEqual(len(c.queues), 3)
self.assertEqual(len(c2.queues), 2)
c.add_consumer(compat.Consumer(self.connection,
queue=prefix + 'xaxxxa',
exchange=prefix + 'xaxxxa'))
self.assertEqual(len(c.queues), 4)
for cq in c.queues:
self.assertIs(cq.channel, c.channel)
c2.add_consumer_from_dict({
'%s.xxx' % prefix: {
'exchange': '%s.xxx' % prefix,
'routing_key': 'xxx',
},
})
self.assertEqual(len(c2.queues), 3)
for c2q in c2.queues:
self.assertIs(c2q.channel, c2.channel)
c.discard_all()
self.assertEqual(c.channel.called.count('queue_purge'), 4)
c.consume()
c.close()
c2.close()
self.assertIn('basic_cancel', c.channel)
self.assertIn('close', c.channel)
self.assertIn('close', c2.channel)
def test_socket_connection(self):
connection = Connection('redis+socket:///tmp/redis.sock',
transport=Transport)
connparams = connection.channel()._connparams()
self.assertEqual(connparams['connection_class'],
redis.redis.UnixDomainSocketConnection)
self.assertEqual(connparams['path'], '/tmp/redis.sock')
class PollingQueueConsumer(object):
""" Implements a minimum interface of the
:class:`~messaging.QueueConsumer`. Instead of processing messages in a
separate thread it provides a polling method to block until a message with
the same correlation ID of the RPC-proxy call arrives.
"""
def register_provider(self, provider):
self.provider = provider
self.connection = Connection(provider.container.config['AMQP_URI'])
self.channel = self.connection.channel()
self.queue = provider.queue
maybe_declare(self.queue, self.channel)
def unregister_provider(self, provider):
self.connection.close()
def ack_message(self, msg):
msg.ack()
def poll_messages(self, correlation_id):
channel = self.channel
conn = channel.connection
for body, msg in itermessages(conn, channel, self.queue, limit=None):
if correlation_id == msg.properties.get('correlation_id'):
self.provider.handle_message(body, msg)
break
def test_get_client(self):
with mock.module_exists(*_redis_modules()):
conn = Connection(transport=Transport)
chan = conn.channel()
assert chan.Client
assert chan.ResponseError
assert conn.transport.connection_errors
assert conn.transport.channel_errors
def test_disable_ack_emulation(self):
conn = Connection(transport=Transport, transport_options={
'ack_emulation': False,
})
chan = conn.channel()
assert not chan.ack_emulation
assert chan.QoS == virtual.QoS
def test_disable_ack_emulation(self):
conn = Connection(transport=Transport, transport_options={
'ack_emulation': False,
})
chan = conn.channel()
self.assertFalse(chan.ack_emulation)
self.assertEqual(chan.QoS, virtual.QoS)
def worker(mq_url):
connection = Connection(mq_url)
channel = connection.channel()
consumer_json = Consumer(channel, task_json_queue, callbacks=[process_json], accept=["json"])
consumer_json.consume()
consumer_pickle = Consumer(channel, task_pickle_queue, callbacks=[process_pickle], accept=["pickle"])
consumer_pickle.consume()
while True:
connection.drain_events()
def init(host,port,virtual_host,usr,psw,queue_name):
global connection,channel,producer,task_queue,consumer
connection = Connection(hostname=host,port=port,userid=usr,password=psw,virtual_host=virtual_host)
channel = connection.channel()
producer=Producer(channel)
task_queue = Queue(queue_name,durable=True)
consumer = Consumer(channel,task_queue,no_ack=False)
consumer.qos(prefetch_count=1)
consumer.register_callback(RequestCallBack)
class test_ConsumerSet(TestCase):
def setUp(self):
self.connection = Connection(transport=Transport)
@patch("kombu.compat._iterconsume")
def test_iterconsume(self, _iterconsume, n="test_iterconsume"):
c = compat.Consumer(self.connection, queue=n, exchange=n)
cs = compat.ConsumerSet(self.connection, consumers=[c])
cs.iterconsume(limit=10, no_ack=True)
_iterconsume.assert_called_with(c.connection, cs, True, 10)
def test_revive(self, n="test_revive"):
c = compat.Consumer(self.connection, queue=n, exchange=n)
cs = compat.ConsumerSet(self.connection, consumers=[c])
with self.connection.channel() as c2:
cs.revive(c2)
self.assertIs(cs.backend, c2)
def test_constructor(self, prefix="0daf8h21"):
dcon = {"%s.xyx" % prefix: {"exchange": "%s.xyx" % prefix,
"routing_key": "xyx"},
"%s.xyz" % prefix: {"exchange": "%s.xyz" % prefix,
"routing_key": "xyz"}}
consumers = [compat.Consumer(self.connection, queue=prefix + str(i),
exchange=prefix + str(i))
for i in range(3)]
c = compat.ConsumerSet(self.connection, consumers=consumers)
c2 = compat.ConsumerSet(self.connection, from_dict=dcon)
self.assertEqual(len(c.queues), 3)
self.assertEqual(len(c2.queues), 2)
c.add_consumer(compat.Consumer(self.connection,
queue=prefix + "xaxxxa",
exchange=prefix + "xaxxxa"))
self.assertEqual(len(c.queues), 4)
for cq in c.queues:
self.assertIs(cq.channel, c.channel)
c2.add_consumer_from_dict({"%s.xxx" % prefix: {
"exchange": "%s.xxx" % prefix,
"routing_key": "xxx"}})
self.assertEqual(len(c2.queues), 3)
for c2q in c2.queues:
self.assertIs(c2q.channel, c2.channel)
c.discard_all()
self.assertEqual(c.channel.called.count("queue_purge"), 4)
c.consume()
c.close()
c2.close()
self.assertIn("basic_cancel", c.channel)
self.assertIn("close", c.channel)
self.assertIn("close", c2.channel)
def test_simple_queueing(self):
conn = Connection('sqlalchemy+sqlite:///:memory:')
conn.connect()
channel = conn.channel()
self.assertEqual(
channel.queue_cls.__table__.name,
'kombu_queue'
)
self.assertEqual(
channel.message_cls.__table__.name,
'kombu_message'
)
channel._put('celery', 'DATA')
assert channel._get('celery') == 'DATA'
class RabbitMQConsumer(ConsumerMixin):
log = logging.getLogger("%s.RabbitMQConsumer" %(__name__))
def __init__(self, url, exchange_type, routing_key, queue_name, bindings, callbacks=None, namespace="local"):
self.namespace = namespace
exchange = Exchange(type=exchange_type)
self.queues = [ Queue(queue_name, exchange=exchange) ]
self.connection = Connection(url)
channel = self.connection.channel()
bound_queue = self.queues[0].bind(channel)
bound_queue.declare()
for binding in bindings:
bound_queue.bind_to(binding, routing_key)
if callbacks != None:
if isinstance(callbacks, list):
self.userCallbacks = callbacks
else:
self.userCallbacks = [ callbacks ]
else:
self.userCallbacks = None
self.log.info("Queue bindings:\n\n\t%s == > %s\n" %(queue_name, bindings))
def get_consumers(self, Consumer, channel):
return [Consumer(queues=self.queues, auto_declare=False,
callbacks=[self.processEvent])]
def processEvent(self, body, message):
#body['event_type'] = "%s.%s" %(self.namespace, body['event_type'])
for callback in self.userCallbacks:
callback(body)
message.ack()
def start(self):
self.log.info("Starting openstack event consumer...")
self.run()
def stop(self):
self.log.info("Stopping openstack event consumer...")
self.should_stop = True
def init(host, port, virtual_host, usr, psw, queue_name, exchange_name=None, routing_key=None):
global connection, channel, producer, task_queue, consumer
connection = Connection(hostname=host, port=port, userid=usr, password=psw, virtual_host=virtual_host)
channel = connection.channel()
channel.auto_decode = False
producer = Producer(channel)
if exchange_name:
exchange = Exchange(exchange_name, "topic", channel, durable=True, delivery_mode=2)
task_queue = Queue(queue_name, routing_key=routing_key, durable=True, exchange=exchange)
else:
task_queue = Queue(queue_name, durable=True)
consumer = Consumer(channel, task_queue, no_ack=False)
consumer.qos(prefetch_count=1)
consumer.register_callback(RequestCallBack)
def run(self):
try:
connection = Connection(hostname=self.host,port=self.port,userid=self.usr,password=self.psw,virtual_host=self.virtual_host)
channel = connection.channel()
self.producer=Producer(channel)
task_queue = Queue(self.queue_name,durable=True)
consumer = Consumer(channel,task_queue,no_ack=False)
consumer.qos(prefetch_count=1)
consumer.register_callback(self.RequestCallBack)
consumer.consume()
while True:
connection.drain_events()
connection.close()
except BaseException,e:
print e
def test_custom_table_names(self):
conn = Connection('sqlalchemy+sqlite:///:memory:', transport_options={
'queue_tablename': 'my_custom_queue',
'message_tablename': 'my_custom_message'
})
conn.connect()
channel = conn.channel()
self.assertEqual(
channel.queue_cls.__table__.name,
'my_custom_queue'
)
self.assertEqual(
channel.message_cls.__table__.name,
'my_custom_message'
)
channel._put('celery', 'DATA')
assert channel._get('celery') == 'DATA'
class test_Message(TestCase):
def setUp(self):
self.conn = Connection('memory://')
self.channel = self.conn.channel()
self.message = Message(self.channel, delivery_tag=313)
def test_ack_respects_no_ack_consumers(self):
self.channel.no_ack_consumers = set(['abc'])
self.message.delivery_info['consumer_tag'] = 'abc'
ack = self.channel.basic_ack = Mock()
self.message.ack()
self.assertNotEqual(self.message._state, 'ACK')
self.assertFalse(ack.called)
def test_ack_missing_consumer_tag(self):
self.channel.no_ack_consumers = set(['abc'])
self.message.delivery_info = {}
ack = self.channel.basic_ack = Mock()
self.message.ack()
ack.assert_called_with(self.message.delivery_tag)
def test_ack_not_no_ack(self):
self.channel.no_ack_consumers = set()
self.message.delivery_info['consumer_tag'] = 'abc'
ack = self.channel.basic_ack = Mock()
self.message.ack()
ack.assert_called_with(self.message.delivery_tag)
def test_ack_log_error_when_no_error(self):
ack = self.message.ack = Mock()
self.message.ack_log_error(Mock(), KeyError)
ack.assert_called_with()
def test_ack_log_error_when_error(self):
ack = self.message.ack = Mock()
ack.side_effect = KeyError('foo')
logger = Mock()
self.message.ack_log_error(logger, KeyError)
ack.assert_called_with()
self.assertTrue(logger.critical.called)
self.assertIn("Couldn't ack", logger.critical.call_args[0][0])
def run(self):
connection = Connection(hostname=self.host,port=self.port,userid=self.usr,password=self.psw,virtual_host=self.virtual_host)
channel = connection.channel()
self.producer=Producer(channel)
queueargs={}
if self.msg_timeout:
queueargs['x-message-ttl']=self.msg_timeout
task_queue = Queue(self.queue_name,durable=True,queue_arguments=queueargs if queueargs else None)
consumer = Consumer(channel,task_queue,no_ack=False)
consumer.qos(prefetch_count=1)
consumer.register_callback(self.RequestCallBack)
consumer.consume()
while self.task_count:
connection.drain_events()
self.task_count-=1
connection.close()
def check_rabbit_connections(self):
if not self._controllers:
self.fail('There are no online controllers')
pwd = self.get_conf_values().strip()
for host in self._controllers:
try:
conn = Connection(host, userid='nova',
password=pwd,
virtual_host='/', port=5673)
conn.connect()
channel = conn.channel()
self.connections.append((channel, host))
LOG.debug('connections is {0}'.format(self.connections))
except Exception:
LOG.debug(traceback.format_exc())
self.fail("Failed to connect to "
"5673 port on host {0}".format(host))
def mock_queue(mock_asb, mock_asb_management, random_queue) -> MockQueue:
exchange = Exchange('test_servicebus', type='direct')
queue = Queue(random_queue, exchange, random_queue)
conn = Connection(URL_CREDS, transport=azureservicebus.Transport)
channel = conn.channel()
channel._queue_service = mock_asb
channel._queue_mgmt_service = mock_asb_management
queue(channel).declare()
producer = messaging.Producer(channel, exchange, routing_key=random_queue)
return MockQueue(
random_queue,
mock_asb,
mock_asb_management,
conn,
channel,
producer,
queue
)
def setup_rabbitMQ():
rabbit_domain = os.environ.get('RABBITMQ_URL')
rabbit_domain = rabbit_domain if rabbit_domain is not None else 'localhost'
rabbit_url = 'amqp://*****:*****@' + rabbit_domain + ':5672//'
# Kombu Connection
conn = Connection(rabbit_url)
channel = conn.channel()
# Kombu Exchange
# - set delivery_mode to transient to prevent disk writes for faster delivery
exchange = Exchange("video-exchange", type="direct", delivery_mode=1)
# Kombu Producer
producer = Producer(exchange=exchange,
channel=channel,
routing_key="video")
# Kombu Queue
queue = Queue(name="video-queue", exchange=exchange, routing_key="video")
queue.maybe_bind(conn)
queue.declare()
return queue, exchange, producer, conn
class MQ:
"""
This class connect to openstack mq to get notifications.
:keyword mq_user: openstack mq's username.
:keyword mq_password: openstack mq's password.
:keyword mq_host: openstack mq's host ip.
"""
def __init__(self, mq_user, mq_password, mq_host):
self.mq_user = mq_user
self.mq_password = mq_password
self.mq_host = mq_host
self.connection = None
def __repr__(self):
return "user={0},password={1},host={2}".format(
self.mq_user, self.mq_password, self.mq_host)
__str__ = __repr__
def create_connection(self):
host = "amqp://{0}:{1}@{2}//".format(self.mq_user, self.mq_password, self.mq_host)
log.info("create connection: " + host)
self.connection = Connection(host)
return self.connection
def create_consumer(self, exchange_name, queue_name, process):
try:
channel = self.connection.channel()
exchange = Exchange(exchange_name, type="topic")
queue = Queue(queue_name, exchange, routing_key="notifications.#")
consumer = Consumer(channel, queue)
consumer.register_callback(process)
consumer.consume()
log.info("create consumer: " + repr(consumer))
except OSError:
raise MQConnectionError("please check your mq user, password and host configuration.")
return self.connection
def _rabbit_mq(messages):
conn = Connection(host)
channel = conn.channel()
exchange = Exchange(exchange_name, type="direct")
queue = Queue(name=queue_name,
exchange=exchange,
routing_key=routing_key)
queue.maybe_bind(conn)
queue.declare()
producer = Producer(exchange=exchange,
channel=channel,
routing_key=routing_key)
for message in messages:
# as_dict = message.asDict(recursive=True)
producer.publish(message)
channel.close()
conn.release()
return messages
def send(self):
try:
# Connection
conn = Connection(self.broker)
# Channel
channel = conn.channel()
# Exchange
task_exchange = Exchange(self._exchange_name,
type=self._queue_type)
# Queues
if self._queue_name:
queue = Queue(name=self._queue_name, channel=channel,
exchange=task_exchange,
routing_key=self._routing_key)
queue.declare()
# Producer
producer = Producer(exchange=task_exchange, channel=channel,
routing_key=self._routing_key)
# Send message
for message in self._msgs:
serialized_message = json.dumps(message, ensure_ascii=False)
producer.publish(serialized_message)
conn.close()
except Exception, e:
self.log.error(
u'QueueManagerError - Error on sending objects from queue.')
self.log.debug(e)
raise Exception(
'QueueManagerError - Error on sending objects to queue.')
class KombuQueueLengther(object):
KOMBU_HEARTBEAT = 2
def __init__(self, url, queue):
self.connection = Connection(url, heartbeat=self.KOMBU_HEARTBEAT)
self.queue = queue
self._maybe_dirty = False
@backoff.on_exception(backoff.fibo,
ConnectionError,
max_value=9,
max_time=30,
jitter=backoff.full_jitter)
def __len__(self):
if self._maybe_dirty:
time.sleep(self.KOMBU_HEARTBEAT * 1.5)
result = self.connection.channel()._size(self.queue)
# Kombu queue length will not change until next heartbeat.
# Would be better to use a token-bucket timeout,
# but some `time.delay()` will do for now.
self._maybe_dirty = True
return result
def republish(self, backoff_exc, message, target_queue):
expiration = backoff_exc.next(message, self.exchange.name)
queue = self.make_queue(expiration)
properties = message.properties.copy()
headers = properties.pop('application_headers', {})
headers['backoff'] = expiration
expiration_seconds = float(expiration) / 1000
amqp_uri = self.container.config[AMQP_URI_CONFIG_KEY]
# force redeclaration;
# In kombu versions prior to 4.3.0, the publisher will skip declaration if
# the entity has previously been declared by the same connection.
# (see https://github.com/celery/kombu/pull/884)
conn = Connection(amqp_uri)
if KOMBU_PRE_4_3: # pragma: no cover
maybe_declare(
queue, conn.channel(), retry=True, **DEFAULT_RETRY_POLICY
)
# republish to appropriate backoff queue
publisher = Publisher(amqp_uri)
publisher.publish(
message.body,
headers=headers,
exchange=self.exchange,
routing_key=target_queue,
expiration=expiration_seconds,
mandatory=True,
retry=True,
retry_policy=DEFAULT_RETRY_POLICY,
declare=[queue.exchange, queue],
**properties
)
class test_StdChannel:
def setup(self):
self.conn = Connection('memory://')
self.channel = self.conn.channel()
self.channel.queues.clear()
self.conn.connection.state.clear()
def test_Consumer(self):
q = Queue('foo', Exchange('foo'))
cons = self.channel.Consumer(q)
assert isinstance(cons, Consumer)
assert cons.channel is self.channel
def test_Producer(self):
prod = self.channel.Producer()
assert isinstance(prod, Producer)
assert prod.channel is self.channel
def test_interface_get_bindings(self):
with pytest.raises(NotImplementedError):
StdChannel().get_bindings()
def test_interface_after_reply_message_received(self):
assert StdChannel().after_reply_message_received(Queue('foo')) is None
def test_publish__consume(self):
connection = Connection(transport=Transport)
channel = connection.channel()
producer = Producer(channel, self.exchange, routing_key='test_Redis')
consumer = Consumer(channel, queues=[self.queue])
producer.publish({'hello2': 'world2'})
_received = []
def callback(message_data, message):
_received.append(message_data)
message.ack()
consumer.register_callback(callback)
consumer.consume()
self.assertIn(channel, channel.connection.cycle._channels)
try:
connection.drain_events(timeout=1)
self.assertTrue(_received)
with self.assertRaises(socket.timeout):
connection.drain_events(timeout=0.01)
finally:
channel.close()
class test_Channel(Case):
def setUp(self):
self.connection = Connection(transport=Transport)
self.channel = self.connection.channel()
def test_disable_ack_emulation(self):
conn = Connection(transport=Transport,
transport_options={
'ack_emulation': False,
})
chan = conn.channel()
self.assertFalse(chan.ack_emulation)
self.assertEqual(chan.QoS, virtual.QoS)
def test_redis_info_raises(self):
pool = Mock(name='pool')
pool_at_init = [pool]
client = Mock(name='client')
class XChannel(Channel):
def __init__(self, *args, **kwargs):
self._pool = pool_at_init[0]
super(XChannel, self).__init__(*args, **kwargs)
def _get_client(self):
return lambda *_, **__: client
class XTransport(Transport):
Channel = XChannel
conn = Connection(transport=XTransport)
client.info.side_effect = RuntimeError()
with self.assertRaises(RuntimeError):
conn.channel()
pool.disconnect.assert_called_with()
pool.disconnect.reset_mock()
pool_at_init = [None]
with self.assertRaises(RuntimeError):
conn.channel()
self.assertFalse(pool.disconnect.called)
def test_after_fork(self):
self.channel._pool = None
self.channel._after_fork()
self.channel._pool = Mock(name='pool')
self.channel._after_fork()
self.channel._pool.disconnect.assert_called_with()
def test_next_delivery_tag(self):
self.assertNotEqual(
self.channel._next_delivery_tag(),
self.channel._next_delivery_tag(),
)
def test_do_restore_message(self):
client = Mock(name='client')
pl1 = {'body': 'BODY'}
spl1 = dumps(pl1)
lookup = self.channel._lookup = Mock(name='_lookup')
lookup.return_value = ['george', 'elaine']
self.channel._do_restore_message(
pl1,
'ex',
'rkey',
client,
)
client.rpush.assert_has_calls([
call('george', spl1),
call('elaine', spl1),
])
pl2 = {'body': 'BODY2', 'headers': {'x-funny': 1}}
headers_after = dict(pl2['headers'], redelivered=True)
spl2 = dumps(dict(pl2, headers=headers_after))
self.channel._do_restore_message(
pl2,
'ex',
'rkey',
client,
)
client.rpush.assert_has_calls([
call('george', spl2),
call('elaine', spl2),
])
client.rpush.side_effect = KeyError()
with patch('kombu.transport.redis.logger') as logger:
self.channel._do_restore_message(
pl2,
'ex',
'rkey',
client,
)
self.assertTrue(logger.critical.called)
def test_restore(self):
message = Mock(name='message')
with patch('kombu.transport.redis.loads') as loads:
loads.return_value = 'M', 'EX', 'RK'
client = self.channel.client = Mock(name='client')
restore = self.channel._do_restore_message = Mock(
name='_do_restore_message', )
pipe = Mock(name='pipe')
client.pipeline.return_value = pipe
pipe_hget = Mock(name='pipe.hget')
pipe.hget.return_value = pipe_hget
pipe_hget_hdel = Mock(name='pipe.hget.hdel')
pipe_hget.hdel.return_value = pipe_hget_hdel
result = Mock(name='result')
pipe_hget_hdel.execute.return_value = None, None
self.channel._restore(message)
client.pipeline.assert_called_with()
unacked_key = self.channel.unacked_key
self.assertFalse(loads.called)
tag = message.delivery_tag
pipe.hget.assert_called_with(unacked_key, tag)
pipe_hget.hdel.assert_called_with(unacked_key, tag)
pipe_hget_hdel.execute.assert_called_with()
pipe_hget_hdel.execute.return_value = result, None
self.channel._restore(message)
loads.assert_called_with(result)
restore.assert_called_with('M', 'EX', 'RK', client, False)
def test_qos_restore_visible(self):
client = self.channel.client = Mock(name='client')
client.zrevrangebyscore.return_value = [
(1, 10),
(2, 20),
(3, 30),
]
qos = redis.QoS(self.channel)
restore = qos.restore_by_tag = Mock(name='restore_by_tag')
qos._vrestore_count = 1
qos.restore_visible()
self.assertFalse(client.zrevrangebyscore.called)
self.assertEqual(qos._vrestore_count, 2)
qos._vrestore_count = 0
qos.restore_visible()
restore.assert_has_calls([
call(1, client),
call(2, client),
call(3, client),
])
self.assertEqual(qos._vrestore_count, 1)
qos._vrestore_count = 0
restore.reset_mock()
client.zrevrangebyscore.return_value = []
qos.restore_visible()
self.assertFalse(restore.called)
self.assertEqual(qos._vrestore_count, 1)
qos._vrestore_count = 0
client.setnx.side_effect = redis.MutexHeld()
qos.restore_visible()
def test_basic_consume_when_fanout_queue(self):
self.channel.exchange_declare(exchange='txconfan', type='fanout')
self.channel.queue_declare(queue='txconfanq')
self.channel.queue_bind(queue='txconfanq', exchange='txconfan')
self.assertIn('txconfanq', self.channel._fanout_queues)
self.channel.basic_consume('txconfanq', False, None, 1)
self.assertIn('txconfanq', self.channel.active_fanout_queues)
self.assertEqual(self.channel._fanout_to_queue.get('txconfan'),
'txconfanq')
def test_basic_cancel_unknown_delivery_tag(self):
self.assertIsNone(self.channel.basic_cancel('txaseqwewq'))
def test_subscribe_no_queues(self):
self.channel.subclient = Mock()
self.channel.active_fanout_queues.clear()
self.channel._subscribe()
self.assertFalse(self.channel.subclient.subscribe.called)
def test_subscribe(self):
self.channel.subclient = Mock()
self.channel.active_fanout_queues.add('a')
self.channel.active_fanout_queues.add('b')
self.channel._fanout_queues.update(a='a', b='b')
self.channel._subscribe()
self.assertTrue(self.channel.subclient.subscribe.called)
s_args, _ = self.channel.subclient.subscribe.call_args
self.assertItemsEqual(s_args[0], ['a', 'b'])
self.channel.subclient.connection._sock = None
self.channel._subscribe()
self.channel.subclient.connection.connect.assert_called_with()
def test_handle_unsubscribe_message(self):
s = self.channel.subclient
s.subscribed = True
self.channel._handle_message(s, ['unsubscribe', 'a', 0])
self.assertFalse(s.subscribed)
def test_handle_pmessage_message(self):
self.assertDictEqual(
self.channel._handle_message(
self.channel.subclient,
['pmessage', 'pattern', 'channel', 'data'],
),
{
'type': 'pmessage',
'pattern': 'pattern',
'channel': 'channel',
'data': 'data',
},
)
def test_handle_message(self):
self.assertDictEqual(
self.channel._handle_message(
self.channel.subclient,
['type', 'channel', 'data'],
),
{
'type': 'type',
'pattern': None,
'channel': 'channel',
'data': 'data',
},
)
def test_brpop_start_but_no_queues(self):
self.assertIsNone(self.channel._brpop_start())
def test_receive(self):
s = self.channel.subclient = Mock()
self.channel._fanout_to_queue['a'] = 'b'
s.parse_response.return_value = [
'message', 'a', dumps({'hello': 'world'})
]
payload, queue = self.channel._receive()
self.assertDictEqual(payload, {'hello': 'world'})
self.assertEqual(queue, 'b')
def test_receive_raises(self):
self.channel._in_listen = True
s = self.channel.subclient = Mock()
s.parse_response.side_effect = KeyError('foo')
with self.assertRaises(redis.Empty):
self.channel._receive()
self.assertFalse(self.channel._in_listen)
def test_receive_empty(self):
s = self.channel.subclient = Mock()
s.parse_response.return_value = None
with self.assertRaises(redis.Empty):
self.channel._receive()
def test_receive_different_message_Type(self):
s = self.channel.subclient = Mock()
s.parse_response.return_value = ['pmessage', '/foo/', 0, 'data']
with self.assertRaises(redis.Empty):
self.channel._receive()
def test_brpop_read_raises(self):
c = self.channel.client = Mock()
c.parse_response.side_effect = KeyError('foo')
with self.assertRaises(redis.Empty):
self.channel._brpop_read()
c.connection.disconnect.assert_called_with()
def test_brpop_read_gives_None(self):
c = self.channel.client = Mock()
c.parse_response.return_value = None
with self.assertRaises(redis.Empty):
self.channel._brpop_read()
def test_poll_error(self):
c = self.channel.client = Mock()
c.parse_response = Mock()
self.channel._poll_error('BRPOP')
c.parse_response.assert_called_with('BRPOP')
c.parse_response.side_effect = KeyError('foo')
self.assertIsNone(self.channel._poll_error('BRPOP'))
def test_put_fanout(self):
self.channel._in_poll = False
c = self.channel.client = Mock()
body = {'hello': 'world'}
self.channel._put_fanout('exchange', body)
c.publish.assert_called_with('exchange', dumps(body))
def test_put_priority(self):
client = self.channel.client = Mock(name='client')
msg1 = {'properties': {'delivery_info': {'priority': 3}}}
self.channel._put('george', msg1)
client.lpush.assert_called_with(
self.channel._q_for_pri('george', 3),
dumps(msg1),
)
msg2 = {'properties': {'delivery_info': {'priority': 313}}}
self.channel._put('george', msg2)
client.lpush.assert_called_with(
self.channel._q_for_pri('george', 9),
dumps(msg2),
)
msg3 = {'properties': {'delivery_info': {}}}
self.channel._put('george', msg3)
client.lpush.assert_called_with(
self.channel._q_for_pri('george', 0),
dumps(msg3),
)
def test_delete(self):
x = self.channel
self.channel._in_poll = False
delete = x.client.delete = Mock()
srem = x.client.srem = Mock()
x._delete('queue', 'exchange', 'routing_key', None)
delete.assert_has_call('queue')
srem.assert_has_call(x.keyprefix_queue % ('exchange', ),
x.sep.join(['routing_key', '', 'queue']))
def test_has_queue(self):
self.channel._in_poll = False
exists = self.channel.client.exists = Mock()
exists.return_value = True
self.assertTrue(self.channel._has_queue('foo'))
exists.assert_has_call('foo')
exists.return_value = False
self.assertFalse(self.channel._has_queue('foo'))
def test_close_when_closed(self):
self.channel.closed = True
self.channel.close()
def test_close_deletes_autodelete_fanout_queues(self):
self.channel._fanout_queues = ['foo', 'bar']
self.channel.auto_delete_queues = ['foo']
self.channel.queue_delete = Mock(name='queue_delete')
self.channel.close()
self.channel.queue_delete.assert_has_calls([call('foo')])
def test_close_client_close_raises(self):
c = self.channel.client = Mock()
c.connection.disconnect.side_effect = self.channel.ResponseError()
self.channel.close()
c.connection.disconnect.assert_called_with()
def test_invalid_database_raises_ValueError(self):
with self.assertRaises(ValueError):
self.channel.connection.client.virtual_host = 'dwqeq'
self.channel._connparams()
def test_connparams_allows_slash_in_db(self):
self.channel.connection.client.virtual_host = '/123'
self.assertEqual(self.channel._connparams()['db'], 123)
def test_connparams_db_can_be_int(self):
self.channel.connection.client.virtual_host = 124
self.assertEqual(self.channel._connparams()['db'], 124)
def test_new_queue_with_auto_delete(self):
redis.Channel._new_queue(self.channel, 'george', auto_delete=False)
self.assertNotIn('george', self.channel.auto_delete_queues)
redis.Channel._new_queue(self.channel, 'elaine', auto_delete=True)
self.assertIn('elaine', self.channel.auto_delete_queues)
def test_connparams_regular_hostname(self):
self.channel.connection.client.hostname = 'george.vandelay.com'
self.assertEqual(
self.channel._connparams()['host'],
'george.vandelay.com',
)
def test_rotate_cycle_ValueError(self):
cycle = self.channel._queue_cycle = ['kramer', 'jerry']
self.channel._rotate_cycle('kramer')
self.assertEqual(cycle, ['jerry', 'kramer'])
self.channel._rotate_cycle('elaine')
@skip_if_not_module('redis')
def test_get_client(self):
import redis as R
KombuRedis = redis.Channel._get_client(self.channel)
self.assertTrue(KombuRedis)
Rv = getattr(R, 'VERSION', None)
try:
R.VERSION = (2, 4, 0)
with self.assertRaises(VersionMismatch):
redis.Channel._get_client(self.channel)
finally:
if Rv is not None:
R.VERSION = Rv
@skip_if_not_module('redis')
def test_get_response_error(self):
from redis.exceptions import ResponseError
self.assertIs(redis.Channel._get_response_error(self.channel),
ResponseError)
def test_avail_client_when_not_in_poll(self):
self.channel._in_poll = False
c = self.channel.client = Mock()
with self.channel.conn_or_acquire() as client:
self.assertIs(client, c)
def test_avail_client_when_in_poll(self):
self.channel._in_poll = True
self.channel._pool = Mock()
cc = self.channel._create_client = Mock()
client = cc.return_value = Mock()
with self.channel.conn_or_acquire():
pass
self.channel.pool.release.assert_called_with(client.connection)
cc.assert_called_with()
def test_register_with_event_loop(self):
transport = self.connection.transport
transport.cycle = Mock(name='cycle')
transport.cycle.fds = {12: 'LISTEN', 13: 'BRPOP'}
conn = Mock(name='conn')
loop = Mock(name='loop')
redis.Transport.register_with_event_loop(transport, conn, loop)
transport.cycle.on_poll_init.assert_called_with(loop.poller)
loop.call_repeatedly.assert_called_with(
10,
transport.cycle.maybe_restore_messages,
)
self.assertTrue(loop.on_tick.add.called)
on_poll_start = loop.on_tick.add.call_args[0][0]
on_poll_start()
transport.cycle.on_poll_start.assert_called_with()
loop.add_reader.assert_has_calls([
call(12, transport.on_readable, 12),
call(13, transport.on_readable, 13),
])
def test_transport_on_readable(self):
transport = self.connection.transport
cycle = transport.cycle = Mock(name='cyle')
cycle.on_readable.return_value = None
redis.Transport.on_readable(transport, 13)
cycle.on_readable.assert_called_with(13)
cycle.on_readable.reset_mock()
queue = Mock(name='queue')
ret = (Mock(name='message'), queue)
cycle.on_readable.return_value = ret
with self.assertRaises(KeyError):
redis.Transport.on_readable(transport, 14)
cb = transport._callbacks[queue] = Mock(name='callback')
redis.Transport.on_readable(transport, 14)
cb.assert_called_with(ret[0])
@skip_if_not_module('redis')
def test_transport_get_errors(self):
self.assertTrue(redis.Transport._get_errors(self.connection.transport))
@skip_if_not_module('redis')
def test_transport_driver_version(self):
self.assertTrue(
redis.Transport.driver_version(self.connection.transport), )
@skip_if_not_module('redis')
def test_transport_get_errors_when_InvalidData_used(self):
from redis import exceptions
class ID(Exception):
pass
DataError = getattr(exceptions, 'DataError', None)
InvalidData = getattr(exceptions, 'InvalidData', None)
exceptions.InvalidData = ID
exceptions.DataError = None
try:
errors = redis.Transport._get_errors(self.connection.transport)
self.assertTrue(errors)
self.assertIn(ID, errors[1])
finally:
if DataError is not None:
exceptions.DataError = DataError
if InvalidData is not None:
exceptions.InvalidData = InvalidData
def test_empty_queues_key(self):
channel = self.channel
channel._in_poll = False
key = channel.keyprefix_queue % 'celery'
# Everything is fine, there is a list of queues.
channel.client.sadd(key, 'celery\x06\x16\x06\x16celery')
self.assertListEqual(channel.get_table('celery'),
[('celery', '', 'celery')])
# ... then for some reason, the _kombu.binding.celery key gets lost
channel.client.srem(key)
# which raises a channel error so that the consumer/publisher
# can recover by redeclaring the required entities.
with self.assertRaises(InconsistencyError):
self.channel.get_table('celery')
@skip_if_not_module('redis')
def test_socket_connection(self):
with patch('kombu.transport.redis.Channel._create_client'):
with Connection('redis+socket:///tmp/redis.sock') as conn:
connparams = conn.default_channel._connparams()
self.assertEqual(connparams['connection_class'],
redis.redis.UnixDomainSocketConnection)
self.assertEqual(connparams['path'], '/tmp/redis.sock')
from kombu import Connection, Exchange, Queue, Consumer
'''
Sample Kombu consumer to drain messages from AMQP message bus
'''
rabbit_url = "amqp://{rabbit_server_ip}:5672/"
conn = Connection(rabbit_url)
channel = conn.channel()
exchange = Exchange('data_exchange', type="direct")
queue = Queue(name="data_q", exchange=exchange, routing_key="data_info")
def process_message(body, message):
print type(body), body
print type(message), message.body
print("The body is {}".format(body))
message.ack()
with Consumer(conn,
queues=queue,
callbacks=[process_message],
accept=["text/plain", "json"]):
conn.drain_events(timeout=6000)
class test_FilesystemTransport(Case):
def setup(self):
try:
data_folder_in = tempfile.mkdtemp()
data_folder_out = tempfile.mkdtemp()
except Exception:
raise SkipTest('filesystem transport: cannot create tempfiles')
self.c = Connection(transport='filesystem',
transport_options={
'data_folder_in': data_folder_in,
'data_folder_out': data_folder_out,
})
self.p = Connection(transport='filesystem',
transport_options={
'data_folder_in': data_folder_out,
'data_folder_out': data_folder_in,
})
self.e = Exchange('test_transport_filesystem')
self.q = Queue('test_transport_filesystem',
exchange=self.e,
routing_key='test_transport_filesystem')
self.q2 = Queue('test_transport_filesystem2',
exchange=self.e,
routing_key='test_transport_filesystem2')
def test_produce_consume_noack(self):
producer = Producer(self.p.channel(), self.e)
consumer = Consumer(self.c.channel(), self.q, no_ack=True)
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem')
_received = []
def callback(message_data, message):
_received.append(message)
consumer.register_callback(callback)
consumer.consume()
while 1:
if len(_received) == 10:
break
self.c.drain_events()
self.assertEqual(len(_received), 10)
def test_produce_consume(self):
producer_channel = self.p.channel()
consumer_channel = self.c.channel()
producer = Producer(producer_channel, self.e)
consumer1 = Consumer(consumer_channel, self.q)
consumer2 = Consumer(consumer_channel, self.q2)
self.q2(consumer_channel).declare()
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem')
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem2')
_received1 = []
_received2 = []
def callback1(message_data, message):
_received1.append(message)
message.ack()
def callback2(message_data, message):
_received2.append(message)
message.ack()
consumer1.register_callback(callback1)
consumer2.register_callback(callback2)
consumer1.consume()
consumer2.consume()
while 1:
if len(_received1) + len(_received2) == 20:
break
self.c.drain_events()
self.assertEqual(len(_received1) + len(_received2), 20)
# compression
producer.publish({'compressed': True},
routing_key='test_transport_filesystem',
compression='zlib')
m = self.q(consumer_channel).get()
self.assertDictEqual(m.payload, {'compressed': True})
# queue.delete
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem')
self.assertTrue(self.q(consumer_channel).get())
self.q(consumer_channel).delete()
self.q(consumer_channel).declare()
self.assertIsNone(self.q(consumer_channel).get())
# queue.purge
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem2')
self.assertTrue(self.q2(consumer_channel).get())
self.q2(consumer_channel).purge()
self.assertIsNone(self.q2(consumer_channel).get())
class test_Redis:
def setup(self):
self.connection = Connection(transport=Transport)
self.exchange = Exchange('test_Redis', type='direct')
self.queue = Queue('test_Redis', self.exchange, 'test_Redis')
def teardown(self):
self.connection.close()
@mock.replace_module_value(redis.redis, 'VERSION', [3, 0, 0])
def test_publish__get_redispyv3(self):
channel = self.connection.channel()
producer = Producer(channel, self.exchange, routing_key='test_Redis')
self.queue(channel).declare()
producer.publish({'hello': 'world'})
assert self.queue(channel).get().payload == {'hello': 'world'}
assert self.queue(channel).get() is None
assert self.queue(channel).get() is None
assert self.queue(channel).get() is None
@mock.replace_module_value(redis.redis, 'VERSION', [2, 5, 10])
def test_publish__get_redispyv2(self):
channel = self.connection.channel()
producer = Producer(channel, self.exchange, routing_key='test_Redis')
self.queue(channel).declare()
producer.publish({'hello': 'world'})
assert self.queue(channel).get().payload == {'hello': 'world'}
assert self.queue(channel).get() is None
assert self.queue(channel).get() is None
assert self.queue(channel).get() is None
def test_publish__consume(self):
connection = Connection(transport=Transport)
channel = connection.channel()
producer = Producer(channel, self.exchange, routing_key='test_Redis')
consumer = Consumer(channel, queues=[self.queue])
producer.publish({'hello2': 'world2'})
_received = []
def callback(message_data, message):
_received.append(message_data)
message.ack()
consumer.register_callback(callback)
consumer.consume()
assert channel in channel.connection.cycle._channels
try:
connection.drain_events(timeout=1)
assert _received
with pytest.raises(socket.timeout):
connection.drain_events(timeout=0.01)
finally:
channel.close()
def test_purge(self):
channel = self.connection.channel()
producer = Producer(channel, self.exchange, routing_key='test_Redis')
self.queue(channel).declare()
for i in range(10):
producer.publish({'hello': 'world-%s' % (i, )})
assert channel._size('test_Redis') == 10
assert self.queue(channel).purge() == 10
channel.close()
def test_db_values(self):
Connection(virtual_host=1, transport=Transport).channel()
Connection(virtual_host='1', transport=Transport).channel()
Connection(virtual_host='/1', transport=Transport).channel()
with pytest.raises(Exception):
Connection('redis:///foo').channel()
def test_db_port(self):
c1 = Connection(port=None, transport=Transport).channel()
c1.close()
c2 = Connection(port=9999, transport=Transport).channel()
c2.close()
def test_close_poller_not_active(self):
c = Connection(transport=Transport).channel()
cycle = c.connection.cycle
c.client.connection
c.close()
assert c not in cycle._channels
def test_close_ResponseError(self):
c = Connection(transport=Transport).channel()
c.client.bgsave_raises_ResponseError = True
c.close()
def test_close_disconnects(self):
c = Connection(transport=Transport).channel()
conn1 = c.client.connection
conn2 = c.subclient.connection
c.close()
assert conn1.disconnected
assert conn2.disconnected
def test_get__Empty(self):
channel = self.connection.channel()
with pytest.raises(Empty):
channel._get('does-not-exist')
channel.close()
def test_get_client(self):
with mock.module_exists(*_redis_modules()):
conn = Connection(transport=Transport)
chan = conn.channel()
assert chan.Client
assert chan.ResponseError
assert conn.transport.connection_errors
assert conn.transport.channel_errors
def test_check_at_least_we_try_to_connect_and_fail(self):
import redis
connection = Connection('redis://localhost:65534/')
with pytest.raises(redis.exceptions.ConnectionError):
chan = connection.channel()
chan._size('some_queue')
class test_Mailbox(TestCase):
def _handler(self, state):
return self.stats['var']
def setUp(self):
class Mailbox(pidbox.Mailbox):
def _collect(self, *args, **kwargs):
return 'COLLECTED'
self.mailbox = Mailbox('test_pidbox')
self.connection = Connection(transport='memory')
self.state = {'var': 1}
self.handlers = {'mymethod': self._handler}
self.bound = self.mailbox(self.connection)
self.default_chan = self.connection.channel()
self.node = self.bound.Node(
'test_pidbox',
state=self.state,
handlers=self.handlers,
channel=self.default_chan,
)
def test_reply__collect(self):
mailbox = pidbox.Mailbox('test_reply__collect')(self.connection)
exchange = mailbox.reply_exchange.name
channel = self.connection.channel()
mailbox.reply_queue(channel).declare()
ticket = uuid()
mailbox._publish_reply({'foo': 'bar'}, exchange, mailbox.oid, ticket)
_callback_called = [False]
def callback(body):
_callback_called[0] = True
reply = mailbox._collect(ticket,
limit=1,
callback=callback,
channel=channel)
self.assertEqual(reply, [{'foo': 'bar'}])
self.assertTrue(_callback_called[0])
ticket = uuid()
mailbox._publish_reply({'biz': 'boz'}, exchange, mailbox.oid, ticket)
reply = mailbox._collect(ticket, limit=1, channel=channel)
self.assertEqual(reply, [{'biz': 'boz'}])
de = mailbox.connection.drain_events = Mock()
de.side_effect = socket.timeout
mailbox._collect(ticket, limit=1, channel=channel)
def test_constructor(self):
self.assertIsNone(self.mailbox.connection)
self.assertTrue(self.mailbox.exchange.name)
self.assertTrue(self.mailbox.reply_exchange.name)
def test_bound(self):
bound = self.mailbox(self.connection)
self.assertIs(bound.connection, self.connection)
def test_Node(self):
self.assertTrue(self.node.hostname)
self.assertTrue(self.node.state)
self.assertIs(self.node.mailbox, self.bound)
self.assertTrue(self.handlers)
# No initial handlers
node2 = self.bound.Node('test_pidbox2', state=self.state)
self.assertDictEqual(node2.handlers, {})
def test_Node_consumer(self):
consumer1 = self.node.Consumer()
self.assertIs(consumer1.channel, self.default_chan)
self.assertTrue(consumer1.no_ack)
chan2 = self.connection.channel()
consumer2 = self.node.Consumer(channel=chan2, no_ack=False)
self.assertIs(consumer2.channel, chan2)
self.assertFalse(consumer2.no_ack)
def test_handler(self):
node = self.bound.Node('test_handler', state=self.state)
@node.handler
def my_handler_name(state):
return 42
self.assertIn('my_handler_name', node.handlers)
def test_dispatch(self):
node = self.bound.Node('test_dispatch', state=self.state)
@node.handler
def my_handler_name(state, x=None, y=None):
return x + y
self.assertEqual(
node.dispatch('my_handler_name', arguments={
'x': 10,
'y': 10
}), 20)
def test_dispatch_raising_SystemExit(self):
node = self.bound.Node('test_dispatch_raising_SystemExit',
state=self.state)
@node.handler
def my_handler_name(state):
raise SystemExit
with self.assertRaises(SystemExit):
node.dispatch('my_handler_name')
def test_dispatch_raising(self):
node = self.bound.Node('test_dispatch_raising', state=self.state)
@node.handler
def my_handler_name(state):
raise KeyError('foo')
res = node.dispatch('my_handler_name')
self.assertIn('error', res)
self.assertIn('KeyError', res['error'])
def test_dispatch_replies(self):
_replied = [False]
def reply(data, **options):
_replied[0] = True
node = self.bound.Node('test_dispatch', state=self.state)
node.reply = reply
@node.handler
def my_handler_name(state, x=None, y=None):
return x + y
node.dispatch('my_handler_name',
arguments={
'x': 10,
'y': 10
},
reply_to={
'exchange': 'foo',
'routing_key': 'bar'
})
self.assertTrue(_replied[0])
def test_reply(self):
_replied = [(None, None, None)]
def publish_reply(data, exchange, routing_key, ticket, **kwargs):
_replied[0] = (data, exchange, routing_key, ticket)
mailbox = self.mailbox(self.connection)
mailbox._publish_reply = publish_reply
node = mailbox.Node('test_reply')
@node.handler
def my_handler_name(state):
return 42
node.dispatch('my_handler_name',
reply_to={
'exchange': 'exchange',
'routing_key': 'rkey'
},
ticket='TICKET')
data, exchange, routing_key, ticket = _replied[0]
self.assertEqual(data, {'test_reply': 42})
self.assertEqual(exchange, 'exchange')
self.assertEqual(routing_key, 'rkey')
self.assertEqual(ticket, 'TICKET')
def test_handle_message(self):
node = self.bound.Node('test_dispatch_from_message')
@node.handler
def my_handler_name(state, x=None, y=None):
return x * y
body = {'method': 'my_handler_name', 'arguments': {'x': 64, 'y': 64}}
self.assertEqual(node.handle_message(body, None), 64 * 64)
# message not for me should not be processed.
body['destination'] = ['some_other_node']
self.assertIsNone(node.handle_message(body, None))
def test_listen(self):
consumer = self.node.listen()
self.assertEqual(consumer.callbacks[0], self.node.handle_message)
self.assertEqual(consumer.channel, self.default_chan)
def test_cast(self):
self.bound.cast(['somenode'], 'mymethod')
consumer = self.node.Consumer()
self.assertIsCast(self.get_next(consumer))
def test_abcast(self):
self.bound.abcast('mymethod')
consumer = self.node.Consumer()
self.assertIsCast(self.get_next(consumer))
def test_call_destination_must_be_sequence(self):
with self.assertRaises(ValueError):
self.bound.call('some_node', 'mymethod')
def test_call(self):
self.assertEqual(
self.bound.call(['some_node'], 'mymethod'),
'COLLECTED',
)
consumer = self.node.Consumer()
self.assertIsCall(self.get_next(consumer))
def test_multi_call(self):
self.assertEqual(self.bound.multi_call('mymethod'), 'COLLECTED')
consumer = self.node.Consumer()
self.assertIsCall(self.get_next(consumer))
def get_next(self, consumer):
m = consumer.queues[0].get()
if m:
return m.payload
def assertIsCast(self, message):
self.assertTrue(message['method'])
def assertIsCall(self, message):
self.assertTrue(message['method'])
self.assertTrue(message['reply_to'])
class SimpleBase(Case):
abstract = True
def Queue(self, name, *args, **kwargs):
q = name
if not isinstance(q, Queue):
q = self.__class__.__name__
if name:
q = '%s.%s' % (q, name)
return self._Queue(q, *args, **kwargs)
def _Queue(self, *args, **kwargs):
raise NotImplementedError()
def setup(self):
if not self.abstract:
self.connection = Connection(transport='memory')
with self.connection.channel() as channel:
channel.exchange_declare('amq.direct')
self.q = self.Queue(None, no_ack=True)
def teardown(self):
if not self.abstract:
self.q.close()
self.connection.close()
def test_produce__consume(self):
if self.abstract:
return
q = self.Queue('test_produce__consume', no_ack=True)
q.put({'hello': 'Simple'})
self.assertEqual(q.get(timeout=1).payload, {'hello': 'Simple'})
with self.assertRaises(q.Empty):
q.get(timeout=0.1)
def test_produce__basic_get(self):
if self.abstract:
return
q = self.Queue('test_produce__basic_get', no_ack=True)
q.put({'hello': 'SimpleSync'})
self.assertEqual(q.get_nowait().payload, {'hello': 'SimpleSync'})
with self.assertRaises(q.Empty):
q.get_nowait()
q.put({'hello': 'SimpleSync'})
self.assertEqual(q.get(block=False).payload, {'hello': 'SimpleSync'})
with self.assertRaises(q.Empty):
q.get(block=False)
def test_clear(self):
if self.abstract:
return
q = self.Queue('test_clear', no_ack=True)
for i in range(10):
q.put({'hello': 'SimplePurge%d' % (i,)})
self.assertEqual(q.clear(), 10)
def test_enter_exit(self):
if self.abstract:
return
q = self.Queue('test_enter_exit')
q.close = Mock()
self.assertIs(q.__enter__(), q)
q.__exit__()
q.close.assert_called_with()
def test_qsize(self):
if self.abstract:
return
q = self.Queue('test_clear', no_ack=True)
for i in range(10):
q.put({'hello': 'SimplePurge%d' % (i,)})
self.assertEqual(q.qsize(), 10)
self.assertEqual(len(q), 10)
def test_autoclose(self):
if self.abstract:
return
channel = self.connection.channel()
q = self.Queue('test_autoclose', no_ack=True, channel=channel)
q.close()
def test_custom_Queue(self):
if self.abstract:
return
n = self.__class__.__name__
exchange = Exchange('%s-test.custom.Queue' % (n,))
queue = Queue('%s-test.custom.Queue' % (n,),
exchange,
'my.routing.key')
q = self.Queue(queue)
self.assertEqual(q.consumer.queues[0], queue)
q.close()
def test_bool(self):
if self.abstract:
return
q = self.Queue('test_nonzero')
self.assertTrue(q)
class test_Producer(Case):
def setUp(self):
self.exchange = Exchange('foo', 'direct')
self.connection = Connection(transport=Transport)
self.connection.connect()
self.assertTrue(self.connection.connection.connected)
self.assertFalse(self.exchange.is_bound)
def test_repr(self):
p = Producer(self.connection)
self.assertTrue(repr(p))
def test_pickle(self):
chan = Mock()
producer = Producer(chan, serializer='pickle')
p2 = pickle.loads(pickle.dumps(producer))
self.assertEqual(p2.serializer, producer.serializer)
def test_no_channel(self):
p = Producer(None)
self.assertFalse(p._channel)
@patch('kombu.messaging.maybe_declare')
def test_maybe_declare(self, maybe_declare):
p = self.connection.Producer()
q = Queue('foo')
p.maybe_declare(q)
maybe_declare.assert_called_with(q, p.channel, False)
@patch('kombu.common.maybe_declare')
def test_maybe_declare_when_entity_false(self, maybe_declare):
p = self.connection.Producer()
p.maybe_declare(None)
self.assertFalse(maybe_declare.called)
def test_auto_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=True)
self.assertIsNot(p.exchange, self.exchange,
'creates Exchange clone at bind')
self.assertTrue(p.exchange.is_bound)
self.assertIn('exchange_declare', channel,
'auto_declare declares exchange')
def test_manual_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=False)
self.assertTrue(p.exchange.is_bound)
self.assertNotIn('exchange_declare', channel,
'auto_declare=False does not declare exchange')
p.declare()
self.assertIn('exchange_declare', channel,
'p.declare() declares exchange')
def test_prepare(self):
message = {'the quick brown fox': 'jumps over the lazy dog'}
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
m, ctype, cencoding = p._prepare(message, headers={})
self.assertDictEqual(message, anyjson.loads(m))
self.assertEqual(ctype, 'application/json')
self.assertEqual(cencoding, 'utf-8')
def test_prepare_compression(self):
message = {'the quick brown fox': 'jumps over the lazy dog'}
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
headers = {}
m, ctype, cencoding = p._prepare(message,
compression='zlib',
headers=headers)
self.assertEqual(ctype, 'application/json')
self.assertEqual(cencoding, 'utf-8')
self.assertEqual(headers['compression'], 'application/x-gzip')
import zlib
self.assertEqual(
anyjson.loads(zlib.decompress(m).decode('utf-8')),
message,
)
def test_prepare_custom_content_type(self):
message = 'the quick brown fox'.encode('utf-8')
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
m, ctype, cencoding = p._prepare(message, content_type='custom')
self.assertEqual(m, message)
self.assertEqual(ctype, 'custom')
self.assertEqual(cencoding, 'binary')
m, ctype, cencoding = p._prepare(message,
content_type='custom',
content_encoding='alien')
self.assertEqual(m, message)
self.assertEqual(ctype, 'custom')
self.assertEqual(cencoding, 'alien')
def test_prepare_is_already_unicode(self):
message = 'the quick brown fox'
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
m, ctype, cencoding = p._prepare(message, content_type='text/plain')
self.assertEqual(m, message.encode('utf-8'))
self.assertEqual(ctype, 'text/plain')
self.assertEqual(cencoding, 'utf-8')
m, ctype, cencoding = p._prepare(message,
content_type='text/plain',
content_encoding='utf-8')
self.assertEqual(m, message.encode('utf-8'))
self.assertEqual(ctype, 'text/plain')
self.assertEqual(cencoding, 'utf-8')
def test_publish_with_Exchange_instance(self):
p = self.connection.Producer()
p.channel = Mock()
p.publish('hello', exchange=Exchange('foo'), delivery_mode='transient')
self.assertEqual(
p._channel.basic_publish.call_args[1]['exchange'],
'foo',
)
def test_set_on_return(self):
chan = Mock()
chan.events = defaultdict(Mock)
p = Producer(ChannelPromise(lambda: chan), on_return='on_return')
p.channel
chan.events['basic_return'].add.assert_called_with('on_return')
def test_publish_retry_calls_ensure(self):
p = Producer(Mock())
p._connection = Mock()
ensure = p.connection.ensure = Mock()
p.publish('foo', exchange='foo', retry=True)
self.assertTrue(ensure.called)
def test_publish_retry_with_declare(self):
p = self.connection.Producer()
p.maybe_declare = Mock()
p.connection.ensure = Mock()
ex = Exchange('foo')
p._publish('hello', 0, '', '', {}, {}, 'rk', 0, 0, ex, declare=[ex])
p.maybe_declare.assert_called_with(ex)
def test_revive_when_channel_is_connection(self):
p = self.connection.Producer()
p.exchange = Mock()
new_conn = Connection('memory://')
defchan = new_conn.default_channel
p.revive(new_conn)
self.assertIs(p.channel, defchan)
p.exchange.revive.assert_called_with(defchan)
def test_enter_exit(self):
p = self.connection.Producer()
p.release = Mock()
self.assertIs(p.__enter__(), p)
p.__exit__()
p.release.assert_called_with()
def test_connection_property_handles_AttributeError(self):
p = self.connection.Producer()
p.channel = object()
p.__connection__ = None
self.assertIsNone(p.connection)
def test_publish(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
message = {'the quick brown fox': 'jumps over the lazy dog'}
ret = p.publish(message, routing_key='process')
self.assertIn('prepare_message', channel)
self.assertIn('basic_publish', channel)
m, exc, rkey = ret
self.assertDictEqual(message, anyjson.loads(m['body']))
self.assertDictContainsSubset(
{
'content_type': 'application/json',
'content_encoding': 'utf-8',
'priority': 0
}, m)
self.assertDictContainsSubset({'delivery_mode': 2}, m['properties'])
self.assertEqual(exc, p.exchange.name)
self.assertEqual(rkey, 'process')
def test_no_exchange(self):
chan = self.connection.channel()
p = Producer(chan)
self.assertFalse(p.exchange.name)
def test_revive(self):
chan = self.connection.channel()
p = Producer(chan)
chan2 = self.connection.channel()
p.revive(chan2)
self.assertIs(p.channel, chan2)
self.assertIs(p.exchange.channel, chan2)
def test_on_return(self):
chan = self.connection.channel()
def on_return(exception, exchange, routing_key, message):
pass
p = Producer(chan, on_return=on_return)
self.assertTrue(on_return in chan.events['basic_return'])
self.assertTrue(p.on_return)
class RpcClient(object):
"""Process a RPC queue and fetch the response."""
reply_received = False
messages = {}
corr_id_server_queue = {}
def __init__(self,
exchange=default_exchange,
prefix=None,
client_queue=None):
"""Constructor for client object.
:exchange: Exchange to use
:prefix: for automatic server routing key generation
:client_queue: name for the client queue. Default is <command>.client
"""
self._prefix = prefix
self._exchange = exchange
self.reply = None
self._client_queue = client_queue
self._queue = None
self._conn = Connection(**CONN_DICT)
def retrieve_messages(self):
"""Process the message queue and ack the one that matches the
correlation_id sent to the server.
:correlation_id: Expected correlation id (uuid).
:queue: The queue to read.
:returns: JSON object retrieved from the queue.
"""
logger.debug("Client queue: {!r}".format(self._client_queue))
client_queue = self._queue
logger.debug("connection is {!r}"
"is connected: {!r}".format(self._conn,
self._conn.connected))
for i in collect_replies(self._conn,
self._conn.channel(),
client_queue,
timeout=1,
limit=1,
callbacks=[self.ack_message]):
logger.debug("Received {!r}".format(i))
if self.reply is not None:
response = self.reply
self.reply = None
try:
client_queue.purge()
client_queue.delete()
self._conn.release()
except Exception:
logger.warn("Unable to purge and delete queues",
exc_info=True)
return response
def ack_message(self, body, message):
logger.info("Processing message: {!r} with body {!r}".format(
message, body))
if 'correlation_id' in message.properties:
corr_id = message.properties['correlation_id']
try:
self.messages.pop(corr_id)
server_queue = self.corr_id_server_queue.pop(corr_id)
logger.info("STOPREQUEST:%s;CORRELATION_ID:%s" %
(server_queue, corr_id))
self.reply = body
message.ack()
except KeyError as e:
logger.exception(e)
logger.error("Malformed message: {!r}".format(body))
def _setup_payload(self, command_name, data):
"""Setup the datastructure for either hase-like or standard.
:command_name: the name of the command
:data: data to be sent with the command
:returns: payload for the request
"""
return data
def _prepare_client_queue(self, command_name):
"""Setup a client queue based on the command.
"""
if self._client_queue is None:
corr_uuid = uuid()
self._client_queue = '.'.join([command_name, 'client', corr_uuid])
def rpc(self, command_name, data=None, server_routing_key=None):
"""Send a RPC request
:command_name: the command to execute (used as routing key)
:data: dict with data to be sent
:client_queue: Queue for this particular request
:server_routing_key: Server routing key. Will
default to <command>.server
"""
self.reply_received = False
payload = self._setup_payload(command_name, data)
logger.info("Preparing request {!r}".format(payload))
if server_routing_key is None:
if self._prefix is None:
server_routing_key = command_name
else:
server_routing_key = '.'.join([self._prefix, command_name])
self._prepare_client_queue(command_name)
logger.info("Set up client queue {!r} "
"to {!r}".format(self._client_queue, server_routing_key))
message_correlation_id = uuid()
properties = {
'reply_to': self._client_queue,
'correlation_id': message_correlation_id
}
self.corr_id_server_queue[message_correlation_id] = server_routing_key
logger.info('STARTREQUEST:%s;CORRELATION_ID:%s' %
(server_routing_key, message_correlation_id))
result = None
try:
self.messages[message_correlation_id] = True
self._send_command(payload, server_routing_key, properties)
result = self.retrieve_messages()
except Exception:
logger.error("Sending message to consumer queue failed.",
exc_info=True)
raise
# Successful so store message correlation id for retrieval.
return result
def task(self, command_name, data=None, server_routing_key=None):
"""Send a RPC request
:command_name: the command to execute (used as routing key)
:data: dict with data to be sent
:client_queue: Queue for this particular request
:server_routing_key: Server routing key. Will default
to <command>.server
"""
if data is None:
data = {}
self.reply_received = False
payload = self._setup_payload(command_name, data)
logger.debug("Preparing request {!r}".format(payload))
if server_routing_key is None:
server_routing_key = command_name
self._prepare_client_queue(command_name)
logger.info("Set up client queue {!r} "
"to {!r}".format(self._client_queue, server_routing_key))
self._send_command(payload, server_routing_key, declare_queue=False)
def _send_command(self,
payload,
server_routing_key,
properties=None,
declare_queue=True):
if properties is None:
properties = {}
self.reply = None
logger.debug("Using connection: {!r}".format(CONN_DICT))
logger.info("Declaring queue %s." % self._client_queue)
queue = Queue(self._client_queue,
channel=self._conn,
durable=self._exchange.durable,
exchange=self._exchange,
routing_key=self._client_queue)
if declare_queue:
queue.declare()
self._queue = queue
with producers[self._conn].acquire(block=True) as producer:
producer.publish(payload,
serializer='json',
exchange=self._exchange,
declare=[self._exchange],
routing_key=server_routing_key,
**properties)
logger.info("Published {!r} to exchange "
"{!r}".format(payload, self._exchange))
class SensorWatcher(ConsumerMixin):
def __init__(self,
create_handler,
update_handler,
delete_handler,
queue_suffix=None):
"""
:param create_handler: Function which is called on SensorDB create event.
:type create_handler: ``callable``
:param update_handler: Function which is called on SensorDB update event.
:type update_handler: ``callable``
:param delete_handler: Function which is called on SensorDB delete event.
:type delete_handler: ``callable``
"""
# TODO: Handle sensor type filtering using routing key
self._create_handler = create_handler
self._update_handler = update_handler
self._delete_handler = delete_handler
self._sensor_watcher_q = self._get_queue(queue_suffix)
self.connection = None
self._updates_thread = None
self._handlers = {
publishers.CREATE_RK: create_handler,
publishers.UPDATE_RK: update_handler,
publishers.DELETE_RK: delete_handler
}
def get_consumers(self, Consumer, channel):
consumers = [
Consumer(queues=[self._sensor_watcher_q],
accept=['pickle'],
callbacks=[self.process_task])
]
return consumers
def process_task(self, body, message):
LOG.debug('process_task')
LOG.debug(' body: %s', body)
LOG.debug(' message.properties: %s', message.properties)
LOG.debug(' message.delivery_info: %s', message.delivery_info)
routing_key = message.delivery_info.get('routing_key', '')
handler = self._handlers.get(routing_key, None)
try:
if not handler:
LOG.info('Skipping message %s as no handler was found.',
message)
return
try:
handler(body)
except Exception as e:
LOG.exception(
'Handling failed. Message body: %s. Exception: %s', body,
e.message)
finally:
message.ack()
def start(self):
try:
self.connection = Connection(transport_utils.get_messaging_urls())
self._updates_thread = eventlet.spawn(self.run)
except:
LOG.exception('Failed to start sensor_watcher.')
self.connection.release()
def stop(self):
LOG.debug('Shutting down sensor watcher.')
try:
if self._updates_thread:
self._updates_thread = eventlet.kill(self._updates_thread)
if self.connection:
channel = self.connection.channel()
bound_sensor_watch_q = self._sensor_watcher_q(channel)
try:
bound_sensor_watch_q.delete()
except:
LOG.error('Unable to delete sensor watcher queue: %s',
self._sensor_watcher_q)
finally:
if self.connection:
self.connection.release()
@staticmethod
def _get_queue(queue_suffix):
queue_name = queue_utils.get_queue_name(
queue_name_base='st2.sensor.watch',
queue_name_suffix=queue_suffix,
add_random_uuid_to_suffix=True)
return reactor.get_sensor_cud_queue(queue_name, routing_key='#')
class test_Producer:
def setup(self):
self.exchange = Exchange('foo', 'direct')
self.connection = Connection(transport=Transport)
self.connection.connect()
assert self.connection.connection.connected
assert not self.exchange.is_bound
def test_repr(self):
p = Producer(self.connection)
assert repr(p)
def test_pickle(self):
chan = Mock()
producer = Producer(chan, serializer='pickle')
p2 = pickle.loads(pickle.dumps(producer))
assert p2.serializer == producer.serializer
def test_no_channel(self):
p = Producer(None)
assert not p._channel
@patch('kombu.messaging.maybe_declare')
def test_maybe_declare(self, maybe_declare):
p = self.connection.Producer()
q = Queue('foo')
p.maybe_declare(q)
maybe_declare.assert_called_with(q, p.channel, False)
@patch('kombu.common.maybe_declare')
def test_maybe_declare_when_entity_false(self, maybe_declare):
p = self.connection.Producer()
p.maybe_declare(None)
maybe_declare.assert_not_called()
def test_auto_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=True)
# creates Exchange clone at bind
assert p.exchange is not self.exchange
assert p.exchange.is_bound
# auto_declare declares exchange'
assert 'exchange_declare' not in channel
p.publish('foo')
assert 'exchange_declare' in channel
def test_manual_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=False)
assert p.exchange.is_bound
# auto_declare=False does not declare exchange
assert 'exchange_declare' not in channel
# p.declare() declares exchange')
p.declare()
assert 'exchange_declare' in channel
def test_prepare(self):
message = {'the quick brown fox': 'jumps over the lazy dog'}
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
m, ctype, cencoding = p._prepare(message, headers={})
assert json.loads(m) == message
assert ctype == 'application/json'
assert cencoding == 'utf-8'
def test_prepare_compression(self):
message = {'the quick brown fox': 'jumps over the lazy dog'}
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
headers = {}
m, ctype, cencoding = p._prepare(message,
compression='zlib',
headers=headers)
assert ctype == 'application/json'
assert cencoding == 'utf-8'
assert headers['compression'] == 'application/x-gzip'
import zlib
assert json.loads(zlib.decompress(m).decode('utf-8')) == message
def test_prepare_custom_content_type(self):
message = 'the quick brown fox'.encode('utf-8')
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
m, ctype, cencoding = p._prepare(message, content_type='custom')
assert m == message
assert ctype == 'custom'
assert cencoding == 'binary'
m, ctype, cencoding = p._prepare(message,
content_type='custom',
content_encoding='alien')
assert m == message
assert ctype == 'custom'
assert cencoding == 'alien'
def test_prepare_is_already_unicode(self):
message = 'the quick brown fox'
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
m, ctype, cencoding = p._prepare(message, content_type='text/plain')
assert m == message.encode('utf-8')
assert ctype == 'text/plain'
assert cencoding == 'utf-8'
m, ctype, cencoding = p._prepare(message,
content_type='text/plain',
content_encoding='utf-8')
assert m == message.encode('utf-8')
assert ctype == 'text/plain'
assert cencoding == 'utf-8'
def test_publish_with_Exchange_instance(self):
p = self.connection.Producer()
p.channel = Mock()
p.channel.connection.client.declared_entities = set()
p.publish('hello', exchange=Exchange('foo'), delivery_mode='transient')
assert p._channel.basic_publish.call_args[1]['exchange'] == 'foo'
def test_publish_with_expiration(self):
p = self.connection.Producer()
p.channel = Mock()
p.channel.connection.client.declared_entities = set()
p.publish('hello', exchange=Exchange('foo'), expiration=10)
properties = p._channel.prepare_message.call_args[0][5]
assert properties['expiration'] == '10000'
def test_publish_with_reply_to(self):
p = self.connection.Producer()
p.channel = Mock()
p.channel.connection.client.declared_entities = set()
assert not p.exchange.name
p.publish('hello', exchange=Exchange('foo'), reply_to=Queue('foo'))
properties = p._channel.prepare_message.call_args[0][5]
assert properties['reply_to'] == 'foo'
def test_set_on_return(self):
chan = Mock()
chan.events = defaultdict(Mock)
p = Producer(ChannelPromise(lambda: chan), on_return='on_return')
p.channel
chan.events['basic_return'].add.assert_called_with('on_return')
def test_publish_retry_calls_ensure(self):
p = Producer(Mock())
p._connection = Mock()
p._connection.declared_entities = set()
ensure = p.connection.ensure = Mock()
p.publish('foo', exchange='foo', retry=True)
ensure.assert_called()
def test_publish_retry_with_declare(self):
p = self.connection.Producer()
p.maybe_declare = Mock()
p.connection.ensure = Mock()
ex = Exchange('foo')
p._publish('hello', 0, '', '', {}, {}, 'rk', 0, 0, ex, declare=[ex])
p.maybe_declare.assert_called_with(ex)
def test_revive_when_channel_is_connection(self):
p = self.connection.Producer()
p.exchange = Mock()
new_conn = Connection('memory://')
defchan = new_conn.default_channel
p.revive(new_conn)
assert p.channel is defchan
p.exchange.revive.assert_called_with(defchan)
def test_enter_exit(self):
p = self.connection.Producer()
p.release = Mock()
assert p.__enter__() is p
p.__exit__()
p.release.assert_called_with()
def test_connection_property_handles_AttributeError(self):
p = self.connection.Producer()
p.channel = object()
p.__connection__ = None
assert p.connection is None
def test_publish(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer='json')
message = {'the quick brown fox': 'jumps over the lazy dog'}
ret = p.publish(message, routing_key='process')
assert 'prepare_message' in channel
assert 'basic_publish' in channel
m, exc, rkey = ret
assert json.loads(m['body']) == message
assert m['content_type'] == 'application/json'
assert m['content_encoding'] == 'utf-8'
assert m['priority'] == 0
assert m['properties']['delivery_mode'] == 2
assert exc == p.exchange.name
assert rkey == 'process'
def test_no_exchange(self):
chan = self.connection.channel()
p = Producer(chan)
assert not p.exchange.name
def test_revive(self):
chan = self.connection.channel()
p = Producer(chan)
chan2 = self.connection.channel()
p.revive(chan2)
assert p.channel is chan2
assert p.exchange.channel is chan2
def test_on_return(self):
chan = self.connection.channel()
def on_return(exception, exchange, routing_key, message):
pass
p = Producer(chan, on_return=on_return)
assert on_return in chan.events['basic_return']
assert p.on_return
class test_Consumer:
def setup(self):
self.connection = Connection(transport=Transport)
self.connection.connect()
assert self.connection.connection.connected
self.exchange = Exchange('foo', 'direct')
def test_accept(self):
a = Consumer(self.connection)
assert a.accept is None
b = Consumer(self.connection, accept=['json', 'pickle'])
assert b.accept == {
'application/json',
'application/x-python-serialize',
}
c = Consumer(self.connection, accept=b.accept)
assert b.accept == c.accept
def test_enter_exit_cancel_raises(self):
c = Consumer(self.connection)
c.cancel = Mock(name='Consumer.cancel')
c.cancel.side_effect = KeyError('foo')
with c:
pass
c.cancel.assert_called_with()
def test_receive_callback_accept(self):
message = Mock(name='Message')
message.errors = []
callback = Mock(name='on_message')
c = Consumer(self.connection, accept=['json'], on_message=callback)
c.on_decode_error = None
c.channel = Mock(name='channel')
c.channel.message_to_python = None
c._receive_callback(message)
callback.assert_called_with(message)
assert message.accept == c.accept
def test_accept__content_disallowed(self):
conn = Connection('memory://')
q = Queue('foo', exchange=self.exchange)
p = conn.Producer()
p.publish(
{'complex': object()},
declare=[q],
exchange=self.exchange,
serializer='pickle',
)
callback = Mock(name='callback')
with conn.Consumer(queues=[q], callbacks=[callback]) as consumer:
with pytest.raises(consumer.ContentDisallowed):
conn.drain_events(timeout=1)
callback.assert_not_called()
def test_accept__content_allowed(self):
conn = Connection('memory://')
q = Queue('foo', exchange=self.exchange)
p = conn.Producer()
p.publish(
{'complex': object()},
declare=[q],
exchange=self.exchange,
serializer='pickle',
)
callback = Mock(name='callback')
with conn.Consumer(queues=[q], accept=['pickle'],
callbacks=[callback]):
conn.drain_events(timeout=1)
callback.assert_called()
body, message = callback.call_args[0]
assert body['complex']
def test_set_no_channel(self):
c = Consumer(None)
assert c.channel is None
c.revive(Mock())
assert c.channel
def test_set_no_ack(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True, no_ack=True)
assert consumer.no_ack
def test_add_queue_when_auto_declare(self):
consumer = self.connection.Consumer(auto_declare=True)
q = Mock()
q.return_value = q
consumer.add_queue(q)
assert q in consumer.queues
q.declare.assert_called_with()
def test_add_queue_when_not_auto_declare(self):
consumer = self.connection.Consumer(auto_declare=False)
q = Mock()
q.return_value = q
consumer.add_queue(q)
assert q in consumer.queues
assert not q.declare.call_count
def test_consume_without_queues_returns(self):
consumer = self.connection.Consumer()
consumer.queues[:] = []
assert consumer.consume() is None
def test_consuming_from(self):
consumer = self.connection.Consumer()
consumer.queues[:] = [Queue('a'), Queue('b'), Queue('d')]
consumer._active_tags = {'a': 1, 'b': 2}
assert not consumer.consuming_from(Queue('c'))
assert not consumer.consuming_from('c')
assert not consumer.consuming_from(Queue('d'))
assert not consumer.consuming_from('d')
assert consumer.consuming_from(Queue('a'))
assert consumer.consuming_from(Queue('b'))
assert consumer.consuming_from('b')
def test_receive_callback_without_m2p(self):
channel = self.connection.channel()
c = channel.Consumer()
m2p = getattr(channel, 'message_to_python')
channel.message_to_python = None
try:
message = Mock()
message.errors = []
message.decode.return_value = 'Hello'
recv = c.receive = Mock()
c._receive_callback(message)
recv.assert_called_with('Hello', message)
finally:
channel.message_to_python = m2p
def test_receive_callback__message_errors(self):
channel = self.connection.channel()
channel.message_to_python = None
c = channel.Consumer()
message = Mock()
try:
raise KeyError('foo')
except KeyError:
message.errors = [sys.exc_info()]
message._reraise_error.side_effect = KeyError()
with pytest.raises(KeyError):
c._receive_callback(message)
def test_set_callbacks(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
callbacks = [lambda x, y: x, lambda x, y: x]
consumer = Consumer(channel,
queue,
auto_declare=True,
callbacks=callbacks)
assert consumer.callbacks == callbacks
def test_auto_declare(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.consume() # twice is a noop
assert consumer.queues[0] is not queue
assert consumer.queues[0].is_bound
assert consumer.queues[0].exchange.is_bound
assert consumer.queues[0].exchange is not self.exchange
for meth in ('exchange_declare', 'queue_declare', 'queue_bind',
'basic_consume'):
assert meth in channel
assert channel.called.count('basic_consume') == 1
assert consumer._active_tags
consumer.cancel_by_queue(queue.name)
consumer.cancel_by_queue(queue.name)
assert not consumer._active_tags
def test_consumer_tag_prefix(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, tag_prefix='consumer_')
consumer.consume()
assert consumer._active_tags[queue.name].startswith('consumer_')
def test_manual_declare(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=False)
assert consumer.queues[0] is not queue
assert consumer.queues[0].is_bound
assert consumer.queues[0].exchange.is_bound
assert consumer.queues[0].exchange is not self.exchange
for meth in ('exchange_declare', 'queue_declare', 'basic_consume'):
assert meth not in channel
consumer.declare()
for meth in ('exchange_declare', 'queue_declare', 'queue_bind'):
assert meth in channel
assert 'basic_consume' not in channel
consumer.consume()
assert 'basic_consume' in channel
def test_consume__cancel(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.cancel()
assert 'basic_cancel' in channel
assert not consumer._active_tags
def test___enter____exit__(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
context = consumer.__enter__()
assert context is consumer
assert consumer._active_tags
res = consumer.__exit__(None, None, None)
assert not res
assert 'basic_cancel' in channel
assert not consumer._active_tags
def test_flow(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.flow(False)
assert 'flow' in channel
def test_qos(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.qos(30, 10, False)
assert 'basic_qos' in channel
def test_purge(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
b2 = Queue('qname2', self.exchange, 'rkey')
b3 = Queue('qname3', self.exchange, 'rkey')
b4 = Queue('qname4', self.exchange, 'rkey')
consumer = Consumer(channel, [b1, b2, b3, b4], auto_declare=True)
consumer.purge()
assert channel.called.count('queue_purge') == 4
def test_multiple_queues(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
b2 = Queue('qname2', self.exchange, 'rkey')
b3 = Queue('qname3', self.exchange, 'rkey')
b4 = Queue('qname4', self.exchange, 'rkey')
consumer = Consumer(channel, [b1, b2, b3, b4])
consumer.consume()
assert channel.called.count('exchange_declare') == 4
assert channel.called.count('queue_declare') == 4
assert channel.called.count('queue_bind') == 4
assert channel.called.count('basic_consume') == 4
assert len(consumer._active_tags) == 4
consumer.cancel()
assert channel.called.count('basic_cancel') == 4
assert not len(consumer._active_tags)
def test_receive_callback(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
received = []
def callback(message_data, message):
received.append(message_data)
message.ack()
message.payload # trigger cache
consumer.register_callback(callback)
consumer._receive_callback({'foo': 'bar'})
assert 'basic_ack' in channel
assert 'message_to_python' in channel
assert received[0] == {'foo': 'bar'}
def test_basic_ack_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.ack()
message.ack()
consumer.register_callback(callback)
with pytest.raises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
def test_basic_reject(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
consumer.register_callback(callback)
consumer._receive_callback({'foo': 'bar'})
assert 'basic_reject' in channel
def test_basic_reject_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
message.reject()
consumer.register_callback(callback)
with pytest.raises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
assert 'basic_reject' in channel
def test_basic_reject__requeue(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
consumer.register_callback(callback)
consumer._receive_callback({'foo': 'bar'})
assert 'basic_reject:requeue' in channel
def test_basic_reject__requeue_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
message.requeue()
consumer.register_callback(callback)
with pytest.raises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
assert 'basic_reject:requeue' in channel
def test_receive_without_callbacks_raises(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
with pytest.raises(NotImplementedError):
consumer.receive(1, 2)
def test_decode_error(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
consumer.channel.throw_decode_error = True
with pytest.raises(ValueError):
consumer._receive_callback({'foo': 'bar'})
def test_on_decode_error_callback(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
thrown = []
def on_decode_error(msg, exc):
thrown.append((msg.body, exc))
consumer = Consumer(channel, [b1], on_decode_error=on_decode_error)
consumer.channel.throw_decode_error = True
consumer._receive_callback({'foo': 'bar'})
assert thrown
m, exc = thrown[0]
assert json.loads(m) == {'foo': 'bar'}
assert isinstance(exc, ValueError)
def test_recover(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
consumer.recover()
assert 'basic_recover' in channel
def test_revive(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
channel2 = self.connection.channel()
consumer.revive(channel2)
assert consumer.channel is channel2
assert consumer.queues[0].channel is channel2
assert consumer.queues[0].exchange.channel is channel2
def test_revive__with_prefetch_count(self):
channel = Mock(name='channel')
b1 = Queue('qname1', self.exchange, 'rkey')
Consumer(channel, [b1], prefetch_count=14)
channel.basic_qos.assert_called_with(0, 14, False)
def test__repr__(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
assert repr(Consumer(channel, [b1]))
def test_connection_property_handles_AttributeError(self):
p = self.connection.Consumer()
p.channel = object()
assert p.connection is None
class test_Consumer(Case):
def setup(self):
self.connection = Connection(transport=Transport)
@patch('kombu.compat._iterconsume')
def test_iterconsume_calls__iterconsume(self, it, n='test_iterconsume'):
c = compat.Consumer(self.connection, queue=n, exchange=n)
c.iterconsume(limit=10, no_ack=True)
it.assert_called_with(c.connection, c, True, 10)
def test_constructor(self, n='test_Consumer_constructor'):
c = compat.Consumer(self.connection,
queue=n,
exchange=n,
routing_key='rkey')
self.assertIsInstance(c.backend, Channel)
q = c.queues[0]
self.assertTrue(q.durable)
self.assertTrue(q.exchange.durable)
self.assertFalse(q.auto_delete)
self.assertFalse(q.exchange.auto_delete)
self.assertEqual(q.name, n)
self.assertEqual(q.exchange.name, n)
c2 = compat.Consumer(self.connection,
queue=n + '2',
exchange=n + '2',
routing_key='rkey',
durable=False,
auto_delete=True,
exclusive=True)
q2 = c2.queues[0]
self.assertFalse(q2.durable)
self.assertFalse(q2.exchange.durable)
self.assertTrue(q2.auto_delete)
self.assertTrue(q2.exchange.auto_delete)
def test__enter__exit__(self, n='test__enter__exit__'):
c = compat.Consumer(self.connection,
queue=n,
exchange=n,
routing_key='rkey')
x = c.__enter__()
self.assertIs(x, c)
x.__exit__()
self.assertTrue(c._closed)
def test_revive(self, n='test_revive'):
c = compat.Consumer(self.connection, queue=n, exchange=n)
with self.connection.channel() as c2:
c.revive(c2)
self.assertIs(c.backend, c2)
def test__iter__(self, n='test__iter__'):
c = compat.Consumer(self.connection, queue=n, exchange=n)
c.iterqueue = Mock()
c.__iter__()
c.iterqueue.assert_called_with(infinite=True)
def test_iter(self, n='test_iterqueue'):
c = compat.Consumer(self.connection,
queue=n,
exchange=n,
routing_key='rkey')
c.close()
def test_process_next(self, n='test_process_next'):
c = compat.Consumer(self.connection,
queue=n,
exchange=n,
routing_key='rkey')
with self.assertRaises(NotImplementedError):
c.process_next()
c.close()
def test_iterconsume(self, n='test_iterconsume'):
c = compat.Consumer(self.connection,
queue=n,
exchange=n,
routing_key='rkey')
c.close()
def test_discard_all(self, n='test_discard_all'):
c = compat.Consumer(self.connection,
queue=n,
exchange=n,
routing_key='rkey')
c.discard_all()
self.assertIn('queue_purge', c.backend)
def test_fetch(self, n='test_fetch'):
c = compat.Consumer(self.connection,
queue=n,
exchange=n,
routing_key='rkey')
self.assertIsNone(c.fetch())
self.assertIsNone(c.fetch(no_ack=True))
self.assertIn('basic_get', c.backend)
callback_called = [False]
def receive(payload, message):
callback_called[0] = True
c.backend.to_deliver.append('42')
payload = c.fetch().payload
self.assertEqual(payload, '42')
c.backend.to_deliver.append('46')
c.register_callback(receive)
self.assertEqual(c.fetch(enable_callbacks=True).payload, '46')
self.assertTrue(callback_called[0])
def test_discard_all_filterfunc_not_supported(self, n='xjf21j21'):
c = compat.Consumer(self.connection,
queue=n,
exchange=n,
routing_key='rkey')
with self.assertRaises(NotImplementedError):
c.discard_all(filterfunc=lambda x: x)
c.close()
def test_wait(self, n='test_wait'):
class C(compat.Consumer):
def iterconsume(self, limit=None):
for i in range(limit):
yield i
c = C(self.connection, queue=n, exchange=n, routing_key='rkey')
self.assertEqual(c.wait(10), list(range(10)))
c.close()
def test_iterqueue(self, n='test_iterqueue'):
i = [0]
class C(compat.Consumer):
def fetch(self, limit=None):
z = i[0]
i[0] += 1
return z
c = C(self.connection, queue=n, exchange=n, routing_key='rkey')
self.assertEqual(list(c.iterqueue(limit=10)), list(range(10)))
c.close()
class test_FilesystemTransport:
def setup(self):
self.channels = set()
try:
data_folder_in = tempfile.mkdtemp()
data_folder_out = tempfile.mkdtemp()
except Exception:
raise SkipTest('filesystem transport: cannot create tempfiles')
self.c = Connection(transport='filesystem',
transport_options={
'data_folder_in': data_folder_in,
'data_folder_out': data_folder_out,
})
self.channels.add(self.c.default_channel)
self.p = Connection(transport='filesystem',
transport_options={
'data_folder_in': data_folder_out,
'data_folder_out': data_folder_in,
})
self.channels.add(self.p.default_channel)
self.e = Exchange('test_transport_filesystem')
self.q = Queue('test_transport_filesystem',
exchange=self.e,
routing_key='test_transport_filesystem')
self.q2 = Queue('test_transport_filesystem2',
exchange=self.e,
routing_key='test_transport_filesystem2')
def teardown(self):
# make sure we don't attempt to restore messages at shutdown.
for channel in self.channels:
try:
channel._qos._dirty.clear()
except AttributeError:
pass
try:
channel._qos._delivered.clear()
except AttributeError:
pass
def _add_channel(self, channel):
self.channels.add(channel)
return channel
def test_produce_consume_noack(self):
producer = Producer(self._add_channel(self.p.channel()), self.e)
consumer = Consumer(self._add_channel(self.c.channel()),
self.q,
no_ack=True)
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem')
_received = []
def callback(message_data, message):
_received.append(message)
consumer.register_callback(callback)
consumer.consume()
while 1:
if len(_received) == 10:
break
self.c.drain_events()
assert len(_received) == 10
def test_produce_consume(self):
producer_channel = self._add_channel(self.p.channel())
consumer_channel = self._add_channel(self.c.channel())
producer = Producer(producer_channel, self.e)
consumer1 = Consumer(consumer_channel, self.q)
consumer2 = Consumer(consumer_channel, self.q2)
self.q2(consumer_channel).declare()
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem')
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem2')
_received1 = []
_received2 = []
def callback1(message_data, message):
_received1.append(message)
message.ack()
def callback2(message_data, message):
_received2.append(message)
message.ack()
consumer1.register_callback(callback1)
consumer2.register_callback(callback2)
consumer1.consume()
consumer2.consume()
while 1:
if len(_received1) + len(_received2) == 20:
break
self.c.drain_events()
assert len(_received1) + len(_received2) == 20
# compression
producer.publish({'compressed': True},
routing_key='test_transport_filesystem',
compression='zlib')
m = self.q(consumer_channel).get()
assert m.payload == {'compressed': True}
# queue.delete
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem')
assert self.q(consumer_channel).get()
self.q(consumer_channel).delete()
self.q(consumer_channel).declare()
assert self.q(consumer_channel).get() is None
# queue.purge
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem2')
assert self.q2(consumer_channel).get()
self.q2(consumer_channel).purge()
assert self.q2(consumer_channel).get() is None
class test_Message(Case):
def setUp(self):
self.conn = Connection('memory://')
self.channel = self.conn.channel()
self.message = Message(self.channel, delivery_tag=313)
def test_postencode(self):
with self.assertRaises(LookupError):
Message(self.channel, text_t('FOO'), postencode='ccyzz')
def test_ack_respects_no_ack_consumers(self):
self.channel.no_ack_consumers = set(['abc'])
self.message.delivery_info['consumer_tag'] = 'abc'
ack = self.channel.basic_ack = Mock()
self.message.ack()
self.assertNotEqual(self.message._state, 'ACK')
self.assertFalse(ack.called)
def test_ack_missing_consumer_tag(self):
self.channel.no_ack_consumers = set(['abc'])
self.message.delivery_info = {}
ack = self.channel.basic_ack = Mock()
self.message.ack()
ack.assert_called_with(self.message.delivery_tag)
def test_ack_not_no_ack(self):
self.channel.no_ack_consumers = set()
self.message.delivery_info['consumer_tag'] = 'abc'
ack = self.channel.basic_ack = Mock()
self.message.ack()
ack.assert_called_with(self.message.delivery_tag)
def test_ack_log_error_when_no_error(self):
ack = self.message.ack = Mock()
self.message.ack_log_error(Mock(), KeyError)
ack.assert_called_with()
def test_ack_log_error_when_error(self):
ack = self.message.ack = Mock()
ack.side_effect = KeyError('foo')
logger = Mock()
self.message.ack_log_error(logger, KeyError)
ack.assert_called_with()
self.assertTrue(logger.critical.called)
self.assertIn("Couldn't ack", logger.critical.call_args[0][0])
def test_reject_log_error_when_no_error(self):
reject = self.message.reject = Mock()
self.message.reject_log_error(Mock(), KeyError)
reject.assert_called_with()
def test_reject_log_error_when_error(self):
reject = self.message.reject = Mock()
reject.side_effect = KeyError('foo')
logger = Mock()
self.message.reject_log_error(logger, KeyError)
reject.assert_called_with()
self.assertTrue(logger.critical.called)
self.assertIn("Couldn't ack", logger.critical.call_args[0][0])
class test_Redis(Case):
def setUp(self):
self.connection = Connection(transport=Transport)
self.exchange = Exchange('test_Redis', type='direct')
self.queue = Queue('test_Redis', self.exchange, 'test_Redis')
def tearDown(self):
self.connection.close()
def test_publish__get(self):
channel = self.connection.channel()
producer = Producer(channel, self.exchange, routing_key='test_Redis')
self.queue(channel).declare()
producer.publish({'hello': 'world'})
self.assertDictEqual(
self.queue(channel).get().payload, {'hello': 'world'})
self.assertIsNone(self.queue(channel).get())
self.assertIsNone(self.queue(channel).get())
self.assertIsNone(self.queue(channel).get())
def test_publish__consume(self):
connection = Connection(transport=Transport)
channel = connection.channel()
producer = Producer(channel, self.exchange, routing_key='test_Redis')
consumer = Consumer(channel, queues=[self.queue])
producer.publish({'hello2': 'world2'})
_received = []
def callback(message_data, message):
_received.append(message_data)
message.ack()
consumer.register_callback(callback)
consumer.consume()
self.assertIn(channel, channel.connection.cycle._channels)
try:
connection.drain_events(timeout=1)
self.assertTrue(_received)
with self.assertRaises(socket.timeout):
connection.drain_events(timeout=0.01)
finally:
channel.close()
def test_purge(self):
channel = self.connection.channel()
producer = Producer(channel, self.exchange, routing_key='test_Redis')
self.queue(channel).declare()
for i in range(10):
producer.publish({'hello': 'world-%s' % (i, )})
self.assertEqual(channel._size('test_Redis'), 10)
self.assertEqual(self.queue(channel).purge(), 10)
channel.close()
def test_db_values(self):
Connection(virtual_host=1, transport=Transport).channel()
Connection(virtual_host='1', transport=Transport).channel()
Connection(virtual_host='/1', transport=Transport).channel()
with self.assertRaises(Exception):
Connection('redis:///foo').channel()
def test_db_port(self):
c1 = Connection(port=None, transport=Transport).channel()
c1.close()
c2 = Connection(port=9999, transport=Transport).channel()
c2.close()
def test_close_poller_not_active(self):
c = Connection(transport=Transport).channel()
cycle = c.connection.cycle
c.client.connection
c.close()
self.assertNotIn(c, cycle._channels)
def test_close_ResponseError(self):
c = Connection(transport=Transport).channel()
c.client.bgsave_raises_ResponseError = True
c.close()
def test_close_disconnects(self):
c = Connection(transport=Transport).channel()
conn1 = c.client.connection
conn2 = c.subclient.connection
c.close()
self.assertTrue(conn1.disconnected)
self.assertTrue(conn2.disconnected)
def test_get__Empty(self):
channel = self.connection.channel()
with self.assertRaises(Empty):
channel._get('does-not-exist')
channel.close()
def test_get_client(self):
myredis, exceptions = _redis_modules()
@module_exists(myredis, exceptions)
def _do_test():
conn = Connection(transport=Transport)
chan = conn.channel()
self.assertTrue(chan.Client)
self.assertTrue(chan.ResponseError)
self.assertTrue(conn.transport.connection_errors)
self.assertTrue(conn.transport.channel_errors)
_do_test()
class Amqp(Thread):
def __init__(self,
host="localhost",
port=5672,
userid="guest",
password="******",
virtual_host="canopsis",
exchange_name="canopsis",
logging_name="Amqp",
logging_level=INFO,
read_config_file=True,
auto_connect=True,
on_ready=None,
max_retries=5):
super(Amqp, self).__init__()
self.logger = getLogger(logging_name)
self.host = host
self.port = port
self.userid = userid
self.password = password
self.virtual_host = virtual_host
self.exchange_name = exchange_name
self.logging_level = logging_level
if read_config_file:
self.read_config("amqp")
self.amqp_uri = "amqp://{0}:{1}@{2}:{3}/{4}".format(
self.userid, self.password, self.host, self.port,
self.virtual_host)
self.logger.setLevel(logging_level)
# Event sent try count before event drop in case of connection problem
if max_retries != 5:
self.logger.info('Custom retries value : {} {}'.format(
max_retries, type(max_retries)))
self.max_retries = max_retries
self.exchange_name_events = exchange_name + ".events"
self.exchange_name_alerts = exchange_name + ".alerts"
self.exchange_name_incidents = exchange_name + ".incidents"
self.chan = None
self.conn = None
self.connected = False
self.on_ready = on_ready
self.RUN = True
self.exchanges = {}
self.queues = {}
self.producers = None
self.paused = False
self.connection_errors = (ConnectionError, error, IOError, OSError)
# Create exchange
self.logger.debug("Create exchanges object")
for exchange_name in [
self.exchange_name, self.exchange_name_events,
self.exchange_name_alerts, self.exchange_name_incidents
]:
self.logger.debug(' + {}'.format(exchange_name))
self.get_exchange(exchange_name)
if auto_connect:
self.connect()
self.logger.debug("Object canamqp initialized")
def run(self):
"""
main thread loop
- connects to rabbit
- calls drain_events
- disconnects on error
"""
self.logger.debug("Start thread ...")
reconnect = False
while self.RUN:
self.connect()
if self.connected:
self.init_queue(reconnect=reconnect)
self.logger.debug("Drain events ...")
while self.RUN:
try:
if not self.paused:
self.conn.drain_events(timeout=0.5)
else:
sleep(0.5)
except KombuSerializationError as exc:
self.logger.error(
"Kombu serialization error: invalid message "
"received: {}".format(exc))
except timeout:
pass
except self.connection_errors as err:
self.logger.error(
"Connection error ! ({})".format(err))
break
except KombuSerializationError as exc:
self.logger.error(
"Kombu serialization error: invalid message "
"received: {}".format(exc))
except Exception as err:
self.logger.error("Error: {} ({})".format(
err, type(err)))
print_exc(file=stdout)
break
self.disconnect()
if self.RUN:
self.logger.error(
'Connection lost, try to reconnect in few seconds ...')
reconnect = True
self.wait_connection(timeout=5)
self.logger.debug("End of thread ...")
def stop(self):
self.logger.debug("Stop thread ...")
self.RUN = False
def connect(self):
"""
Create the connection
Inits the producers
Init the only channel
Declares exchanges
"""
if not self.connected:
self.logger.info("Connect to AMQP Broker (%s:%s)" %
(self.host, self.port))
self.conn = Connection(self.amqp_uri)
try:
self.logger.debug(" + Connect")
self.conn.connect()
self.logger.info("Connected to AMQP Broker.")
self.producers = pools.Producers(limit=10)
self.connected = True
except Exception as err:
self.conn.release()
self.logger.error("Impossible to connect ({})".format(err))
if self.connected:
self.logger.debug(" + Open channel")
try:
self.chan = self.conn.channel()
self.logger.info("Channel openned. Ready to send messages")
try:
# Declare exchange
self.logger.debug("Declare exchanges")
for exchange_name in self.exchanges:
self.logger.debug(" + {}".format(exchange_name))
self.exchanges[exchange_name](self.chan).declare()
except Exception as err:
self.logger.error(
"Impossible to declare exchange ({})".format(err))
except Exception as err:
self.logger.error(err)
else:
self.logger.debug("Already connected")
def get_exchange(self, name):
"""
Returns an exchange if stored in self.exchanges.
Otherwise, creates it and returns it
:param string name: name of the exchange to get/create
:rtype: Exchange|None
"""
if name:
try:
return self.exchanges[name]
except Exception:
if name == "amq.direct":
self.exchanges[name] = Exchange(name,
"direct",
durable=True)
else:
self.exchanges[name] = Exchange(name,
"topic",
durable=True,
auto_delete=False)
return self.exchanges[name]
else:
return None
def init_queue(self, reconnect=False):
"""
Loads queue settings
Creates queues and attaches the same channel to each of them
Binds queues to exchange
Revives or creates consumers
calls consume
"""
if self.queues:
self.logger.debug("Init queues")
for queue_name in self.queues.keys():
self.logger.debug(" + {}".format(queue_name))
qsettings = self.queues[queue_name]
if not qsettings['queue']:
self.logger.debug(" + Create queue")
# copy list
routing_keys = list(qsettings['routing_keys'])
routing_key = None
if len(routing_keys):
routing_key = routing_keys[0]
routing_keys = routing_keys[1:]
exchange = self.get_exchange(qsettings['exchange_name'])
if qsettings['exchange_name'] == "amq.direct" \
and not routing_key:
routing_key = queue_name
self.logger.debug("exchange: '{}', exclusive: {},"
" auto_delete: {},no_ack: {}".format(
qsettings['exchange_name'],
qsettings['exclusive'],
qsettings['auto_delete'],
qsettings['no_ack']))
qsettings['queue'] = Queue(
queue_name,
exchange=exchange,
routing_key=routing_key,
exclusive=qsettings['exclusive'],
auto_delete=qsettings['auto_delete'],
no_ack=qsettings['no_ack'],
channel=self.conn.channel())
qsettings['queue'].declare()
if len(routing_keys):
self.logger.debug(" + Bind on all routing keys")
for routing_key in routing_keys:
self.logger.debug(
" + routing_key: '{}'".format(routing_key))
try:
qsettings['queue'].bind_to(
exchange=exchange, routing_key=routing_key)
except Exception:
self.logger.error(
"You need upgrade your Kombu version ({})".
format(__version__))
if qsettings['consumer'] and not reconnect:
qsettings['consumer'].revive(self.chan)
elif not qsettings['consumer'] or reconnect:
self.logger.debug(" + Create Consumer")
qsettings['consumer'] = self.conn.Consumer(
qsettings['queue'], callbacks=[qsettings['callback']])
self.logger.debug(" + Consume queue")
qsettings['consumer'].consume()
if self.on_ready:
self.on_ready()
else:
self.logger.info('Queue already inited')
def add_queue(self,
queue_name,
routing_keys,
callback,
exchange_name=None,
no_ack=True,
exclusive=False,
auto_delete=True):
"""
Initializes the queue configuration
maps the callback on the queue
"""
c_routing_keys = []
if not isinstance(routing_keys, list):
if isinstance(routing_keys, basestring):
c_routing_keys = [routing_keys]
else:
c_routing_keys = routing_keys
# aka if rk is nor a list nor a basetring, leave it empty
if not exchange_name:
exchange_name = self.exchange_name
self.queues[queue_name] = {
'queue': False,
'consumer': False,
'queue_name': queue_name,
'routing_keys': c_routing_keys,
'callback': callback,
'exchange_name': exchange_name,
'no_ack': no_ack,
'exclusive': exclusive,
'auto_delete': auto_delete
}
def publish(self,
msg,
routing_key,
exchange_name=None,
serializer="json",
compression=None,
content_type=None,
content_encoding=None):
"""
Tries to publish an event
In case of failure, tries to reconnect and retry until (self.max_retries)
:returns: operation success status
:rtype: bool
"""
self.logger.warning("Publishing from old.rabbitmq.Amqp is deprecated")
operation_success = False
retries = 0
while not operation_success and retries < self.max_retries:
retries += 1
if self.connected:
if not exchange_name:
exchange_name = self.exchange_name
with self.producers[self.conn].acquire(block=True) as producer:
try:
_msg = msg.copy()
Amqp._clean_msg_for_serialization(_msg)
exchange = self.get_exchange(
exchange_name.encode('utf-8'))
producer.publish(_msg,
serializer=serializer,
compression=compression,
routing_key=routing_key,
exchange=exchange)
self.logger.debug('publish {} in exchange {}'.format(
routing_key, exchange_name))
operation_success = True
except AmqpStructError:
self.logger.warning(
'Malformed message: routing key is too long. '
'Cancelling message')
return False
except Exception:
self.logger.error(' + Impossible to send {}'.format(
traceback.format_exc()))
self.disconnect()
self.connect()
self.init_queue(reconnect=False)
else:
self.logger.error('Not connected ... try reconnecting')
self.connect()
if not operation_success:
# Event and it's information are buffered until next send retry
self.logger.info('Retry count {}'.format(retries))
if not operation_success:
# Event and it's information are buffered until next send retry
self.logger.error(
'Too much retries for event {}, give up'.format(routing_key))
return operation_success
@staticmethod
def _clean_msg_for_serialization(msg):
from bson import objectid
for key in msg:
if isinstance(msg[key], objectid.ObjectId):
msg[key] = str(msg[key])
def cancel_queues(self):
if self.connected:
for queue_name in self.queues.keys():
if self.queues[queue_name]['consumer']:
self.logger.debug(
" + Cancel consumer on {}".format(queue_name))
try:
self.queues[queue_name]['consumer'].cancel()
except Exception:
pass
del (self.queues[queue_name]['consumer'])
self.queues[queue_name]['consumer'] = False
del (self.queues[queue_name]['queue'])
self.queues[queue_name]['queue'] = False
def disconnect(self):
if self.connected:
self.logger.info("Disconnect from AMQP Broker")
self.cancel_queues()
# Force producers closing to permit a clean reconnect after
# ... especially on timeout errors
self.producers[self.conn].force_close_all()
for exchange in self.exchanges:
del exchange
self.exchanges = {}
try:
pools.reset()
except Exception as err:
self.logger.error(
"Impossible to reset kombu pools: {} ({})".format(
err, type(err)))
try:
self.conn.release()
del self.conn
except Exception as err:
self.logger.error(
"Impossible to release connection: {} ({})".format(
err, type(err)))
self.connected = False
def wait_connection(self, timeout=5):
i = 0
while self.RUN and not self.connected and i < (timeout * 2):
sleep(0.5)
i += 1
def read_config(self, name):
filename = join(root_path, 'etc', '{0}.conf'.format(name))
import ConfigParser
self.config = ConfigParser.RawConfigParser()
try:
self.config.read(filename)
section = 'master'
self.host = self.config.get(section, "host")
self.port = self.config.getint(section, "port")
self.userid = self.config.get(section, "userid")
self.password = self.config.get(section, "password")
self.virtual_host = self.config.get(section, "virtual_host")
self.exchange_name = self.config.get(section, "exchange_name")
except Exception as err:
self.logger.error(
"Can't to load configurations ({}), use default ...".format(
err))
def __del__(self):
self.stop()
Queue_PassWord = "******"
Queue_Server = '127.0.0.1'
Queue_Port = 5672
Queue_Path = '/websocketserver'
def callback(body, message):
print body
connection = Connection(hostname=Queue_Server,
port=Queue_Port,
userid=Queue_User,
password=Queue_PassWord,
virtual_host=Queue_Path)
channel = connection.channel()
smsExchange = Exchange("sys.sms",
type='topic',
channel=channel,
durable=True,
delivery_mode=2)
task_queue = Queue('test_recv',
exchange=smsExchange,
routing_key='sms.code',
durable=False,
channel=channel)
consumer = Consumer(channel, task_queue, no_ack=True, callbacks=[callback])
consumer.qos(prefetch_count=1)
consumer.consume()
while True:
connection.drain_events()
class test_Consumer(Case):
def setUp(self):
self.connection = Connection(transport=Transport)
self.connection.connect()
self.assertTrue(self.connection.connection.connected)
self.exchange = Exchange('foo', 'direct')
def test_accept(self):
a = Consumer(self.connection)
self.assertIsNone(a.accept)
b = Consumer(self.connection, accept=['json', 'pickle'])
self.assertSetEqual(
b.accept,
set(['application/json', 'application/x-python-serialize']),
)
c = Consumer(self.connection, accept=b.accept)
self.assertSetEqual(b.accept, c.accept)
def test_enter_exit_cancel_raises(self):
c = Consumer(self.connection)
c.cancel = Mock(name='Consumer.cancel')
c.cancel.side_effect = KeyError('foo')
with c:
pass
c.cancel.assert_called_with()
def test_receive_callback_accept(self):
message = Mock(name='Message')
message.errors = []
callback = Mock(name='on_message')
c = Consumer(self.connection, accept=['json'], on_message=callback)
c.on_decode_error = None
c.channel = Mock(name='channel')
c.channel.message_to_python = None
c._receive_callback(message)
callback.assert_called_with(message)
self.assertSetEqual(message.accept, c.accept)
def test_accept__content_disallowed(self):
conn = Connection('memory://')
q = Queue('foo', exchange=self.exchange)
p = conn.Producer()
p.publish(
{'complex': object()},
declare=[q],
exchange=self.exchange,
serializer='pickle',
)
callback = Mock(name='callback')
with conn.Consumer(queues=[q], callbacks=[callback]) as consumer:
with self.assertRaises(consumer.ContentDisallowed):
conn.drain_events(timeout=1)
self.assertFalse(callback.called)
def test_accept__content_allowed(self):
conn = Connection('memory://')
q = Queue('foo', exchange=self.exchange)
p = conn.Producer()
p.publish(
{'complex': object()},
declare=[q],
exchange=self.exchange,
serializer='pickle',
)
callback = Mock(name='callback')
with conn.Consumer(queues=[q], accept=['pickle'],
callbacks=[callback]):
conn.drain_events(timeout=1)
self.assertTrue(callback.called)
body, message = callback.call_args[0]
self.assertTrue(body['complex'])
def test_set_no_channel(self):
c = Consumer(None)
self.assertIsNone(c.channel)
c.revive(Mock())
self.assertTrue(c.channel)
def test_set_no_ack(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True, no_ack=True)
self.assertTrue(consumer.no_ack)
def test_add_queue_when_auto_declare(self):
consumer = self.connection.Consumer(auto_declare=True)
q = Mock()
q.return_value = q
consumer.add_queue(q)
self.assertIn(q, consumer.queues)
q.declare.assert_called_with()
def test_add_queue_when_not_auto_declare(self):
consumer = self.connection.Consumer(auto_declare=False)
q = Mock()
q.return_value = q
consumer.add_queue(q)
self.assertIn(q, consumer.queues)
self.assertFalse(q.declare.call_count)
def test_consume_without_queues_returns(self):
consumer = self.connection.Consumer()
consumer.queues[:] = []
self.assertIsNone(consumer.consume())
def test_consuming_from(self):
consumer = self.connection.Consumer()
consumer.queues[:] = [Queue('a'), Queue('b'), Queue('d')]
consumer._active_tags = {'a': 1, 'b': 2}
self.assertFalse(consumer.consuming_from(Queue('c')))
self.assertFalse(consumer.consuming_from('c'))
self.assertFalse(consumer.consuming_from(Queue('d')))
self.assertFalse(consumer.consuming_from('d'))
self.assertTrue(consumer.consuming_from(Queue('a')))
self.assertTrue(consumer.consuming_from(Queue('b')))
self.assertTrue(consumer.consuming_from('b'))
def test_receive_callback_without_m2p(self):
channel = self.connection.channel()
c = channel.Consumer()
m2p = getattr(channel, 'message_to_python')
channel.message_to_python = None
try:
message = Mock()
message.errors = []
message.decode.return_value = 'Hello'
recv = c.receive = Mock()
c._receive_callback(message)
recv.assert_called_with('Hello', message)
finally:
channel.message_to_python = m2p
def test_set_callbacks(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
callbacks = [lambda x, y: x, lambda x, y: x]
consumer = Consumer(channel,
queue,
auto_declare=True,
callbacks=callbacks)
self.assertEqual(consumer.callbacks, callbacks)
def test_auto_declare(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.consume() # twice is a noop
self.assertIsNot(consumer.queues[0], queue)
self.assertTrue(consumer.queues[0].is_bound)
self.assertTrue(consumer.queues[0].exchange.is_bound)
self.assertIsNot(consumer.queues[0].exchange, self.exchange)
for meth in ('exchange_declare', 'queue_declare', 'queue_bind',
'basic_consume'):
self.assertIn(meth, channel)
self.assertEqual(channel.called.count('basic_consume'), 1)
self.assertTrue(consumer._active_tags)
consumer.cancel_by_queue(queue.name)
consumer.cancel_by_queue(queue.name)
self.assertFalse(consumer._active_tags)
def test_consumer_tag_prefix(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, tag_prefix='consumer_')
consumer.consume()
self.assertTrue(
consumer._active_tags[queue.name].startswith('consumer_'), )
def test_manual_declare(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=False)
self.assertIsNot(consumer.queues[0], queue)
self.assertTrue(consumer.queues[0].is_bound)
self.assertTrue(consumer.queues[0].exchange.is_bound)
self.assertIsNot(consumer.queues[0].exchange, self.exchange)
for meth in ('exchange_declare', 'queue_declare', 'basic_consume'):
self.assertNotIn(meth, channel)
consumer.declare()
for meth in ('exchange_declare', 'queue_declare', 'queue_bind'):
self.assertIn(meth, channel)
self.assertNotIn('basic_consume', channel)
consumer.consume()
self.assertIn('basic_consume', channel)
def test_consume__cancel(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.cancel()
self.assertIn('basic_cancel', channel)
self.assertFalse(consumer._active_tags)
def test___enter____exit__(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
context = consumer.__enter__()
self.assertIs(context, consumer)
self.assertTrue(consumer._active_tags)
res = consumer.__exit__(None, None, None)
self.assertFalse(res)
self.assertIn('basic_cancel', channel)
self.assertFalse(consumer._active_tags)
def test_flow(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.flow(False)
self.assertIn('flow', channel)
def test_qos(self):
channel = self.connection.channel()
queue = Queue('qname', self.exchange, 'rkey')
consumer = Consumer(channel, queue, auto_declare=True)
consumer.qos(30, 10, False)
self.assertIn('basic_qos', channel)
def test_purge(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
b2 = Queue('qname2', self.exchange, 'rkey')
b3 = Queue('qname3', self.exchange, 'rkey')
b4 = Queue('qname4', self.exchange, 'rkey')
consumer = Consumer(channel, [b1, b2, b3, b4], auto_declare=True)
consumer.purge()
self.assertEqual(channel.called.count('queue_purge'), 4)
def test_multiple_queues(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
b2 = Queue('qname2', self.exchange, 'rkey')
b3 = Queue('qname3', self.exchange, 'rkey')
b4 = Queue('qname4', self.exchange, 'rkey')
consumer = Consumer(channel, [b1, b2, b3, b4])
consumer.consume()
self.assertEqual(channel.called.count('exchange_declare'), 4)
self.assertEqual(channel.called.count('queue_declare'), 4)
self.assertEqual(channel.called.count('queue_bind'), 4)
self.assertEqual(channel.called.count('basic_consume'), 4)
self.assertEqual(len(consumer._active_tags), 4)
consumer.cancel()
self.assertEqual(channel.called.count('basic_cancel'), 4)
self.assertFalse(len(consumer._active_tags))
def test_receive_callback(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
received = []
def callback(message_data, message):
received.append(message_data)
message.ack()
message.payload # trigger cache
consumer.register_callback(callback)
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_ack', channel)
self.assertIn('message_to_python', channel)
self.assertEqual(received[0], {'foo': 'bar'})
def test_basic_ack_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.ack()
message.ack()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
def test_basic_reject(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
consumer.register_callback(callback)
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject', channel)
def test_basic_reject_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
message.reject()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject', channel)
def test_basic_reject__requeue(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
consumer.register_callback(callback)
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject:requeue', channel)
def test_basic_reject__requeue_twice(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
message.requeue()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({'foo': 'bar'})
self.assertIn('basic_reject:requeue', channel)
def test_receive_without_callbacks_raises(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
with self.assertRaises(NotImplementedError):
consumer.receive(1, 2)
def test_decode_error(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
consumer.channel.throw_decode_error = True
with self.assertRaises(ValueError):
consumer._receive_callback({'foo': 'bar'})
def test_on_decode_error_callback(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
thrown = []
def on_decode_error(msg, exc):
thrown.append((msg.body, exc))
consumer = Consumer(channel, [b1], on_decode_error=on_decode_error)
consumer.channel.throw_decode_error = True
consumer._receive_callback({'foo': 'bar'})
self.assertTrue(thrown)
m, exc = thrown[0]
self.assertEqual(anyjson.loads(m), {'foo': 'bar'})
self.assertIsInstance(exc, ValueError)
def test_recover(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
consumer.recover()
self.assertIn('basic_recover', channel)
def test_revive(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
consumer = Consumer(channel, [b1])
channel2 = self.connection.channel()
consumer.revive(channel2)
self.assertIs(consumer.channel, channel2)
self.assertIs(consumer.queues[0].channel, channel2)
self.assertIs(consumer.queues[0].exchange.channel, channel2)
def test__repr__(self):
channel = self.connection.channel()
b1 = Queue('qname1', self.exchange, 'rkey')
self.assertTrue(repr(Consumer(channel, [b1])))
def test_connection_property_handles_AttributeError(self):
p = self.connection.Consumer()
p.channel = object()
self.assertIsNone(p.connection)
def _get_producer_kombu_objects(cls, url, exchange_name):
connection = Connection(url)
channel = connection.channel()
return connection, channel
class test_MemoryTransport(TestCase):
def setUp(self):
self.c = Connection(transport='memory')
self.e = Exchange('test_transport_memory')
self.q = Queue('test_transport_memory',
exchange=self.e,
routing_key='test_transport_memory')
self.q2 = Queue('test_transport_memory2',
exchange=self.e,
routing_key='test_transport_memory2')
def test_produce_consume_noack(self):
channel = self.c.channel()
producer = Producer(channel, self.e)
consumer = Consumer(channel, self.q, no_ack=True)
for i in range(10):
producer.publish({'foo': i}, routing_key='test_transport_memory')
_received = []
def callback(message_data, message):
_received.append(message)
consumer.register_callback(callback)
consumer.consume()
while 1:
if len(_received) == 10:
break
self.c.drain_events()
self.assertEqual(len(_received), 10)
def test_produce_consume(self):
channel = self.c.channel()
producer = Producer(channel, self.e)
consumer1 = Consumer(channel, self.q)
consumer2 = Consumer(channel, self.q2)
self.q2(channel).declare()
for i in range(10):
producer.publish({'foo': i}, routing_key='test_transport_memory')
for i in range(10):
producer.publish({'foo': i}, routing_key='test_transport_memory2')
_received1 = []
_received2 = []
def callback1(message_data, message):
_received1.append(message)
message.ack()
def callback2(message_data, message):
_received2.append(message)
message.ack()
consumer1.register_callback(callback1)
consumer2.register_callback(callback2)
consumer1.consume()
consumer2.consume()
while 1:
if len(_received1) + len(_received2) == 20:
break
self.c.drain_events()
self.assertEqual(len(_received1) + len(_received2), 20)
# compression
producer.publish({'compressed': True},
routing_key='test_transport_memory',
compression='zlib')
m = self.q(channel).get()
self.assertDictEqual(m.payload, {'compressed': True})
# queue.delete
for i in range(10):
producer.publish({'foo': i}, routing_key='test_transport_memory')
self.assertTrue(self.q(channel).get())
self.q(channel).delete()
self.q(channel).declare()
self.assertIsNone(self.q(channel).get())
# queue.purge
for i in range(10):
producer.publish({'foo': i}, routing_key='test_transport_memory2')
self.assertTrue(self.q2(channel).get())
self.q2(channel).purge()
self.assertIsNone(self.q2(channel).get())
def test_drain_events(self):
with self.assertRaises(socket.timeout):
self.c.drain_events(timeout=0.1)
c1 = self.c.channel()
c2 = self.c.channel()
with self.assertRaises(socket.timeout):
self.c.drain_events(timeout=0.1)
del (c1) # so pyflakes doesn't complain.
del (c2)
def test_drain_events_unregistered_queue(self):
c1 = self.c.channel()
class Cycle(object):
def get(self, timeout=None):
return ('foo', 'foo'), c1
self.c.transport.cycle = Cycle()
with self.assertRaises(KeyError):
self.c.drain_events()
def test_queue_for(self):
chan = self.c.channel()
chan.queues.clear()
x = chan._queue_for('foo')
self.assertTrue(x)
self.assertIs(chan._queue_for('foo'), x)
class test_ConsumerSet(Case):
def setup(self):
self.connection = Connection(transport=Transport)
def test_providing_channel(self):
chan = Mock(name='channel')
cs = compat.ConsumerSet(self.connection, channel=chan)
self.assertTrue(cs._provided_channel)
self.assertIs(cs.backend, chan)
cs.cancel = Mock(name='cancel')
cs.close()
chan.close.assert_not_called()
@patch('kombu.compat._iterconsume')
def test_iterconsume(self, _iterconsume, n='test_iterconsume'):
c = compat.Consumer(self.connection, queue=n, exchange=n)
cs = compat.ConsumerSet(self.connection, consumers=[c])
cs.iterconsume(limit=10, no_ack=True)
_iterconsume.assert_called_with(c.connection, cs, True, 10)
def test_revive(self, n='test_revive'):
c = compat.Consumer(self.connection, queue=n, exchange=n)
cs = compat.ConsumerSet(self.connection, consumers=[c])
with self.connection.channel() as c2:
cs.revive(c2)
self.assertIs(cs.backend, c2)
def test_constructor(self, prefix='0daf8h21'):
dcon = {
'%s.xyx' % prefix: {
'exchange': '%s.xyx' % prefix,
'routing_key': 'xyx'
},
'%s.xyz' % prefix: {
'exchange': '%s.xyz' % prefix,
'routing_key': 'xyz'
}
}
consumers = [
compat.Consumer(self.connection,
queue=prefix + str(i),
exchange=prefix + str(i)) for i in range(3)
]
c = compat.ConsumerSet(self.connection, consumers=consumers)
c2 = compat.ConsumerSet(self.connection, from_dict=dcon)
self.assertEqual(len(c.queues), 3)
self.assertEqual(len(c2.queues), 2)
c.add_consumer(
compat.Consumer(self.connection,
queue=prefix + 'xaxxxa',
exchange=prefix + 'xaxxxa'))
self.assertEqual(len(c.queues), 4)
for cq in c.queues:
self.assertIs(cq.channel, c.channel)
c2.add_consumer_from_dict(
'%s.xxx' % prefix,
exchange='%s.xxx' % prefix,
routing_key='xxx',
)
self.assertEqual(len(c2.queues), 3)
for c2q in c2.queues:
self.assertIs(c2q.channel, c2.channel)
c.discard_all()
self.assertEqual(c.channel.called.count('queue_purge'), 4)
c.consume()
c.close()
c2.close()
self.assertIn('basic_cancel', c.channel)
self.assertIn('close', c.channel)
self.assertIn('close', c2.channel)