def test_redis_info_raises(self): pool = Mock(name='pool') pool_at_init = [pool] client = Mock(name='client') class XChannel(Channel): def __init__(self, *args, **kwargs): self._pool = pool_at_init[0] super(XChannel, self).__init__(*args, **kwargs) def _get_client(self): return lambda *_, **__: client class XTransport(Transport): Channel = XChannel conn = Connection(transport=XTransport) client.info.side_effect = RuntimeError() with self.assertRaises(RuntimeError): conn.channel() pool.disconnect.assert_called_with() pool.disconnect.reset_mock() pool_at_init = [None] with self.assertRaises(RuntimeError): conn.channel() self.assertFalse(pool.disconnect.called)
def test_region(self): import boto3 _environ = dict(os.environ) # when the region is unspecified connection = Connection(transport=SQS.Transport) channel = connection.channel() assert channel.transport_options.get('region') is None # the default region is us-east-1 assert channel.region == 'us-east-1' # when boto3 picks a region os.environ['AWS_DEFAULT_REGION'] = 'us-east-2' assert boto3.Session().region_name == 'us-east-2' # the default region should match connection = Connection(transport=SQS.Transport) channel = connection.channel() assert channel.region == 'us-east-2' # when transport_options are provided connection = Connection(transport=SQS.Transport, transport_options={ 'region': 'us-west-2' }) channel = connection.channel() assert channel.transport_options.get('region') == 'us-west-2' # the specified region should be used assert connection.channel().region == 'us-west-2' os.environ.clear() os.environ.update(_environ)
def test_can_create_connection(self): from redis.exceptions import ConnectionError with self.assertRaises(ConnectionError): connection = Connection('sentinel://localhost:65534/', transport_options={ 'master_name': 'not_important' }) connection.channel()
def test_method_called(self): from kombu.transport.redis import SentinelChannel with patch.object(SentinelChannel, '_sentinel_managed_pool') as patched: connection = Connection('sentinel://localhost:65534/', transport_options={ 'master_name': 'not_important' }) connection.channel() self.assertTrue(patched.called)
def test_getting_master_from_sentinel(self): from redis.sentinel import Sentinel with patch.object(Sentinel, '__new__') as patched: connection = Connection('sentinel://localhost:65534/', transport_options={ 'master_name': 'not_important' }) connection.channel() self.assertTrue(patched) sentinel_obj = patched.return_value self.assertTrue(sentinel_obj.master_for.called, 'master_for was not called') sentinel_obj.master_for.assert_called_with('not_important', ANY) self.assertTrue(sentinel_obj.master_for().connection_pool.get_connection.called, 'get_connection on redis connection pool was not called')
def test_getting_master_from_sentinel(self): with patch('redis.sentinel.Sentinel') as patched: connection = Connection( 'sentinel://localhost:65534/', transport_options={ 'master_name': 'not_important', }, ) connection.channel() assert patched master_for = patched.return_value.master_for master_for.assert_called() master_for.assert_called_with('not_important', ANY) master_for().connection_pool.get_connection.assert_called()
class AMQPConnection(object): """ A class representing a connection to an amqp broker. """ def __init__(self, configuration): self.logger = logging.getLogger('narc.amqp.AMQPConnection') self.url = str.format("amqp://{hostname}:{port}", **dict(configuration['AMQP'].items())) if 'virtual host' in configuration['AMQP'] and configuration['AMQP']['virtual host'] != '': self.url = str.format("{}/{}", self.url, configuration['AMQP']['virtual host']) self.logger.debug("AMQPConnection configured with url {}", self.url) self.exchange = Exchange(configuration['AMQP'].get('exchange', "amqp.topic"), type='topic') self.logger.debug("AMQPConnection is using exchange {}", self.exchange) if 'username' in configuration['AMQP'] and 'password' in configuration['AMQP']: username = configuration['AMQP']['username'] password = configuration['AMQP']['password'] self.logger.debug("Using username {} and password {} to connect to AMQP Broker", username, password) self.connection = Connection(self.url, userid=username, password=password) else: self.connection = Connection(self.url) self.channels = [] self.consumers = [] self.default_channel = self.connection.default_channel def add_channel(self): self.logger.debug("Adding channel number {} to the list of channels", len(self.channels) + 1) self.channels.append(self.connection.channel()) return self.channels[-1] def add_consumer(self, consumer): assert(isinstance(consumer, Consumer)) self.logger.debug("Adding consumer number {} to the list of consumers", len(self.consumers) + 1) self.consumers.append(consumer)
class test_StdChannel: def setup(self): self.conn = Connection('memory://') self.channel = self.conn.channel() self.channel.queues.clear() self.conn.connection.state.clear() def test_Consumer(self): q = Queue('foo', Exchange('foo')) print(self.channel.queues) cons = self.channel.Consumer(q) assert isinstance(cons, Consumer) assert cons.channel is self.channel def test_Producer(self): prod = self.channel.Producer() assert isinstance(prod, Producer) assert prod.channel is self.channel def test_interface_get_bindings(self): with pytest.raises(NotImplementedError): StdChannel().get_bindings() def test_interface_after_reply_message_received(self): assert StdChannel().after_reply_message_received(Queue('foo')) is None
class QueuePush(object): def __init__(self,Queue_Server,Queue_Port,Queue_User,Queue_PassWord,Queue_Path): self.usr=Queue_User self.psw=Queue_PassWord self.server=Queue_Server self.port=Queue_Port self.path=Queue_Path self.connection=None self.smsExchange=None def _InitConnect(self): if self.connection is None or self.connection.connected==False: self.connection = Connection(hostname=self.server,port=self.port,userid=self.usr,password=self.psw,virtual_host=self.path) self.channel = self.connection.channel() self.producer=Producer(self.channel) self.smsExchange=Exchange("sys.sms",type='topic',channel=self.channel,durable=True,delivery_mode=2) self.smsCodeProduce=Producer(self.channel,self.smsExchange,routing_key='sms.code') def Push(self,queueid,connectid,body): self.rawPush(queueid,{'connid':connectid},body) def Close(self,queueid,connectid): self.rawPush(queueid,{'connid':connectid,'close_connect':'1'},'close') def rawPush(self,routing_key,headers,body): self._InitConnect() self.producer.publish(body=body,delivery_mode=2,headers=headers, routing_key=routing_key,retry=True,compression='gzip') def sendCode(self,phone,code): self._InitConnect() json_str=json.dumps({'phone':str(phone),"content":u"您的莱信验证码为:%s,请在5分钟内输入完成验证。【莱福思】"%str(code)},ensure_ascii=False) self.smsCodeProduce.publish(body=json_str,retry=True,compression='gzip')
class test_StdChannel(TestCase): def setUp(self): self.conn = Connection('memory://') self.channel = self.conn.channel() self.channel.queues.clear() self.conn.connection.state.clear() def test_Consumer(self): q = Queue('foo') print(self.channel.queues) cons = self.channel.Consumer(q) self.assertIsInstance(cons, Consumer) self.assertIs(cons.channel, self.channel) def test_Producer(self): prod = self.channel.Producer() self.assertIsInstance(prod, Producer) self.assertIs(prod.channel, self.channel) def test_interface_get_bindings(self): with self.assertRaises(NotImplementedError): StdChannel().get_bindings() def test_interface_after_reply_message_received(self): self.assertIsNone( StdChannel().after_reply_message_received(Queue('foo')), )
class AMQPConnection(object): conf = None connection = None producer = None needs_reconnect = False def __init__(self, conf, exchange): self.conf = conf self.exchange = exchange self.reconnect() def reconnect(self): self.needs_reconnect = False LOG.info("Opening new AMQP connection to amqp://%s@%s:%s%s (%s)" % ( self.conf.amqp_userid, self.conf.amqp_hostname, self.conf.amqp_port, self.conf.amqp_virtual_host, self.exchange.name)) if self.connection: self.connection.release() try: self.connection = Connection( hostname=self.conf.amqp_hostname, userid=self.conf.amqp_userid, password=self.conf.amqp_password, virtual_host=self.conf.amqp_virtual_host, port=self.conf.amqp_port) channel = self.connection.channel() # get a new channel self.producer = Producer(channel, self.exchange, auto_declare=False) except Exception, e: LOG.error("Error opening AMQP connection: %s" % e) self.needs_reconnect = True
def _do_test(): conn = Connection(transport=Transport) chan = conn.channel() self.assertTrue(chan.Client) self.assertTrue(chan.ResponseError) self.assertTrue(conn.transport.connection_errors) self.assertTrue(conn.transport.channel_errors)
def test_check_at_least_we_try_to_connect_and_fail(self): import redis connection = Connection('redis://localhost:65534/') with pytest.raises(redis.exceptions.ConnectionError): chan = connection.channel() chan._size('some_queue')
class test_ConsumerSet(TestCase): def setUp(self): self.connection = Connection(transport=Transport) @patch('kombu.compat._iterconsume') def test_iterconsume(self, _iterconsume, n='test_iterconsume'): c = compat.Consumer(self.connection, queue=n, exchange=n) cs = compat.ConsumerSet(self.connection, consumers=[c]) cs.iterconsume(limit=10, no_ack=True) _iterconsume.assert_called_with(c.connection, cs, True, 10) def test_revive(self, n='test_revive'): c = compat.Consumer(self.connection, queue=n, exchange=n) cs = compat.ConsumerSet(self.connection, consumers=[c]) with self.connection.channel() as c2: cs.revive(c2) self.assertIs(cs.backend, c2) def test_constructor(self, prefix='0daf8h21'): dcon = {'%s.xyx' % prefix: {'exchange': '%s.xyx' % prefix, 'routing_key': 'xyx'}, '%s.xyz' % prefix: {'exchange': '%s.xyz' % prefix, 'routing_key': 'xyz'}} consumers = [compat.Consumer(self.connection, queue=prefix + str(i), exchange=prefix + str(i)) for i in range(3)] c = compat.ConsumerSet(self.connection, consumers=consumers) c2 = compat.ConsumerSet(self.connection, from_dict=dcon) self.assertEqual(len(c.queues), 3) self.assertEqual(len(c2.queues), 2) c.add_consumer(compat.Consumer(self.connection, queue=prefix + 'xaxxxa', exchange=prefix + 'xaxxxa')) self.assertEqual(len(c.queues), 4) for cq in c.queues: self.assertIs(cq.channel, c.channel) c2.add_consumer_from_dict({ '%s.xxx' % prefix: { 'exchange': '%s.xxx' % prefix, 'routing_key': 'xxx', }, }) self.assertEqual(len(c2.queues), 3) for c2q in c2.queues: self.assertIs(c2q.channel, c2.channel) c.discard_all() self.assertEqual(c.channel.called.count('queue_purge'), 4) c.consume() c.close() c2.close() self.assertIn('basic_cancel', c.channel) self.assertIn('close', c.channel) self.assertIn('close', c2.channel)
def test_socket_connection(self): connection = Connection('redis+socket:///tmp/redis.sock', transport=Transport) connparams = connection.channel()._connparams() self.assertEqual(connparams['connection_class'], redis.redis.UnixDomainSocketConnection) self.assertEqual(connparams['path'], '/tmp/redis.sock')
class PollingQueueConsumer(object): """ Implements a minimum interface of the :class:`~messaging.QueueConsumer`. Instead of processing messages in a separate thread it provides a polling method to block until a message with the same correlation ID of the RPC-proxy call arrives. """ def register_provider(self, provider): self.provider = provider self.connection = Connection(provider.container.config['AMQP_URI']) self.channel = self.connection.channel() self.queue = provider.queue maybe_declare(self.queue, self.channel) def unregister_provider(self, provider): self.connection.close() def ack_message(self, msg): msg.ack() def poll_messages(self, correlation_id): channel = self.channel conn = channel.connection for body, msg in itermessages(conn, channel, self.queue, limit=None): if correlation_id == msg.properties.get('correlation_id'): self.provider.handle_message(body, msg) break
def test_get_client(self): with mock.module_exists(*_redis_modules()): conn = Connection(transport=Transport) chan = conn.channel() assert chan.Client assert chan.ResponseError assert conn.transport.connection_errors assert conn.transport.channel_errors
def test_disable_ack_emulation(self): conn = Connection(transport=Transport, transport_options={ 'ack_emulation': False, }) chan = conn.channel() assert not chan.ack_emulation assert chan.QoS == virtual.QoS
def test_disable_ack_emulation(self): conn = Connection(transport=Transport, transport_options={ 'ack_emulation': False, }) chan = conn.channel() self.assertFalse(chan.ack_emulation) self.assertEqual(chan.QoS, virtual.QoS)
def worker(mq_url): connection = Connection(mq_url) channel = connection.channel() consumer_json = Consumer(channel, task_json_queue, callbacks=[process_json], accept=["json"]) consumer_json.consume() consumer_pickle = Consumer(channel, task_pickle_queue, callbacks=[process_pickle], accept=["pickle"]) consumer_pickle.consume() while True: connection.drain_events()
def init(host,port,virtual_host,usr,psw,queue_name): global connection,channel,producer,task_queue,consumer connection = Connection(hostname=host,port=port,userid=usr,password=psw,virtual_host=virtual_host) channel = connection.channel() producer=Producer(channel) task_queue = Queue(queue_name,durable=True) consumer = Consumer(channel,task_queue,no_ack=False) consumer.qos(prefetch_count=1) consumer.register_callback(RequestCallBack)
class test_ConsumerSet(TestCase): def setUp(self): self.connection = Connection(transport=Transport) @patch("kombu.compat._iterconsume") def test_iterconsume(self, _iterconsume, n="test_iterconsume"): c = compat.Consumer(self.connection, queue=n, exchange=n) cs = compat.ConsumerSet(self.connection, consumers=[c]) cs.iterconsume(limit=10, no_ack=True) _iterconsume.assert_called_with(c.connection, cs, True, 10) def test_revive(self, n="test_revive"): c = compat.Consumer(self.connection, queue=n, exchange=n) cs = compat.ConsumerSet(self.connection, consumers=[c]) with self.connection.channel() as c2: cs.revive(c2) self.assertIs(cs.backend, c2) def test_constructor(self, prefix="0daf8h21"): dcon = {"%s.xyx" % prefix: {"exchange": "%s.xyx" % prefix, "routing_key": "xyx"}, "%s.xyz" % prefix: {"exchange": "%s.xyz" % prefix, "routing_key": "xyz"}} consumers = [compat.Consumer(self.connection, queue=prefix + str(i), exchange=prefix + str(i)) for i in range(3)] c = compat.ConsumerSet(self.connection, consumers=consumers) c2 = compat.ConsumerSet(self.connection, from_dict=dcon) self.assertEqual(len(c.queues), 3) self.assertEqual(len(c2.queues), 2) c.add_consumer(compat.Consumer(self.connection, queue=prefix + "xaxxxa", exchange=prefix + "xaxxxa")) self.assertEqual(len(c.queues), 4) for cq in c.queues: self.assertIs(cq.channel, c.channel) c2.add_consumer_from_dict({"%s.xxx" % prefix: { "exchange": "%s.xxx" % prefix, "routing_key": "xxx"}}) self.assertEqual(len(c2.queues), 3) for c2q in c2.queues: self.assertIs(c2q.channel, c2.channel) c.discard_all() self.assertEqual(c.channel.called.count("queue_purge"), 4) c.consume() c.close() c2.close() self.assertIn("basic_cancel", c.channel) self.assertIn("close", c.channel) self.assertIn("close", c2.channel)
def test_simple_queueing(self): conn = Connection('sqlalchemy+sqlite:///:memory:') conn.connect() channel = conn.channel() self.assertEqual( channel.queue_cls.__table__.name, 'kombu_queue' ) self.assertEqual( channel.message_cls.__table__.name, 'kombu_message' ) channel._put('celery', 'DATA') assert channel._get('celery') == 'DATA'
class RabbitMQConsumer(ConsumerMixin): log = logging.getLogger("%s.RabbitMQConsumer" %(__name__)) def __init__(self, url, exchange_type, routing_key, queue_name, bindings, callbacks=None, namespace="local"): self.namespace = namespace exchange = Exchange(type=exchange_type) self.queues = [ Queue(queue_name, exchange=exchange) ] self.connection = Connection(url) channel = self.connection.channel() bound_queue = self.queues[0].bind(channel) bound_queue.declare() for binding in bindings: bound_queue.bind_to(binding, routing_key) if callbacks != None: if isinstance(callbacks, list): self.userCallbacks = callbacks else: self.userCallbacks = [ callbacks ] else: self.userCallbacks = None self.log.info("Queue bindings:\n\n\t%s == > %s\n" %(queue_name, bindings)) def get_consumers(self, Consumer, channel): return [Consumer(queues=self.queues, auto_declare=False, callbacks=[self.processEvent])] def processEvent(self, body, message): #body['event_type'] = "%s.%s" %(self.namespace, body['event_type']) for callback in self.userCallbacks: callback(body) message.ack() def start(self): self.log.info("Starting openstack event consumer...") self.run() def stop(self): self.log.info("Stopping openstack event consumer...") self.should_stop = True
def init(host, port, virtual_host, usr, psw, queue_name, exchange_name=None, routing_key=None): global connection, channel, producer, task_queue, consumer connection = Connection(hostname=host, port=port, userid=usr, password=psw, virtual_host=virtual_host) channel = connection.channel() channel.auto_decode = False producer = Producer(channel) if exchange_name: exchange = Exchange(exchange_name, "topic", channel, durable=True, delivery_mode=2) task_queue = Queue(queue_name, routing_key=routing_key, durable=True, exchange=exchange) else: task_queue = Queue(queue_name, durable=True) consumer = Consumer(channel, task_queue, no_ack=False) consumer.qos(prefetch_count=1) consumer.register_callback(RequestCallBack)
def run(self): try: connection = Connection(hostname=self.host,port=self.port,userid=self.usr,password=self.psw,virtual_host=self.virtual_host) channel = connection.channel() self.producer=Producer(channel) task_queue = Queue(self.queue_name,durable=True) consumer = Consumer(channel,task_queue,no_ack=False) consumer.qos(prefetch_count=1) consumer.register_callback(self.RequestCallBack) consumer.consume() while True: connection.drain_events() connection.close() except BaseException,e: print e
def test_custom_table_names(self): conn = Connection('sqlalchemy+sqlite:///:memory:', transport_options={ 'queue_tablename': 'my_custom_queue', 'message_tablename': 'my_custom_message' }) conn.connect() channel = conn.channel() self.assertEqual( channel.queue_cls.__table__.name, 'my_custom_queue' ) self.assertEqual( channel.message_cls.__table__.name, 'my_custom_message' ) channel._put('celery', 'DATA') assert channel._get('celery') == 'DATA'
class test_Message(TestCase): def setUp(self): self.conn = Connection('memory://') self.channel = self.conn.channel() self.message = Message(self.channel, delivery_tag=313) def test_ack_respects_no_ack_consumers(self): self.channel.no_ack_consumers = set(['abc']) self.message.delivery_info['consumer_tag'] = 'abc' ack = self.channel.basic_ack = Mock() self.message.ack() self.assertNotEqual(self.message._state, 'ACK') self.assertFalse(ack.called) def test_ack_missing_consumer_tag(self): self.channel.no_ack_consumers = set(['abc']) self.message.delivery_info = {} ack = self.channel.basic_ack = Mock() self.message.ack() ack.assert_called_with(self.message.delivery_tag) def test_ack_not_no_ack(self): self.channel.no_ack_consumers = set() self.message.delivery_info['consumer_tag'] = 'abc' ack = self.channel.basic_ack = Mock() self.message.ack() ack.assert_called_with(self.message.delivery_tag) def test_ack_log_error_when_no_error(self): ack = self.message.ack = Mock() self.message.ack_log_error(Mock(), KeyError) ack.assert_called_with() def test_ack_log_error_when_error(self): ack = self.message.ack = Mock() ack.side_effect = KeyError('foo') logger = Mock() self.message.ack_log_error(logger, KeyError) ack.assert_called_with() self.assertTrue(logger.critical.called) self.assertIn("Couldn't ack", logger.critical.call_args[0][0])
def run(self): connection = Connection(hostname=self.host,port=self.port,userid=self.usr,password=self.psw,virtual_host=self.virtual_host) channel = connection.channel() self.producer=Producer(channel) queueargs={} if self.msg_timeout: queueargs['x-message-ttl']=self.msg_timeout task_queue = Queue(self.queue_name,durable=True,queue_arguments=queueargs if queueargs else None) consumer = Consumer(channel,task_queue,no_ack=False) consumer.qos(prefetch_count=1) consumer.register_callback(self.RequestCallBack) consumer.consume() while self.task_count: connection.drain_events() self.task_count-=1 connection.close()
def check_rabbit_connections(self): if not self._controllers: self.fail('There are no online controllers') pwd = self.get_conf_values().strip() for host in self._controllers: try: conn = Connection(host, userid='nova', password=pwd, virtual_host='/', port=5673) conn.connect() channel = conn.channel() self.connections.append((channel, host)) LOG.debug('connections is {0}'.format(self.connections)) except Exception: LOG.debug(traceback.format_exc()) self.fail("Failed to connect to " "5673 port on host {0}".format(host))
def mock_queue(mock_asb, mock_asb_management, random_queue) -> MockQueue: exchange = Exchange('test_servicebus', type='direct') queue = Queue(random_queue, exchange, random_queue) conn = Connection(URL_CREDS, transport=azureservicebus.Transport) channel = conn.channel() channel._queue_service = mock_asb channel._queue_mgmt_service = mock_asb_management queue(channel).declare() producer = messaging.Producer(channel, exchange, routing_key=random_queue) return MockQueue( random_queue, mock_asb, mock_asb_management, conn, channel, producer, queue )
def setup_rabbitMQ(): rabbit_domain = os.environ.get('RABBITMQ_URL') rabbit_domain = rabbit_domain if rabbit_domain is not None else 'localhost' rabbit_url = 'amqp://*****:*****@' + rabbit_domain + ':5672//' # Kombu Connection conn = Connection(rabbit_url) channel = conn.channel() # Kombu Exchange # - set delivery_mode to transient to prevent disk writes for faster delivery exchange = Exchange("video-exchange", type="direct", delivery_mode=1) # Kombu Producer producer = Producer(exchange=exchange, channel=channel, routing_key="video") # Kombu Queue queue = Queue(name="video-queue", exchange=exchange, routing_key="video") queue.maybe_bind(conn) queue.declare() return queue, exchange, producer, conn
class MQ: """ This class connect to openstack mq to get notifications. :keyword mq_user: openstack mq's username. :keyword mq_password: openstack mq's password. :keyword mq_host: openstack mq's host ip. """ def __init__(self, mq_user, mq_password, mq_host): self.mq_user = mq_user self.mq_password = mq_password self.mq_host = mq_host self.connection = None def __repr__(self): return "user={0},password={1},host={2}".format( self.mq_user, self.mq_password, self.mq_host) __str__ = __repr__ def create_connection(self): host = "amqp://{0}:{1}@{2}//".format(self.mq_user, self.mq_password, self.mq_host) log.info("create connection: " + host) self.connection = Connection(host) return self.connection def create_consumer(self, exchange_name, queue_name, process): try: channel = self.connection.channel() exchange = Exchange(exchange_name, type="topic") queue = Queue(queue_name, exchange, routing_key="notifications.#") consumer = Consumer(channel, queue) consumer.register_callback(process) consumer.consume() log.info("create consumer: " + repr(consumer)) except OSError: raise MQConnectionError("please check your mq user, password and host configuration.") return self.connection
def _rabbit_mq(messages): conn = Connection(host) channel = conn.channel() exchange = Exchange(exchange_name, type="direct") queue = Queue(name=queue_name, exchange=exchange, routing_key=routing_key) queue.maybe_bind(conn) queue.declare() producer = Producer(exchange=exchange, channel=channel, routing_key=routing_key) for message in messages: # as_dict = message.asDict(recursive=True) producer.publish(message) channel.close() conn.release() return messages
def send(self): try: # Connection conn = Connection(self.broker) # Channel channel = conn.channel() # Exchange task_exchange = Exchange(self._exchange_name, type=self._queue_type) # Queues if self._queue_name: queue = Queue(name=self._queue_name, channel=channel, exchange=task_exchange, routing_key=self._routing_key) queue.declare() # Producer producer = Producer(exchange=task_exchange, channel=channel, routing_key=self._routing_key) # Send message for message in self._msgs: serialized_message = json.dumps(message, ensure_ascii=False) producer.publish(serialized_message) conn.close() except Exception, e: self.log.error( u'QueueManagerError - Error on sending objects from queue.') self.log.debug(e) raise Exception( 'QueueManagerError - Error on sending objects to queue.')
class KombuQueueLengther(object): KOMBU_HEARTBEAT = 2 def __init__(self, url, queue): self.connection = Connection(url, heartbeat=self.KOMBU_HEARTBEAT) self.queue = queue self._maybe_dirty = False @backoff.on_exception(backoff.fibo, ConnectionError, max_value=9, max_time=30, jitter=backoff.full_jitter) def __len__(self): if self._maybe_dirty: time.sleep(self.KOMBU_HEARTBEAT * 1.5) result = self.connection.channel()._size(self.queue) # Kombu queue length will not change until next heartbeat. # Would be better to use a token-bucket timeout, # but some `time.delay()` will do for now. self._maybe_dirty = True return result
def republish(self, backoff_exc, message, target_queue): expiration = backoff_exc.next(message, self.exchange.name) queue = self.make_queue(expiration) properties = message.properties.copy() headers = properties.pop('application_headers', {}) headers['backoff'] = expiration expiration_seconds = float(expiration) / 1000 amqp_uri = self.container.config[AMQP_URI_CONFIG_KEY] # force redeclaration; # In kombu versions prior to 4.3.0, the publisher will skip declaration if # the entity has previously been declared by the same connection. # (see https://github.com/celery/kombu/pull/884) conn = Connection(amqp_uri) if KOMBU_PRE_4_3: # pragma: no cover maybe_declare( queue, conn.channel(), retry=True, **DEFAULT_RETRY_POLICY ) # republish to appropriate backoff queue publisher = Publisher(amqp_uri) publisher.publish( message.body, headers=headers, exchange=self.exchange, routing_key=target_queue, expiration=expiration_seconds, mandatory=True, retry=True, retry_policy=DEFAULT_RETRY_POLICY, declare=[queue.exchange, queue], **properties )
class test_StdChannel: def setup(self): self.conn = Connection('memory://') self.channel = self.conn.channel() self.channel.queues.clear() self.conn.connection.state.clear() def test_Consumer(self): q = Queue('foo', Exchange('foo')) cons = self.channel.Consumer(q) assert isinstance(cons, Consumer) assert cons.channel is self.channel def test_Producer(self): prod = self.channel.Producer() assert isinstance(prod, Producer) assert prod.channel is self.channel def test_interface_get_bindings(self): with pytest.raises(NotImplementedError): StdChannel().get_bindings() def test_interface_after_reply_message_received(self): assert StdChannel().after_reply_message_received(Queue('foo')) is None
def test_publish__consume(self): connection = Connection(transport=Transport) channel = connection.channel() producer = Producer(channel, self.exchange, routing_key='test_Redis') consumer = Consumer(channel, queues=[self.queue]) producer.publish({'hello2': 'world2'}) _received = [] def callback(message_data, message): _received.append(message_data) message.ack() consumer.register_callback(callback) consumer.consume() self.assertIn(channel, channel.connection.cycle._channels) try: connection.drain_events(timeout=1) self.assertTrue(_received) with self.assertRaises(socket.timeout): connection.drain_events(timeout=0.01) finally: channel.close()
class test_Channel(Case): def setUp(self): self.connection = Connection(transport=Transport) self.channel = self.connection.channel() def test_disable_ack_emulation(self): conn = Connection(transport=Transport, transport_options={ 'ack_emulation': False, }) chan = conn.channel() self.assertFalse(chan.ack_emulation) self.assertEqual(chan.QoS, virtual.QoS) def test_redis_info_raises(self): pool = Mock(name='pool') pool_at_init = [pool] client = Mock(name='client') class XChannel(Channel): def __init__(self, *args, **kwargs): self._pool = pool_at_init[0] super(XChannel, self).__init__(*args, **kwargs) def _get_client(self): return lambda *_, **__: client class XTransport(Transport): Channel = XChannel conn = Connection(transport=XTransport) client.info.side_effect = RuntimeError() with self.assertRaises(RuntimeError): conn.channel() pool.disconnect.assert_called_with() pool.disconnect.reset_mock() pool_at_init = [None] with self.assertRaises(RuntimeError): conn.channel() self.assertFalse(pool.disconnect.called) def test_after_fork(self): self.channel._pool = None self.channel._after_fork() self.channel._pool = Mock(name='pool') self.channel._after_fork() self.channel._pool.disconnect.assert_called_with() def test_next_delivery_tag(self): self.assertNotEqual( self.channel._next_delivery_tag(), self.channel._next_delivery_tag(), ) def test_do_restore_message(self): client = Mock(name='client') pl1 = {'body': 'BODY'} spl1 = dumps(pl1) lookup = self.channel._lookup = Mock(name='_lookup') lookup.return_value = ['george', 'elaine'] self.channel._do_restore_message( pl1, 'ex', 'rkey', client, ) client.rpush.assert_has_calls([ call('george', spl1), call('elaine', spl1), ]) pl2 = {'body': 'BODY2', 'headers': {'x-funny': 1}} headers_after = dict(pl2['headers'], redelivered=True) spl2 = dumps(dict(pl2, headers=headers_after)) self.channel._do_restore_message( pl2, 'ex', 'rkey', client, ) client.rpush.assert_has_calls([ call('george', spl2), call('elaine', spl2), ]) client.rpush.side_effect = KeyError() with patch('kombu.transport.redis.logger') as logger: self.channel._do_restore_message( pl2, 'ex', 'rkey', client, ) self.assertTrue(logger.critical.called) def test_restore(self): message = Mock(name='message') with patch('kombu.transport.redis.loads') as loads: loads.return_value = 'M', 'EX', 'RK' client = self.channel.client = Mock(name='client') restore = self.channel._do_restore_message = Mock( name='_do_restore_message', ) pipe = Mock(name='pipe') client.pipeline.return_value = pipe pipe_hget = Mock(name='pipe.hget') pipe.hget.return_value = pipe_hget pipe_hget_hdel = Mock(name='pipe.hget.hdel') pipe_hget.hdel.return_value = pipe_hget_hdel result = Mock(name='result') pipe_hget_hdel.execute.return_value = None, None self.channel._restore(message) client.pipeline.assert_called_with() unacked_key = self.channel.unacked_key self.assertFalse(loads.called) tag = message.delivery_tag pipe.hget.assert_called_with(unacked_key, tag) pipe_hget.hdel.assert_called_with(unacked_key, tag) pipe_hget_hdel.execute.assert_called_with() pipe_hget_hdel.execute.return_value = result, None self.channel._restore(message) loads.assert_called_with(result) restore.assert_called_with('M', 'EX', 'RK', client, False) def test_qos_restore_visible(self): client = self.channel.client = Mock(name='client') client.zrevrangebyscore.return_value = [ (1, 10), (2, 20), (3, 30), ] qos = redis.QoS(self.channel) restore = qos.restore_by_tag = Mock(name='restore_by_tag') qos._vrestore_count = 1 qos.restore_visible() self.assertFalse(client.zrevrangebyscore.called) self.assertEqual(qos._vrestore_count, 2) qos._vrestore_count = 0 qos.restore_visible() restore.assert_has_calls([ call(1, client), call(2, client), call(3, client), ]) self.assertEqual(qos._vrestore_count, 1) qos._vrestore_count = 0 restore.reset_mock() client.zrevrangebyscore.return_value = [] qos.restore_visible() self.assertFalse(restore.called) self.assertEqual(qos._vrestore_count, 1) qos._vrestore_count = 0 client.setnx.side_effect = redis.MutexHeld() qos.restore_visible() def test_basic_consume_when_fanout_queue(self): self.channel.exchange_declare(exchange='txconfan', type='fanout') self.channel.queue_declare(queue='txconfanq') self.channel.queue_bind(queue='txconfanq', exchange='txconfan') self.assertIn('txconfanq', self.channel._fanout_queues) self.channel.basic_consume('txconfanq', False, None, 1) self.assertIn('txconfanq', self.channel.active_fanout_queues) self.assertEqual(self.channel._fanout_to_queue.get('txconfan'), 'txconfanq') def test_basic_cancel_unknown_delivery_tag(self): self.assertIsNone(self.channel.basic_cancel('txaseqwewq')) def test_subscribe_no_queues(self): self.channel.subclient = Mock() self.channel.active_fanout_queues.clear() self.channel._subscribe() self.assertFalse(self.channel.subclient.subscribe.called) def test_subscribe(self): self.channel.subclient = Mock() self.channel.active_fanout_queues.add('a') self.channel.active_fanout_queues.add('b') self.channel._fanout_queues.update(a='a', b='b') self.channel._subscribe() self.assertTrue(self.channel.subclient.subscribe.called) s_args, _ = self.channel.subclient.subscribe.call_args self.assertItemsEqual(s_args[0], ['a', 'b']) self.channel.subclient.connection._sock = None self.channel._subscribe() self.channel.subclient.connection.connect.assert_called_with() def test_handle_unsubscribe_message(self): s = self.channel.subclient s.subscribed = True self.channel._handle_message(s, ['unsubscribe', 'a', 0]) self.assertFalse(s.subscribed) def test_handle_pmessage_message(self): self.assertDictEqual( self.channel._handle_message( self.channel.subclient, ['pmessage', 'pattern', 'channel', 'data'], ), { 'type': 'pmessage', 'pattern': 'pattern', 'channel': 'channel', 'data': 'data', }, ) def test_handle_message(self): self.assertDictEqual( self.channel._handle_message( self.channel.subclient, ['type', 'channel', 'data'], ), { 'type': 'type', 'pattern': None, 'channel': 'channel', 'data': 'data', }, ) def test_brpop_start_but_no_queues(self): self.assertIsNone(self.channel._brpop_start()) def test_receive(self): s = self.channel.subclient = Mock() self.channel._fanout_to_queue['a'] = 'b' s.parse_response.return_value = [ 'message', 'a', dumps({'hello': 'world'}) ] payload, queue = self.channel._receive() self.assertDictEqual(payload, {'hello': 'world'}) self.assertEqual(queue, 'b') def test_receive_raises(self): self.channel._in_listen = True s = self.channel.subclient = Mock() s.parse_response.side_effect = KeyError('foo') with self.assertRaises(redis.Empty): self.channel._receive() self.assertFalse(self.channel._in_listen) def test_receive_empty(self): s = self.channel.subclient = Mock() s.parse_response.return_value = None with self.assertRaises(redis.Empty): self.channel._receive() def test_receive_different_message_Type(self): s = self.channel.subclient = Mock() s.parse_response.return_value = ['pmessage', '/foo/', 0, 'data'] with self.assertRaises(redis.Empty): self.channel._receive() def test_brpop_read_raises(self): c = self.channel.client = Mock() c.parse_response.side_effect = KeyError('foo') with self.assertRaises(redis.Empty): self.channel._brpop_read() c.connection.disconnect.assert_called_with() def test_brpop_read_gives_None(self): c = self.channel.client = Mock() c.parse_response.return_value = None with self.assertRaises(redis.Empty): self.channel._brpop_read() def test_poll_error(self): c = self.channel.client = Mock() c.parse_response = Mock() self.channel._poll_error('BRPOP') c.parse_response.assert_called_with('BRPOP') c.parse_response.side_effect = KeyError('foo') self.assertIsNone(self.channel._poll_error('BRPOP')) def test_put_fanout(self): self.channel._in_poll = False c = self.channel.client = Mock() body = {'hello': 'world'} self.channel._put_fanout('exchange', body) c.publish.assert_called_with('exchange', dumps(body)) def test_put_priority(self): client = self.channel.client = Mock(name='client') msg1 = {'properties': {'delivery_info': {'priority': 3}}} self.channel._put('george', msg1) client.lpush.assert_called_with( self.channel._q_for_pri('george', 3), dumps(msg1), ) msg2 = {'properties': {'delivery_info': {'priority': 313}}} self.channel._put('george', msg2) client.lpush.assert_called_with( self.channel._q_for_pri('george', 9), dumps(msg2), ) msg3 = {'properties': {'delivery_info': {}}} self.channel._put('george', msg3) client.lpush.assert_called_with( self.channel._q_for_pri('george', 0), dumps(msg3), ) def test_delete(self): x = self.channel self.channel._in_poll = False delete = x.client.delete = Mock() srem = x.client.srem = Mock() x._delete('queue', 'exchange', 'routing_key', None) delete.assert_has_call('queue') srem.assert_has_call(x.keyprefix_queue % ('exchange', ), x.sep.join(['routing_key', '', 'queue'])) def test_has_queue(self): self.channel._in_poll = False exists = self.channel.client.exists = Mock() exists.return_value = True self.assertTrue(self.channel._has_queue('foo')) exists.assert_has_call('foo') exists.return_value = False self.assertFalse(self.channel._has_queue('foo')) def test_close_when_closed(self): self.channel.closed = True self.channel.close() def test_close_deletes_autodelete_fanout_queues(self): self.channel._fanout_queues = ['foo', 'bar'] self.channel.auto_delete_queues = ['foo'] self.channel.queue_delete = Mock(name='queue_delete') self.channel.close() self.channel.queue_delete.assert_has_calls([call('foo')]) def test_close_client_close_raises(self): c = self.channel.client = Mock() c.connection.disconnect.side_effect = self.channel.ResponseError() self.channel.close() c.connection.disconnect.assert_called_with() def test_invalid_database_raises_ValueError(self): with self.assertRaises(ValueError): self.channel.connection.client.virtual_host = 'dwqeq' self.channel._connparams() def test_connparams_allows_slash_in_db(self): self.channel.connection.client.virtual_host = '/123' self.assertEqual(self.channel._connparams()['db'], 123) def test_connparams_db_can_be_int(self): self.channel.connection.client.virtual_host = 124 self.assertEqual(self.channel._connparams()['db'], 124) def test_new_queue_with_auto_delete(self): redis.Channel._new_queue(self.channel, 'george', auto_delete=False) self.assertNotIn('george', self.channel.auto_delete_queues) redis.Channel._new_queue(self.channel, 'elaine', auto_delete=True) self.assertIn('elaine', self.channel.auto_delete_queues) def test_connparams_regular_hostname(self): self.channel.connection.client.hostname = 'george.vandelay.com' self.assertEqual( self.channel._connparams()['host'], 'george.vandelay.com', ) def test_rotate_cycle_ValueError(self): cycle = self.channel._queue_cycle = ['kramer', 'jerry'] self.channel._rotate_cycle('kramer') self.assertEqual(cycle, ['jerry', 'kramer']) self.channel._rotate_cycle('elaine') @skip_if_not_module('redis') def test_get_client(self): import redis as R KombuRedis = redis.Channel._get_client(self.channel) self.assertTrue(KombuRedis) Rv = getattr(R, 'VERSION', None) try: R.VERSION = (2, 4, 0) with self.assertRaises(VersionMismatch): redis.Channel._get_client(self.channel) finally: if Rv is not None: R.VERSION = Rv @skip_if_not_module('redis') def test_get_response_error(self): from redis.exceptions import ResponseError self.assertIs(redis.Channel._get_response_error(self.channel), ResponseError) def test_avail_client_when_not_in_poll(self): self.channel._in_poll = False c = self.channel.client = Mock() with self.channel.conn_or_acquire() as client: self.assertIs(client, c) def test_avail_client_when_in_poll(self): self.channel._in_poll = True self.channel._pool = Mock() cc = self.channel._create_client = Mock() client = cc.return_value = Mock() with self.channel.conn_or_acquire(): pass self.channel.pool.release.assert_called_with(client.connection) cc.assert_called_with() def test_register_with_event_loop(self): transport = self.connection.transport transport.cycle = Mock(name='cycle') transport.cycle.fds = {12: 'LISTEN', 13: 'BRPOP'} conn = Mock(name='conn') loop = Mock(name='loop') redis.Transport.register_with_event_loop(transport, conn, loop) transport.cycle.on_poll_init.assert_called_with(loop.poller) loop.call_repeatedly.assert_called_with( 10, transport.cycle.maybe_restore_messages, ) self.assertTrue(loop.on_tick.add.called) on_poll_start = loop.on_tick.add.call_args[0][0] on_poll_start() transport.cycle.on_poll_start.assert_called_with() loop.add_reader.assert_has_calls([ call(12, transport.on_readable, 12), call(13, transport.on_readable, 13), ]) def test_transport_on_readable(self): transport = self.connection.transport cycle = transport.cycle = Mock(name='cyle') cycle.on_readable.return_value = None redis.Transport.on_readable(transport, 13) cycle.on_readable.assert_called_with(13) cycle.on_readable.reset_mock() queue = Mock(name='queue') ret = (Mock(name='message'), queue) cycle.on_readable.return_value = ret with self.assertRaises(KeyError): redis.Transport.on_readable(transport, 14) cb = transport._callbacks[queue] = Mock(name='callback') redis.Transport.on_readable(transport, 14) cb.assert_called_with(ret[0]) @skip_if_not_module('redis') def test_transport_get_errors(self): self.assertTrue(redis.Transport._get_errors(self.connection.transport)) @skip_if_not_module('redis') def test_transport_driver_version(self): self.assertTrue( redis.Transport.driver_version(self.connection.transport), ) @skip_if_not_module('redis') def test_transport_get_errors_when_InvalidData_used(self): from redis import exceptions class ID(Exception): pass DataError = getattr(exceptions, 'DataError', None) InvalidData = getattr(exceptions, 'InvalidData', None) exceptions.InvalidData = ID exceptions.DataError = None try: errors = redis.Transport._get_errors(self.connection.transport) self.assertTrue(errors) self.assertIn(ID, errors[1]) finally: if DataError is not None: exceptions.DataError = DataError if InvalidData is not None: exceptions.InvalidData = InvalidData def test_empty_queues_key(self): channel = self.channel channel._in_poll = False key = channel.keyprefix_queue % 'celery' # Everything is fine, there is a list of queues. channel.client.sadd(key, 'celery\x06\x16\x06\x16celery') self.assertListEqual(channel.get_table('celery'), [('celery', '', 'celery')]) # ... then for some reason, the _kombu.binding.celery key gets lost channel.client.srem(key) # which raises a channel error so that the consumer/publisher # can recover by redeclaring the required entities. with self.assertRaises(InconsistencyError): self.channel.get_table('celery') @skip_if_not_module('redis') def test_socket_connection(self): with patch('kombu.transport.redis.Channel._create_client'): with Connection('redis+socket:///tmp/redis.sock') as conn: connparams = conn.default_channel._connparams() self.assertEqual(connparams['connection_class'], redis.redis.UnixDomainSocketConnection) self.assertEqual(connparams['path'], '/tmp/redis.sock')
from kombu import Connection, Exchange, Queue, Consumer ''' Sample Kombu consumer to drain messages from AMQP message bus ''' rabbit_url = "amqp://{rabbit_server_ip}:5672/" conn = Connection(rabbit_url) channel = conn.channel() exchange = Exchange('data_exchange', type="direct") queue = Queue(name="data_q", exchange=exchange, routing_key="data_info") def process_message(body, message): print type(body), body print type(message), message.body print("The body is {}".format(body)) message.ack() with Consumer(conn, queues=queue, callbacks=[process_message], accept=["text/plain", "json"]): conn.drain_events(timeout=6000)
class test_FilesystemTransport(Case): def setup(self): try: data_folder_in = tempfile.mkdtemp() data_folder_out = tempfile.mkdtemp() except Exception: raise SkipTest('filesystem transport: cannot create tempfiles') self.c = Connection(transport='filesystem', transport_options={ 'data_folder_in': data_folder_in, 'data_folder_out': data_folder_out, }) self.p = Connection(transport='filesystem', transport_options={ 'data_folder_in': data_folder_out, 'data_folder_out': data_folder_in, }) self.e = Exchange('test_transport_filesystem') self.q = Queue('test_transport_filesystem', exchange=self.e, routing_key='test_transport_filesystem') self.q2 = Queue('test_transport_filesystem2', exchange=self.e, routing_key='test_transport_filesystem2') def test_produce_consume_noack(self): producer = Producer(self.p.channel(), self.e) consumer = Consumer(self.c.channel(), self.q, no_ack=True) for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') _received = [] def callback(message_data, message): _received.append(message) consumer.register_callback(callback) consumer.consume() while 1: if len(_received) == 10: break self.c.drain_events() self.assertEqual(len(_received), 10) def test_produce_consume(self): producer_channel = self.p.channel() consumer_channel = self.c.channel() producer = Producer(producer_channel, self.e) consumer1 = Consumer(consumer_channel, self.q) consumer2 = Consumer(consumer_channel, self.q2) self.q2(consumer_channel).declare() for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem2') _received1 = [] _received2 = [] def callback1(message_data, message): _received1.append(message) message.ack() def callback2(message_data, message): _received2.append(message) message.ack() consumer1.register_callback(callback1) consumer2.register_callback(callback2) consumer1.consume() consumer2.consume() while 1: if len(_received1) + len(_received2) == 20: break self.c.drain_events() self.assertEqual(len(_received1) + len(_received2), 20) # compression producer.publish({'compressed': True}, routing_key='test_transport_filesystem', compression='zlib') m = self.q(consumer_channel).get() self.assertDictEqual(m.payload, {'compressed': True}) # queue.delete for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') self.assertTrue(self.q(consumer_channel).get()) self.q(consumer_channel).delete() self.q(consumer_channel).declare() self.assertIsNone(self.q(consumer_channel).get()) # queue.purge for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem2') self.assertTrue(self.q2(consumer_channel).get()) self.q2(consumer_channel).purge() self.assertIsNone(self.q2(consumer_channel).get())
class test_Redis: def setup(self): self.connection = Connection(transport=Transport) self.exchange = Exchange('test_Redis', type='direct') self.queue = Queue('test_Redis', self.exchange, 'test_Redis') def teardown(self): self.connection.close() @mock.replace_module_value(redis.redis, 'VERSION', [3, 0, 0]) def test_publish__get_redispyv3(self): channel = self.connection.channel() producer = Producer(channel, self.exchange, routing_key='test_Redis') self.queue(channel).declare() producer.publish({'hello': 'world'}) assert self.queue(channel).get().payload == {'hello': 'world'} assert self.queue(channel).get() is None assert self.queue(channel).get() is None assert self.queue(channel).get() is None @mock.replace_module_value(redis.redis, 'VERSION', [2, 5, 10]) def test_publish__get_redispyv2(self): channel = self.connection.channel() producer = Producer(channel, self.exchange, routing_key='test_Redis') self.queue(channel).declare() producer.publish({'hello': 'world'}) assert self.queue(channel).get().payload == {'hello': 'world'} assert self.queue(channel).get() is None assert self.queue(channel).get() is None assert self.queue(channel).get() is None def test_publish__consume(self): connection = Connection(transport=Transport) channel = connection.channel() producer = Producer(channel, self.exchange, routing_key='test_Redis') consumer = Consumer(channel, queues=[self.queue]) producer.publish({'hello2': 'world2'}) _received = [] def callback(message_data, message): _received.append(message_data) message.ack() consumer.register_callback(callback) consumer.consume() assert channel in channel.connection.cycle._channels try: connection.drain_events(timeout=1) assert _received with pytest.raises(socket.timeout): connection.drain_events(timeout=0.01) finally: channel.close() def test_purge(self): channel = self.connection.channel() producer = Producer(channel, self.exchange, routing_key='test_Redis') self.queue(channel).declare() for i in range(10): producer.publish({'hello': 'world-%s' % (i, )}) assert channel._size('test_Redis') == 10 assert self.queue(channel).purge() == 10 channel.close() def test_db_values(self): Connection(virtual_host=1, transport=Transport).channel() Connection(virtual_host='1', transport=Transport).channel() Connection(virtual_host='/1', transport=Transport).channel() with pytest.raises(Exception): Connection('redis:///foo').channel() def test_db_port(self): c1 = Connection(port=None, transport=Transport).channel() c1.close() c2 = Connection(port=9999, transport=Transport).channel() c2.close() def test_close_poller_not_active(self): c = Connection(transport=Transport).channel() cycle = c.connection.cycle c.client.connection c.close() assert c not in cycle._channels def test_close_ResponseError(self): c = Connection(transport=Transport).channel() c.client.bgsave_raises_ResponseError = True c.close() def test_close_disconnects(self): c = Connection(transport=Transport).channel() conn1 = c.client.connection conn2 = c.subclient.connection c.close() assert conn1.disconnected assert conn2.disconnected def test_get__Empty(self): channel = self.connection.channel() with pytest.raises(Empty): channel._get('does-not-exist') channel.close() def test_get_client(self): with mock.module_exists(*_redis_modules()): conn = Connection(transport=Transport) chan = conn.channel() assert chan.Client assert chan.ResponseError assert conn.transport.connection_errors assert conn.transport.channel_errors def test_check_at_least_we_try_to_connect_and_fail(self): import redis connection = Connection('redis://localhost:65534/') with pytest.raises(redis.exceptions.ConnectionError): chan = connection.channel() chan._size('some_queue')
class test_Mailbox(TestCase): def _handler(self, state): return self.stats['var'] def setUp(self): class Mailbox(pidbox.Mailbox): def _collect(self, *args, **kwargs): return 'COLLECTED' self.mailbox = Mailbox('test_pidbox') self.connection = Connection(transport='memory') self.state = {'var': 1} self.handlers = {'mymethod': self._handler} self.bound = self.mailbox(self.connection) self.default_chan = self.connection.channel() self.node = self.bound.Node( 'test_pidbox', state=self.state, handlers=self.handlers, channel=self.default_chan, ) def test_reply__collect(self): mailbox = pidbox.Mailbox('test_reply__collect')(self.connection) exchange = mailbox.reply_exchange.name channel = self.connection.channel() mailbox.reply_queue(channel).declare() ticket = uuid() mailbox._publish_reply({'foo': 'bar'}, exchange, mailbox.oid, ticket) _callback_called = [False] def callback(body): _callback_called[0] = True reply = mailbox._collect(ticket, limit=1, callback=callback, channel=channel) self.assertEqual(reply, [{'foo': 'bar'}]) self.assertTrue(_callback_called[0]) ticket = uuid() mailbox._publish_reply({'biz': 'boz'}, exchange, mailbox.oid, ticket) reply = mailbox._collect(ticket, limit=1, channel=channel) self.assertEqual(reply, [{'biz': 'boz'}]) de = mailbox.connection.drain_events = Mock() de.side_effect = socket.timeout mailbox._collect(ticket, limit=1, channel=channel) def test_constructor(self): self.assertIsNone(self.mailbox.connection) self.assertTrue(self.mailbox.exchange.name) self.assertTrue(self.mailbox.reply_exchange.name) def test_bound(self): bound = self.mailbox(self.connection) self.assertIs(bound.connection, self.connection) def test_Node(self): self.assertTrue(self.node.hostname) self.assertTrue(self.node.state) self.assertIs(self.node.mailbox, self.bound) self.assertTrue(self.handlers) # No initial handlers node2 = self.bound.Node('test_pidbox2', state=self.state) self.assertDictEqual(node2.handlers, {}) def test_Node_consumer(self): consumer1 = self.node.Consumer() self.assertIs(consumer1.channel, self.default_chan) self.assertTrue(consumer1.no_ack) chan2 = self.connection.channel() consumer2 = self.node.Consumer(channel=chan2, no_ack=False) self.assertIs(consumer2.channel, chan2) self.assertFalse(consumer2.no_ack) def test_handler(self): node = self.bound.Node('test_handler', state=self.state) @node.handler def my_handler_name(state): return 42 self.assertIn('my_handler_name', node.handlers) def test_dispatch(self): node = self.bound.Node('test_dispatch', state=self.state) @node.handler def my_handler_name(state, x=None, y=None): return x + y self.assertEqual( node.dispatch('my_handler_name', arguments={ 'x': 10, 'y': 10 }), 20) def test_dispatch_raising_SystemExit(self): node = self.bound.Node('test_dispatch_raising_SystemExit', state=self.state) @node.handler def my_handler_name(state): raise SystemExit with self.assertRaises(SystemExit): node.dispatch('my_handler_name') def test_dispatch_raising(self): node = self.bound.Node('test_dispatch_raising', state=self.state) @node.handler def my_handler_name(state): raise KeyError('foo') res = node.dispatch('my_handler_name') self.assertIn('error', res) self.assertIn('KeyError', res['error']) def test_dispatch_replies(self): _replied = [False] def reply(data, **options): _replied[0] = True node = self.bound.Node('test_dispatch', state=self.state) node.reply = reply @node.handler def my_handler_name(state, x=None, y=None): return x + y node.dispatch('my_handler_name', arguments={ 'x': 10, 'y': 10 }, reply_to={ 'exchange': 'foo', 'routing_key': 'bar' }) self.assertTrue(_replied[0]) def test_reply(self): _replied = [(None, None, None)] def publish_reply(data, exchange, routing_key, ticket, **kwargs): _replied[0] = (data, exchange, routing_key, ticket) mailbox = self.mailbox(self.connection) mailbox._publish_reply = publish_reply node = mailbox.Node('test_reply') @node.handler def my_handler_name(state): return 42 node.dispatch('my_handler_name', reply_to={ 'exchange': 'exchange', 'routing_key': 'rkey' }, ticket='TICKET') data, exchange, routing_key, ticket = _replied[0] self.assertEqual(data, {'test_reply': 42}) self.assertEqual(exchange, 'exchange') self.assertEqual(routing_key, 'rkey') self.assertEqual(ticket, 'TICKET') def test_handle_message(self): node = self.bound.Node('test_dispatch_from_message') @node.handler def my_handler_name(state, x=None, y=None): return x * y body = {'method': 'my_handler_name', 'arguments': {'x': 64, 'y': 64}} self.assertEqual(node.handle_message(body, None), 64 * 64) # message not for me should not be processed. body['destination'] = ['some_other_node'] self.assertIsNone(node.handle_message(body, None)) def test_listen(self): consumer = self.node.listen() self.assertEqual(consumer.callbacks[0], self.node.handle_message) self.assertEqual(consumer.channel, self.default_chan) def test_cast(self): self.bound.cast(['somenode'], 'mymethod') consumer = self.node.Consumer() self.assertIsCast(self.get_next(consumer)) def test_abcast(self): self.bound.abcast('mymethod') consumer = self.node.Consumer() self.assertIsCast(self.get_next(consumer)) def test_call_destination_must_be_sequence(self): with self.assertRaises(ValueError): self.bound.call('some_node', 'mymethod') def test_call(self): self.assertEqual( self.bound.call(['some_node'], 'mymethod'), 'COLLECTED', ) consumer = self.node.Consumer() self.assertIsCall(self.get_next(consumer)) def test_multi_call(self): self.assertEqual(self.bound.multi_call('mymethod'), 'COLLECTED') consumer = self.node.Consumer() self.assertIsCall(self.get_next(consumer)) def get_next(self, consumer): m = consumer.queues[0].get() if m: return m.payload def assertIsCast(self, message): self.assertTrue(message['method']) def assertIsCall(self, message): self.assertTrue(message['method']) self.assertTrue(message['reply_to'])
class SimpleBase(Case): abstract = True def Queue(self, name, *args, **kwargs): q = name if not isinstance(q, Queue): q = self.__class__.__name__ if name: q = '%s.%s' % (q, name) return self._Queue(q, *args, **kwargs) def _Queue(self, *args, **kwargs): raise NotImplementedError() def setup(self): if not self.abstract: self.connection = Connection(transport='memory') with self.connection.channel() as channel: channel.exchange_declare('amq.direct') self.q = self.Queue(None, no_ack=True) def teardown(self): if not self.abstract: self.q.close() self.connection.close() def test_produce__consume(self): if self.abstract: return q = self.Queue('test_produce__consume', no_ack=True) q.put({'hello': 'Simple'}) self.assertEqual(q.get(timeout=1).payload, {'hello': 'Simple'}) with self.assertRaises(q.Empty): q.get(timeout=0.1) def test_produce__basic_get(self): if self.abstract: return q = self.Queue('test_produce__basic_get', no_ack=True) q.put({'hello': 'SimpleSync'}) self.assertEqual(q.get_nowait().payload, {'hello': 'SimpleSync'}) with self.assertRaises(q.Empty): q.get_nowait() q.put({'hello': 'SimpleSync'}) self.assertEqual(q.get(block=False).payload, {'hello': 'SimpleSync'}) with self.assertRaises(q.Empty): q.get(block=False) def test_clear(self): if self.abstract: return q = self.Queue('test_clear', no_ack=True) for i in range(10): q.put({'hello': 'SimplePurge%d' % (i,)}) self.assertEqual(q.clear(), 10) def test_enter_exit(self): if self.abstract: return q = self.Queue('test_enter_exit') q.close = Mock() self.assertIs(q.__enter__(), q) q.__exit__() q.close.assert_called_with() def test_qsize(self): if self.abstract: return q = self.Queue('test_clear', no_ack=True) for i in range(10): q.put({'hello': 'SimplePurge%d' % (i,)}) self.assertEqual(q.qsize(), 10) self.assertEqual(len(q), 10) def test_autoclose(self): if self.abstract: return channel = self.connection.channel() q = self.Queue('test_autoclose', no_ack=True, channel=channel) q.close() def test_custom_Queue(self): if self.abstract: return n = self.__class__.__name__ exchange = Exchange('%s-test.custom.Queue' % (n,)) queue = Queue('%s-test.custom.Queue' % (n,), exchange, 'my.routing.key') q = self.Queue(queue) self.assertEqual(q.consumer.queues[0], queue) q.close() def test_bool(self): if self.abstract: return q = self.Queue('test_nonzero') self.assertTrue(q)
class test_Producer(Case): def setUp(self): self.exchange = Exchange('foo', 'direct') self.connection = Connection(transport=Transport) self.connection.connect() self.assertTrue(self.connection.connection.connected) self.assertFalse(self.exchange.is_bound) def test_repr(self): p = Producer(self.connection) self.assertTrue(repr(p)) def test_pickle(self): chan = Mock() producer = Producer(chan, serializer='pickle') p2 = pickle.loads(pickle.dumps(producer)) self.assertEqual(p2.serializer, producer.serializer) def test_no_channel(self): p = Producer(None) self.assertFalse(p._channel) @patch('kombu.messaging.maybe_declare') def test_maybe_declare(self, maybe_declare): p = self.connection.Producer() q = Queue('foo') p.maybe_declare(q) maybe_declare.assert_called_with(q, p.channel, False) @patch('kombu.common.maybe_declare') def test_maybe_declare_when_entity_false(self, maybe_declare): p = self.connection.Producer() p.maybe_declare(None) self.assertFalse(maybe_declare.called) def test_auto_declare(self): channel = self.connection.channel() p = Producer(channel, self.exchange, auto_declare=True) self.assertIsNot(p.exchange, self.exchange, 'creates Exchange clone at bind') self.assertTrue(p.exchange.is_bound) self.assertIn('exchange_declare', channel, 'auto_declare declares exchange') def test_manual_declare(self): channel = self.connection.channel() p = Producer(channel, self.exchange, auto_declare=False) self.assertTrue(p.exchange.is_bound) self.assertNotIn('exchange_declare', channel, 'auto_declare=False does not declare exchange') p.declare() self.assertIn('exchange_declare', channel, 'p.declare() declares exchange') def test_prepare(self): message = {'the quick brown fox': 'jumps over the lazy dog'} channel = self.connection.channel() p = Producer(channel, self.exchange, serializer='json') m, ctype, cencoding = p._prepare(message, headers={}) self.assertDictEqual(message, anyjson.loads(m)) self.assertEqual(ctype, 'application/json') self.assertEqual(cencoding, 'utf-8') def test_prepare_compression(self): message = {'the quick brown fox': 'jumps over the lazy dog'} channel = self.connection.channel() p = Producer(channel, self.exchange, serializer='json') headers = {} m, ctype, cencoding = p._prepare(message, compression='zlib', headers=headers) self.assertEqual(ctype, 'application/json') self.assertEqual(cencoding, 'utf-8') self.assertEqual(headers['compression'], 'application/x-gzip') import zlib self.assertEqual( anyjson.loads(zlib.decompress(m).decode('utf-8')), message, ) def test_prepare_custom_content_type(self): message = 'the quick brown fox'.encode('utf-8') channel = self.connection.channel() p = Producer(channel, self.exchange, serializer='json') m, ctype, cencoding = p._prepare(message, content_type='custom') self.assertEqual(m, message) self.assertEqual(ctype, 'custom') self.assertEqual(cencoding, 'binary') m, ctype, cencoding = p._prepare(message, content_type='custom', content_encoding='alien') self.assertEqual(m, message) self.assertEqual(ctype, 'custom') self.assertEqual(cencoding, 'alien') def test_prepare_is_already_unicode(self): message = 'the quick brown fox' channel = self.connection.channel() p = Producer(channel, self.exchange, serializer='json') m, ctype, cencoding = p._prepare(message, content_type='text/plain') self.assertEqual(m, message.encode('utf-8')) self.assertEqual(ctype, 'text/plain') self.assertEqual(cencoding, 'utf-8') m, ctype, cencoding = p._prepare(message, content_type='text/plain', content_encoding='utf-8') self.assertEqual(m, message.encode('utf-8')) self.assertEqual(ctype, 'text/plain') self.assertEqual(cencoding, 'utf-8') def test_publish_with_Exchange_instance(self): p = self.connection.Producer() p.channel = Mock() p.publish('hello', exchange=Exchange('foo'), delivery_mode='transient') self.assertEqual( p._channel.basic_publish.call_args[1]['exchange'], 'foo', ) def test_set_on_return(self): chan = Mock() chan.events = defaultdict(Mock) p = Producer(ChannelPromise(lambda: chan), on_return='on_return') p.channel chan.events['basic_return'].add.assert_called_with('on_return') def test_publish_retry_calls_ensure(self): p = Producer(Mock()) p._connection = Mock() ensure = p.connection.ensure = Mock() p.publish('foo', exchange='foo', retry=True) self.assertTrue(ensure.called) def test_publish_retry_with_declare(self): p = self.connection.Producer() p.maybe_declare = Mock() p.connection.ensure = Mock() ex = Exchange('foo') p._publish('hello', 0, '', '', {}, {}, 'rk', 0, 0, ex, declare=[ex]) p.maybe_declare.assert_called_with(ex) def test_revive_when_channel_is_connection(self): p = self.connection.Producer() p.exchange = Mock() new_conn = Connection('memory://') defchan = new_conn.default_channel p.revive(new_conn) self.assertIs(p.channel, defchan) p.exchange.revive.assert_called_with(defchan) def test_enter_exit(self): p = self.connection.Producer() p.release = Mock() self.assertIs(p.__enter__(), p) p.__exit__() p.release.assert_called_with() def test_connection_property_handles_AttributeError(self): p = self.connection.Producer() p.channel = object() p.__connection__ = None self.assertIsNone(p.connection) def test_publish(self): channel = self.connection.channel() p = Producer(channel, self.exchange, serializer='json') message = {'the quick brown fox': 'jumps over the lazy dog'} ret = p.publish(message, routing_key='process') self.assertIn('prepare_message', channel) self.assertIn('basic_publish', channel) m, exc, rkey = ret self.assertDictEqual(message, anyjson.loads(m['body'])) self.assertDictContainsSubset( { 'content_type': 'application/json', 'content_encoding': 'utf-8', 'priority': 0 }, m) self.assertDictContainsSubset({'delivery_mode': 2}, m['properties']) self.assertEqual(exc, p.exchange.name) self.assertEqual(rkey, 'process') def test_no_exchange(self): chan = self.connection.channel() p = Producer(chan) self.assertFalse(p.exchange.name) def test_revive(self): chan = self.connection.channel() p = Producer(chan) chan2 = self.connection.channel() p.revive(chan2) self.assertIs(p.channel, chan2) self.assertIs(p.exchange.channel, chan2) def test_on_return(self): chan = self.connection.channel() def on_return(exception, exchange, routing_key, message): pass p = Producer(chan, on_return=on_return) self.assertTrue(on_return in chan.events['basic_return']) self.assertTrue(p.on_return)
class RpcClient(object): """Process a RPC queue and fetch the response.""" reply_received = False messages = {} corr_id_server_queue = {} def __init__(self, exchange=default_exchange, prefix=None, client_queue=None): """Constructor for client object. :exchange: Exchange to use :prefix: for automatic server routing key generation :client_queue: name for the client queue. Default is <command>.client """ self._prefix = prefix self._exchange = exchange self.reply = None self._client_queue = client_queue self._queue = None self._conn = Connection(**CONN_DICT) def retrieve_messages(self): """Process the message queue and ack the one that matches the correlation_id sent to the server. :correlation_id: Expected correlation id (uuid). :queue: The queue to read. :returns: JSON object retrieved from the queue. """ logger.debug("Client queue: {!r}".format(self._client_queue)) client_queue = self._queue logger.debug("connection is {!r}" "is connected: {!r}".format(self._conn, self._conn.connected)) for i in collect_replies(self._conn, self._conn.channel(), client_queue, timeout=1, limit=1, callbacks=[self.ack_message]): logger.debug("Received {!r}".format(i)) if self.reply is not None: response = self.reply self.reply = None try: client_queue.purge() client_queue.delete() self._conn.release() except Exception: logger.warn("Unable to purge and delete queues", exc_info=True) return response def ack_message(self, body, message): logger.info("Processing message: {!r} with body {!r}".format( message, body)) if 'correlation_id' in message.properties: corr_id = message.properties['correlation_id'] try: self.messages.pop(corr_id) server_queue = self.corr_id_server_queue.pop(corr_id) logger.info("STOPREQUEST:%s;CORRELATION_ID:%s" % (server_queue, corr_id)) self.reply = body message.ack() except KeyError as e: logger.exception(e) logger.error("Malformed message: {!r}".format(body)) def _setup_payload(self, command_name, data): """Setup the datastructure for either hase-like or standard. :command_name: the name of the command :data: data to be sent with the command :returns: payload for the request """ return data def _prepare_client_queue(self, command_name): """Setup a client queue based on the command. """ if self._client_queue is None: corr_uuid = uuid() self._client_queue = '.'.join([command_name, 'client', corr_uuid]) def rpc(self, command_name, data=None, server_routing_key=None): """Send a RPC request :command_name: the command to execute (used as routing key) :data: dict with data to be sent :client_queue: Queue for this particular request :server_routing_key: Server routing key. Will default to <command>.server """ self.reply_received = False payload = self._setup_payload(command_name, data) logger.info("Preparing request {!r}".format(payload)) if server_routing_key is None: if self._prefix is None: server_routing_key = command_name else: server_routing_key = '.'.join([self._prefix, command_name]) self._prepare_client_queue(command_name) logger.info("Set up client queue {!r} " "to {!r}".format(self._client_queue, server_routing_key)) message_correlation_id = uuid() properties = { 'reply_to': self._client_queue, 'correlation_id': message_correlation_id } self.corr_id_server_queue[message_correlation_id] = server_routing_key logger.info('STARTREQUEST:%s;CORRELATION_ID:%s' % (server_routing_key, message_correlation_id)) result = None try: self.messages[message_correlation_id] = True self._send_command(payload, server_routing_key, properties) result = self.retrieve_messages() except Exception: logger.error("Sending message to consumer queue failed.", exc_info=True) raise # Successful so store message correlation id for retrieval. return result def task(self, command_name, data=None, server_routing_key=None): """Send a RPC request :command_name: the command to execute (used as routing key) :data: dict with data to be sent :client_queue: Queue for this particular request :server_routing_key: Server routing key. Will default to <command>.server """ if data is None: data = {} self.reply_received = False payload = self._setup_payload(command_name, data) logger.debug("Preparing request {!r}".format(payload)) if server_routing_key is None: server_routing_key = command_name self._prepare_client_queue(command_name) logger.info("Set up client queue {!r} " "to {!r}".format(self._client_queue, server_routing_key)) self._send_command(payload, server_routing_key, declare_queue=False) def _send_command(self, payload, server_routing_key, properties=None, declare_queue=True): if properties is None: properties = {} self.reply = None logger.debug("Using connection: {!r}".format(CONN_DICT)) logger.info("Declaring queue %s." % self._client_queue) queue = Queue(self._client_queue, channel=self._conn, durable=self._exchange.durable, exchange=self._exchange, routing_key=self._client_queue) if declare_queue: queue.declare() self._queue = queue with producers[self._conn].acquire(block=True) as producer: producer.publish(payload, serializer='json', exchange=self._exchange, declare=[self._exchange], routing_key=server_routing_key, **properties) logger.info("Published {!r} to exchange " "{!r}".format(payload, self._exchange))
class SensorWatcher(ConsumerMixin): def __init__(self, create_handler, update_handler, delete_handler, queue_suffix=None): """ :param create_handler: Function which is called on SensorDB create event. :type create_handler: ``callable`` :param update_handler: Function which is called on SensorDB update event. :type update_handler: ``callable`` :param delete_handler: Function which is called on SensorDB delete event. :type delete_handler: ``callable`` """ # TODO: Handle sensor type filtering using routing key self._create_handler = create_handler self._update_handler = update_handler self._delete_handler = delete_handler self._sensor_watcher_q = self._get_queue(queue_suffix) self.connection = None self._updates_thread = None self._handlers = { publishers.CREATE_RK: create_handler, publishers.UPDATE_RK: update_handler, publishers.DELETE_RK: delete_handler } def get_consumers(self, Consumer, channel): consumers = [ Consumer(queues=[self._sensor_watcher_q], accept=['pickle'], callbacks=[self.process_task]) ] return consumers def process_task(self, body, message): LOG.debug('process_task') LOG.debug(' body: %s', body) LOG.debug(' message.properties: %s', message.properties) LOG.debug(' message.delivery_info: %s', message.delivery_info) routing_key = message.delivery_info.get('routing_key', '') handler = self._handlers.get(routing_key, None) try: if not handler: LOG.info('Skipping message %s as no handler was found.', message) return try: handler(body) except Exception as e: LOG.exception( 'Handling failed. Message body: %s. Exception: %s', body, e.message) finally: message.ack() def start(self): try: self.connection = Connection(transport_utils.get_messaging_urls()) self._updates_thread = eventlet.spawn(self.run) except: LOG.exception('Failed to start sensor_watcher.') self.connection.release() def stop(self): LOG.debug('Shutting down sensor watcher.') try: if self._updates_thread: self._updates_thread = eventlet.kill(self._updates_thread) if self.connection: channel = self.connection.channel() bound_sensor_watch_q = self._sensor_watcher_q(channel) try: bound_sensor_watch_q.delete() except: LOG.error('Unable to delete sensor watcher queue: %s', self._sensor_watcher_q) finally: if self.connection: self.connection.release() @staticmethod def _get_queue(queue_suffix): queue_name = queue_utils.get_queue_name( queue_name_base='st2.sensor.watch', queue_name_suffix=queue_suffix, add_random_uuid_to_suffix=True) return reactor.get_sensor_cud_queue(queue_name, routing_key='#')
class test_Producer: def setup(self): self.exchange = Exchange('foo', 'direct') self.connection = Connection(transport=Transport) self.connection.connect() assert self.connection.connection.connected assert not self.exchange.is_bound def test_repr(self): p = Producer(self.connection) assert repr(p) def test_pickle(self): chan = Mock() producer = Producer(chan, serializer='pickle') p2 = pickle.loads(pickle.dumps(producer)) assert p2.serializer == producer.serializer def test_no_channel(self): p = Producer(None) assert not p._channel @patch('kombu.messaging.maybe_declare') def test_maybe_declare(self, maybe_declare): p = self.connection.Producer() q = Queue('foo') p.maybe_declare(q) maybe_declare.assert_called_with(q, p.channel, False) @patch('kombu.common.maybe_declare') def test_maybe_declare_when_entity_false(self, maybe_declare): p = self.connection.Producer() p.maybe_declare(None) maybe_declare.assert_not_called() def test_auto_declare(self): channel = self.connection.channel() p = Producer(channel, self.exchange, auto_declare=True) # creates Exchange clone at bind assert p.exchange is not self.exchange assert p.exchange.is_bound # auto_declare declares exchange' assert 'exchange_declare' not in channel p.publish('foo') assert 'exchange_declare' in channel def test_manual_declare(self): channel = self.connection.channel() p = Producer(channel, self.exchange, auto_declare=False) assert p.exchange.is_bound # auto_declare=False does not declare exchange assert 'exchange_declare' not in channel # p.declare() declares exchange') p.declare() assert 'exchange_declare' in channel def test_prepare(self): message = {'the quick brown fox': 'jumps over the lazy dog'} channel = self.connection.channel() p = Producer(channel, self.exchange, serializer='json') m, ctype, cencoding = p._prepare(message, headers={}) assert json.loads(m) == message assert ctype == 'application/json' assert cencoding == 'utf-8' def test_prepare_compression(self): message = {'the quick brown fox': 'jumps over the lazy dog'} channel = self.connection.channel() p = Producer(channel, self.exchange, serializer='json') headers = {} m, ctype, cencoding = p._prepare(message, compression='zlib', headers=headers) assert ctype == 'application/json' assert cencoding == 'utf-8' assert headers['compression'] == 'application/x-gzip' import zlib assert json.loads(zlib.decompress(m).decode('utf-8')) == message def test_prepare_custom_content_type(self): message = 'the quick brown fox'.encode('utf-8') channel = self.connection.channel() p = Producer(channel, self.exchange, serializer='json') m, ctype, cencoding = p._prepare(message, content_type='custom') assert m == message assert ctype == 'custom' assert cencoding == 'binary' m, ctype, cencoding = p._prepare(message, content_type='custom', content_encoding='alien') assert m == message assert ctype == 'custom' assert cencoding == 'alien' def test_prepare_is_already_unicode(self): message = 'the quick brown fox' channel = self.connection.channel() p = Producer(channel, self.exchange, serializer='json') m, ctype, cencoding = p._prepare(message, content_type='text/plain') assert m == message.encode('utf-8') assert ctype == 'text/plain' assert cencoding == 'utf-8' m, ctype, cencoding = p._prepare(message, content_type='text/plain', content_encoding='utf-8') assert m == message.encode('utf-8') assert ctype == 'text/plain' assert cencoding == 'utf-8' def test_publish_with_Exchange_instance(self): p = self.connection.Producer() p.channel = Mock() p.channel.connection.client.declared_entities = set() p.publish('hello', exchange=Exchange('foo'), delivery_mode='transient') assert p._channel.basic_publish.call_args[1]['exchange'] == 'foo' def test_publish_with_expiration(self): p = self.connection.Producer() p.channel = Mock() p.channel.connection.client.declared_entities = set() p.publish('hello', exchange=Exchange('foo'), expiration=10) properties = p._channel.prepare_message.call_args[0][5] assert properties['expiration'] == '10000' def test_publish_with_reply_to(self): p = self.connection.Producer() p.channel = Mock() p.channel.connection.client.declared_entities = set() assert not p.exchange.name p.publish('hello', exchange=Exchange('foo'), reply_to=Queue('foo')) properties = p._channel.prepare_message.call_args[0][5] assert properties['reply_to'] == 'foo' def test_set_on_return(self): chan = Mock() chan.events = defaultdict(Mock) p = Producer(ChannelPromise(lambda: chan), on_return='on_return') p.channel chan.events['basic_return'].add.assert_called_with('on_return') def test_publish_retry_calls_ensure(self): p = Producer(Mock()) p._connection = Mock() p._connection.declared_entities = set() ensure = p.connection.ensure = Mock() p.publish('foo', exchange='foo', retry=True) ensure.assert_called() def test_publish_retry_with_declare(self): p = self.connection.Producer() p.maybe_declare = Mock() p.connection.ensure = Mock() ex = Exchange('foo') p._publish('hello', 0, '', '', {}, {}, 'rk', 0, 0, ex, declare=[ex]) p.maybe_declare.assert_called_with(ex) def test_revive_when_channel_is_connection(self): p = self.connection.Producer() p.exchange = Mock() new_conn = Connection('memory://') defchan = new_conn.default_channel p.revive(new_conn) assert p.channel is defchan p.exchange.revive.assert_called_with(defchan) def test_enter_exit(self): p = self.connection.Producer() p.release = Mock() assert p.__enter__() is p p.__exit__() p.release.assert_called_with() def test_connection_property_handles_AttributeError(self): p = self.connection.Producer() p.channel = object() p.__connection__ = None assert p.connection is None def test_publish(self): channel = self.connection.channel() p = Producer(channel, self.exchange, serializer='json') message = {'the quick brown fox': 'jumps over the lazy dog'} ret = p.publish(message, routing_key='process') assert 'prepare_message' in channel assert 'basic_publish' in channel m, exc, rkey = ret assert json.loads(m['body']) == message assert m['content_type'] == 'application/json' assert m['content_encoding'] == 'utf-8' assert m['priority'] == 0 assert m['properties']['delivery_mode'] == 2 assert exc == p.exchange.name assert rkey == 'process' def test_no_exchange(self): chan = self.connection.channel() p = Producer(chan) assert not p.exchange.name def test_revive(self): chan = self.connection.channel() p = Producer(chan) chan2 = self.connection.channel() p.revive(chan2) assert p.channel is chan2 assert p.exchange.channel is chan2 def test_on_return(self): chan = self.connection.channel() def on_return(exception, exchange, routing_key, message): pass p = Producer(chan, on_return=on_return) assert on_return in chan.events['basic_return'] assert p.on_return
class test_Consumer: def setup(self): self.connection = Connection(transport=Transport) self.connection.connect() assert self.connection.connection.connected self.exchange = Exchange('foo', 'direct') def test_accept(self): a = Consumer(self.connection) assert a.accept is None b = Consumer(self.connection, accept=['json', 'pickle']) assert b.accept == { 'application/json', 'application/x-python-serialize', } c = Consumer(self.connection, accept=b.accept) assert b.accept == c.accept def test_enter_exit_cancel_raises(self): c = Consumer(self.connection) c.cancel = Mock(name='Consumer.cancel') c.cancel.side_effect = KeyError('foo') with c: pass c.cancel.assert_called_with() def test_receive_callback_accept(self): message = Mock(name='Message') message.errors = [] callback = Mock(name='on_message') c = Consumer(self.connection, accept=['json'], on_message=callback) c.on_decode_error = None c.channel = Mock(name='channel') c.channel.message_to_python = None c._receive_callback(message) callback.assert_called_with(message) assert message.accept == c.accept def test_accept__content_disallowed(self): conn = Connection('memory://') q = Queue('foo', exchange=self.exchange) p = conn.Producer() p.publish( {'complex': object()}, declare=[q], exchange=self.exchange, serializer='pickle', ) callback = Mock(name='callback') with conn.Consumer(queues=[q], callbacks=[callback]) as consumer: with pytest.raises(consumer.ContentDisallowed): conn.drain_events(timeout=1) callback.assert_not_called() def test_accept__content_allowed(self): conn = Connection('memory://') q = Queue('foo', exchange=self.exchange) p = conn.Producer() p.publish( {'complex': object()}, declare=[q], exchange=self.exchange, serializer='pickle', ) callback = Mock(name='callback') with conn.Consumer(queues=[q], accept=['pickle'], callbacks=[callback]): conn.drain_events(timeout=1) callback.assert_called() body, message = callback.call_args[0] assert body['complex'] def test_set_no_channel(self): c = Consumer(None) assert c.channel is None c.revive(Mock()) assert c.channel def test_set_no_ack(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') consumer = Consumer(channel, queue, auto_declare=True, no_ack=True) assert consumer.no_ack def test_add_queue_when_auto_declare(self): consumer = self.connection.Consumer(auto_declare=True) q = Mock() q.return_value = q consumer.add_queue(q) assert q in consumer.queues q.declare.assert_called_with() def test_add_queue_when_not_auto_declare(self): consumer = self.connection.Consumer(auto_declare=False) q = Mock() q.return_value = q consumer.add_queue(q) assert q in consumer.queues assert not q.declare.call_count def test_consume_without_queues_returns(self): consumer = self.connection.Consumer() consumer.queues[:] = [] assert consumer.consume() is None def test_consuming_from(self): consumer = self.connection.Consumer() consumer.queues[:] = [Queue('a'), Queue('b'), Queue('d')] consumer._active_tags = {'a': 1, 'b': 2} assert not consumer.consuming_from(Queue('c')) assert not consumer.consuming_from('c') assert not consumer.consuming_from(Queue('d')) assert not consumer.consuming_from('d') assert consumer.consuming_from(Queue('a')) assert consumer.consuming_from(Queue('b')) assert consumer.consuming_from('b') def test_receive_callback_without_m2p(self): channel = self.connection.channel() c = channel.Consumer() m2p = getattr(channel, 'message_to_python') channel.message_to_python = None try: message = Mock() message.errors = [] message.decode.return_value = 'Hello' recv = c.receive = Mock() c._receive_callback(message) recv.assert_called_with('Hello', message) finally: channel.message_to_python = m2p def test_receive_callback__message_errors(self): channel = self.connection.channel() channel.message_to_python = None c = channel.Consumer() message = Mock() try: raise KeyError('foo') except KeyError: message.errors = [sys.exc_info()] message._reraise_error.side_effect = KeyError() with pytest.raises(KeyError): c._receive_callback(message) def test_set_callbacks(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') callbacks = [lambda x, y: x, lambda x, y: x] consumer = Consumer(channel, queue, auto_declare=True, callbacks=callbacks) assert consumer.callbacks == callbacks def test_auto_declare(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') consumer = Consumer(channel, queue, auto_declare=True) consumer.consume() consumer.consume() # twice is a noop assert consumer.queues[0] is not queue assert consumer.queues[0].is_bound assert consumer.queues[0].exchange.is_bound assert consumer.queues[0].exchange is not self.exchange for meth in ('exchange_declare', 'queue_declare', 'queue_bind', 'basic_consume'): assert meth in channel assert channel.called.count('basic_consume') == 1 assert consumer._active_tags consumer.cancel_by_queue(queue.name) consumer.cancel_by_queue(queue.name) assert not consumer._active_tags def test_consumer_tag_prefix(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') consumer = Consumer(channel, queue, tag_prefix='consumer_') consumer.consume() assert consumer._active_tags[queue.name].startswith('consumer_') def test_manual_declare(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') consumer = Consumer(channel, queue, auto_declare=False) assert consumer.queues[0] is not queue assert consumer.queues[0].is_bound assert consumer.queues[0].exchange.is_bound assert consumer.queues[0].exchange is not self.exchange for meth in ('exchange_declare', 'queue_declare', 'basic_consume'): assert meth not in channel consumer.declare() for meth in ('exchange_declare', 'queue_declare', 'queue_bind'): assert meth in channel assert 'basic_consume' not in channel consumer.consume() assert 'basic_consume' in channel def test_consume__cancel(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') consumer = Consumer(channel, queue, auto_declare=True) consumer.consume() consumer.cancel() assert 'basic_cancel' in channel assert not consumer._active_tags def test___enter____exit__(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') consumer = Consumer(channel, queue, auto_declare=True) context = consumer.__enter__() assert context is consumer assert consumer._active_tags res = consumer.__exit__(None, None, None) assert not res assert 'basic_cancel' in channel assert not consumer._active_tags def test_flow(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') consumer = Consumer(channel, queue, auto_declare=True) consumer.flow(False) assert 'flow' in channel def test_qos(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') consumer = Consumer(channel, queue, auto_declare=True) consumer.qos(30, 10, False) assert 'basic_qos' in channel def test_purge(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') b2 = Queue('qname2', self.exchange, 'rkey') b3 = Queue('qname3', self.exchange, 'rkey') b4 = Queue('qname4', self.exchange, 'rkey') consumer = Consumer(channel, [b1, b2, b3, b4], auto_declare=True) consumer.purge() assert channel.called.count('queue_purge') == 4 def test_multiple_queues(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') b2 = Queue('qname2', self.exchange, 'rkey') b3 = Queue('qname3', self.exchange, 'rkey') b4 = Queue('qname4', self.exchange, 'rkey') consumer = Consumer(channel, [b1, b2, b3, b4]) consumer.consume() assert channel.called.count('exchange_declare') == 4 assert channel.called.count('queue_declare') == 4 assert channel.called.count('queue_bind') == 4 assert channel.called.count('basic_consume') == 4 assert len(consumer._active_tags) == 4 consumer.cancel() assert channel.called.count('basic_cancel') == 4 assert not len(consumer._active_tags) def test_receive_callback(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) received = [] def callback(message_data, message): received.append(message_data) message.ack() message.payload # trigger cache consumer.register_callback(callback) consumer._receive_callback({'foo': 'bar'}) assert 'basic_ack' in channel assert 'message_to_python' in channel assert received[0] == {'foo': 'bar'} def test_basic_ack_twice(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) def callback(message_data, message): message.ack() message.ack() consumer.register_callback(callback) with pytest.raises(MessageStateError): consumer._receive_callback({'foo': 'bar'}) def test_basic_reject(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) def callback(message_data, message): message.reject() consumer.register_callback(callback) consumer._receive_callback({'foo': 'bar'}) assert 'basic_reject' in channel def test_basic_reject_twice(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) def callback(message_data, message): message.reject() message.reject() consumer.register_callback(callback) with pytest.raises(MessageStateError): consumer._receive_callback({'foo': 'bar'}) assert 'basic_reject' in channel def test_basic_reject__requeue(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) def callback(message_data, message): message.requeue() consumer.register_callback(callback) consumer._receive_callback({'foo': 'bar'}) assert 'basic_reject:requeue' in channel def test_basic_reject__requeue_twice(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) def callback(message_data, message): message.requeue() message.requeue() consumer.register_callback(callback) with pytest.raises(MessageStateError): consumer._receive_callback({'foo': 'bar'}) assert 'basic_reject:requeue' in channel def test_receive_without_callbacks_raises(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) with pytest.raises(NotImplementedError): consumer.receive(1, 2) def test_decode_error(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) consumer.channel.throw_decode_error = True with pytest.raises(ValueError): consumer._receive_callback({'foo': 'bar'}) def test_on_decode_error_callback(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') thrown = [] def on_decode_error(msg, exc): thrown.append((msg.body, exc)) consumer = Consumer(channel, [b1], on_decode_error=on_decode_error) consumer.channel.throw_decode_error = True consumer._receive_callback({'foo': 'bar'}) assert thrown m, exc = thrown[0] assert json.loads(m) == {'foo': 'bar'} assert isinstance(exc, ValueError) def test_recover(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) consumer.recover() assert 'basic_recover' in channel def test_revive(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) channel2 = self.connection.channel() consumer.revive(channel2) assert consumer.channel is channel2 assert consumer.queues[0].channel is channel2 assert consumer.queues[0].exchange.channel is channel2 def test_revive__with_prefetch_count(self): channel = Mock(name='channel') b1 = Queue('qname1', self.exchange, 'rkey') Consumer(channel, [b1], prefetch_count=14) channel.basic_qos.assert_called_with(0, 14, False) def test__repr__(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') assert repr(Consumer(channel, [b1])) def test_connection_property_handles_AttributeError(self): p = self.connection.Consumer() p.channel = object() assert p.connection is None
class test_Consumer(Case): def setup(self): self.connection = Connection(transport=Transport) @patch('kombu.compat._iterconsume') def test_iterconsume_calls__iterconsume(self, it, n='test_iterconsume'): c = compat.Consumer(self.connection, queue=n, exchange=n) c.iterconsume(limit=10, no_ack=True) it.assert_called_with(c.connection, c, True, 10) def test_constructor(self, n='test_Consumer_constructor'): c = compat.Consumer(self.connection, queue=n, exchange=n, routing_key='rkey') self.assertIsInstance(c.backend, Channel) q = c.queues[0] self.assertTrue(q.durable) self.assertTrue(q.exchange.durable) self.assertFalse(q.auto_delete) self.assertFalse(q.exchange.auto_delete) self.assertEqual(q.name, n) self.assertEqual(q.exchange.name, n) c2 = compat.Consumer(self.connection, queue=n + '2', exchange=n + '2', routing_key='rkey', durable=False, auto_delete=True, exclusive=True) q2 = c2.queues[0] self.assertFalse(q2.durable) self.assertFalse(q2.exchange.durable) self.assertTrue(q2.auto_delete) self.assertTrue(q2.exchange.auto_delete) def test__enter__exit__(self, n='test__enter__exit__'): c = compat.Consumer(self.connection, queue=n, exchange=n, routing_key='rkey') x = c.__enter__() self.assertIs(x, c) x.__exit__() self.assertTrue(c._closed) def test_revive(self, n='test_revive'): c = compat.Consumer(self.connection, queue=n, exchange=n) with self.connection.channel() as c2: c.revive(c2) self.assertIs(c.backend, c2) def test__iter__(self, n='test__iter__'): c = compat.Consumer(self.connection, queue=n, exchange=n) c.iterqueue = Mock() c.__iter__() c.iterqueue.assert_called_with(infinite=True) def test_iter(self, n='test_iterqueue'): c = compat.Consumer(self.connection, queue=n, exchange=n, routing_key='rkey') c.close() def test_process_next(self, n='test_process_next'): c = compat.Consumer(self.connection, queue=n, exchange=n, routing_key='rkey') with self.assertRaises(NotImplementedError): c.process_next() c.close() def test_iterconsume(self, n='test_iterconsume'): c = compat.Consumer(self.connection, queue=n, exchange=n, routing_key='rkey') c.close() def test_discard_all(self, n='test_discard_all'): c = compat.Consumer(self.connection, queue=n, exchange=n, routing_key='rkey') c.discard_all() self.assertIn('queue_purge', c.backend) def test_fetch(self, n='test_fetch'): c = compat.Consumer(self.connection, queue=n, exchange=n, routing_key='rkey') self.assertIsNone(c.fetch()) self.assertIsNone(c.fetch(no_ack=True)) self.assertIn('basic_get', c.backend) callback_called = [False] def receive(payload, message): callback_called[0] = True c.backend.to_deliver.append('42') payload = c.fetch().payload self.assertEqual(payload, '42') c.backend.to_deliver.append('46') c.register_callback(receive) self.assertEqual(c.fetch(enable_callbacks=True).payload, '46') self.assertTrue(callback_called[0]) def test_discard_all_filterfunc_not_supported(self, n='xjf21j21'): c = compat.Consumer(self.connection, queue=n, exchange=n, routing_key='rkey') with self.assertRaises(NotImplementedError): c.discard_all(filterfunc=lambda x: x) c.close() def test_wait(self, n='test_wait'): class C(compat.Consumer): def iterconsume(self, limit=None): for i in range(limit): yield i c = C(self.connection, queue=n, exchange=n, routing_key='rkey') self.assertEqual(c.wait(10), list(range(10))) c.close() def test_iterqueue(self, n='test_iterqueue'): i = [0] class C(compat.Consumer): def fetch(self, limit=None): z = i[0] i[0] += 1 return z c = C(self.connection, queue=n, exchange=n, routing_key='rkey') self.assertEqual(list(c.iterqueue(limit=10)), list(range(10))) c.close()
class test_FilesystemTransport: def setup(self): self.channels = set() try: data_folder_in = tempfile.mkdtemp() data_folder_out = tempfile.mkdtemp() except Exception: raise SkipTest('filesystem transport: cannot create tempfiles') self.c = Connection(transport='filesystem', transport_options={ 'data_folder_in': data_folder_in, 'data_folder_out': data_folder_out, }) self.channels.add(self.c.default_channel) self.p = Connection(transport='filesystem', transport_options={ 'data_folder_in': data_folder_out, 'data_folder_out': data_folder_in, }) self.channels.add(self.p.default_channel) self.e = Exchange('test_transport_filesystem') self.q = Queue('test_transport_filesystem', exchange=self.e, routing_key='test_transport_filesystem') self.q2 = Queue('test_transport_filesystem2', exchange=self.e, routing_key='test_transport_filesystem2') def teardown(self): # make sure we don't attempt to restore messages at shutdown. for channel in self.channels: try: channel._qos._dirty.clear() except AttributeError: pass try: channel._qos._delivered.clear() except AttributeError: pass def _add_channel(self, channel): self.channels.add(channel) return channel def test_produce_consume_noack(self): producer = Producer(self._add_channel(self.p.channel()), self.e) consumer = Consumer(self._add_channel(self.c.channel()), self.q, no_ack=True) for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') _received = [] def callback(message_data, message): _received.append(message) consumer.register_callback(callback) consumer.consume() while 1: if len(_received) == 10: break self.c.drain_events() assert len(_received) == 10 def test_produce_consume(self): producer_channel = self._add_channel(self.p.channel()) consumer_channel = self._add_channel(self.c.channel()) producer = Producer(producer_channel, self.e) consumer1 = Consumer(consumer_channel, self.q) consumer2 = Consumer(consumer_channel, self.q2) self.q2(consumer_channel).declare() for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem2') _received1 = [] _received2 = [] def callback1(message_data, message): _received1.append(message) message.ack() def callback2(message_data, message): _received2.append(message) message.ack() consumer1.register_callback(callback1) consumer2.register_callback(callback2) consumer1.consume() consumer2.consume() while 1: if len(_received1) + len(_received2) == 20: break self.c.drain_events() assert len(_received1) + len(_received2) == 20 # compression producer.publish({'compressed': True}, routing_key='test_transport_filesystem', compression='zlib') m = self.q(consumer_channel).get() assert m.payload == {'compressed': True} # queue.delete for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem') assert self.q(consumer_channel).get() self.q(consumer_channel).delete() self.q(consumer_channel).declare() assert self.q(consumer_channel).get() is None # queue.purge for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_filesystem2') assert self.q2(consumer_channel).get() self.q2(consumer_channel).purge() assert self.q2(consumer_channel).get() is None
class test_Message(Case): def setUp(self): self.conn = Connection('memory://') self.channel = self.conn.channel() self.message = Message(self.channel, delivery_tag=313) def test_postencode(self): with self.assertRaises(LookupError): Message(self.channel, text_t('FOO'), postencode='ccyzz') def test_ack_respects_no_ack_consumers(self): self.channel.no_ack_consumers = set(['abc']) self.message.delivery_info['consumer_tag'] = 'abc' ack = self.channel.basic_ack = Mock() self.message.ack() self.assertNotEqual(self.message._state, 'ACK') self.assertFalse(ack.called) def test_ack_missing_consumer_tag(self): self.channel.no_ack_consumers = set(['abc']) self.message.delivery_info = {} ack = self.channel.basic_ack = Mock() self.message.ack() ack.assert_called_with(self.message.delivery_tag) def test_ack_not_no_ack(self): self.channel.no_ack_consumers = set() self.message.delivery_info['consumer_tag'] = 'abc' ack = self.channel.basic_ack = Mock() self.message.ack() ack.assert_called_with(self.message.delivery_tag) def test_ack_log_error_when_no_error(self): ack = self.message.ack = Mock() self.message.ack_log_error(Mock(), KeyError) ack.assert_called_with() def test_ack_log_error_when_error(self): ack = self.message.ack = Mock() ack.side_effect = KeyError('foo') logger = Mock() self.message.ack_log_error(logger, KeyError) ack.assert_called_with() self.assertTrue(logger.critical.called) self.assertIn("Couldn't ack", logger.critical.call_args[0][0]) def test_reject_log_error_when_no_error(self): reject = self.message.reject = Mock() self.message.reject_log_error(Mock(), KeyError) reject.assert_called_with() def test_reject_log_error_when_error(self): reject = self.message.reject = Mock() reject.side_effect = KeyError('foo') logger = Mock() self.message.reject_log_error(logger, KeyError) reject.assert_called_with() self.assertTrue(logger.critical.called) self.assertIn("Couldn't ack", logger.critical.call_args[0][0])
class test_Redis(Case): def setUp(self): self.connection = Connection(transport=Transport) self.exchange = Exchange('test_Redis', type='direct') self.queue = Queue('test_Redis', self.exchange, 'test_Redis') def tearDown(self): self.connection.close() def test_publish__get(self): channel = self.connection.channel() producer = Producer(channel, self.exchange, routing_key='test_Redis') self.queue(channel).declare() producer.publish({'hello': 'world'}) self.assertDictEqual( self.queue(channel).get().payload, {'hello': 'world'}) self.assertIsNone(self.queue(channel).get()) self.assertIsNone(self.queue(channel).get()) self.assertIsNone(self.queue(channel).get()) def test_publish__consume(self): connection = Connection(transport=Transport) channel = connection.channel() producer = Producer(channel, self.exchange, routing_key='test_Redis') consumer = Consumer(channel, queues=[self.queue]) producer.publish({'hello2': 'world2'}) _received = [] def callback(message_data, message): _received.append(message_data) message.ack() consumer.register_callback(callback) consumer.consume() self.assertIn(channel, channel.connection.cycle._channels) try: connection.drain_events(timeout=1) self.assertTrue(_received) with self.assertRaises(socket.timeout): connection.drain_events(timeout=0.01) finally: channel.close() def test_purge(self): channel = self.connection.channel() producer = Producer(channel, self.exchange, routing_key='test_Redis') self.queue(channel).declare() for i in range(10): producer.publish({'hello': 'world-%s' % (i, )}) self.assertEqual(channel._size('test_Redis'), 10) self.assertEqual(self.queue(channel).purge(), 10) channel.close() def test_db_values(self): Connection(virtual_host=1, transport=Transport).channel() Connection(virtual_host='1', transport=Transport).channel() Connection(virtual_host='/1', transport=Transport).channel() with self.assertRaises(Exception): Connection('redis:///foo').channel() def test_db_port(self): c1 = Connection(port=None, transport=Transport).channel() c1.close() c2 = Connection(port=9999, transport=Transport).channel() c2.close() def test_close_poller_not_active(self): c = Connection(transport=Transport).channel() cycle = c.connection.cycle c.client.connection c.close() self.assertNotIn(c, cycle._channels) def test_close_ResponseError(self): c = Connection(transport=Transport).channel() c.client.bgsave_raises_ResponseError = True c.close() def test_close_disconnects(self): c = Connection(transport=Transport).channel() conn1 = c.client.connection conn2 = c.subclient.connection c.close() self.assertTrue(conn1.disconnected) self.assertTrue(conn2.disconnected) def test_get__Empty(self): channel = self.connection.channel() with self.assertRaises(Empty): channel._get('does-not-exist') channel.close() def test_get_client(self): myredis, exceptions = _redis_modules() @module_exists(myredis, exceptions) def _do_test(): conn = Connection(transport=Transport) chan = conn.channel() self.assertTrue(chan.Client) self.assertTrue(chan.ResponseError) self.assertTrue(conn.transport.connection_errors) self.assertTrue(conn.transport.channel_errors) _do_test()
class Amqp(Thread): def __init__(self, host="localhost", port=5672, userid="guest", password="******", virtual_host="canopsis", exchange_name="canopsis", logging_name="Amqp", logging_level=INFO, read_config_file=True, auto_connect=True, on_ready=None, max_retries=5): super(Amqp, self).__init__() self.logger = getLogger(logging_name) self.host = host self.port = port self.userid = userid self.password = password self.virtual_host = virtual_host self.exchange_name = exchange_name self.logging_level = logging_level if read_config_file: self.read_config("amqp") self.amqp_uri = "amqp://{0}:{1}@{2}:{3}/{4}".format( self.userid, self.password, self.host, self.port, self.virtual_host) self.logger.setLevel(logging_level) # Event sent try count before event drop in case of connection problem if max_retries != 5: self.logger.info('Custom retries value : {} {}'.format( max_retries, type(max_retries))) self.max_retries = max_retries self.exchange_name_events = exchange_name + ".events" self.exchange_name_alerts = exchange_name + ".alerts" self.exchange_name_incidents = exchange_name + ".incidents" self.chan = None self.conn = None self.connected = False self.on_ready = on_ready self.RUN = True self.exchanges = {} self.queues = {} self.producers = None self.paused = False self.connection_errors = (ConnectionError, error, IOError, OSError) # Create exchange self.logger.debug("Create exchanges object") for exchange_name in [ self.exchange_name, self.exchange_name_events, self.exchange_name_alerts, self.exchange_name_incidents ]: self.logger.debug(' + {}'.format(exchange_name)) self.get_exchange(exchange_name) if auto_connect: self.connect() self.logger.debug("Object canamqp initialized") def run(self): """ main thread loop - connects to rabbit - calls drain_events - disconnects on error """ self.logger.debug("Start thread ...") reconnect = False while self.RUN: self.connect() if self.connected: self.init_queue(reconnect=reconnect) self.logger.debug("Drain events ...") while self.RUN: try: if not self.paused: self.conn.drain_events(timeout=0.5) else: sleep(0.5) except KombuSerializationError as exc: self.logger.error( "Kombu serialization error: invalid message " "received: {}".format(exc)) except timeout: pass except self.connection_errors as err: self.logger.error( "Connection error ! ({})".format(err)) break except KombuSerializationError as exc: self.logger.error( "Kombu serialization error: invalid message " "received: {}".format(exc)) except Exception as err: self.logger.error("Error: {} ({})".format( err, type(err))) print_exc(file=stdout) break self.disconnect() if self.RUN: self.logger.error( 'Connection lost, try to reconnect in few seconds ...') reconnect = True self.wait_connection(timeout=5) self.logger.debug("End of thread ...") def stop(self): self.logger.debug("Stop thread ...") self.RUN = False def connect(self): """ Create the connection Inits the producers Init the only channel Declares exchanges """ if not self.connected: self.logger.info("Connect to AMQP Broker (%s:%s)" % (self.host, self.port)) self.conn = Connection(self.amqp_uri) try: self.logger.debug(" + Connect") self.conn.connect() self.logger.info("Connected to AMQP Broker.") self.producers = pools.Producers(limit=10) self.connected = True except Exception as err: self.conn.release() self.logger.error("Impossible to connect ({})".format(err)) if self.connected: self.logger.debug(" + Open channel") try: self.chan = self.conn.channel() self.logger.info("Channel openned. Ready to send messages") try: # Declare exchange self.logger.debug("Declare exchanges") for exchange_name in self.exchanges: self.logger.debug(" + {}".format(exchange_name)) self.exchanges[exchange_name](self.chan).declare() except Exception as err: self.logger.error( "Impossible to declare exchange ({})".format(err)) except Exception as err: self.logger.error(err) else: self.logger.debug("Already connected") def get_exchange(self, name): """ Returns an exchange if stored in self.exchanges. Otherwise, creates it and returns it :param string name: name of the exchange to get/create :rtype: Exchange|None """ if name: try: return self.exchanges[name] except Exception: if name == "amq.direct": self.exchanges[name] = Exchange(name, "direct", durable=True) else: self.exchanges[name] = Exchange(name, "topic", durable=True, auto_delete=False) return self.exchanges[name] else: return None def init_queue(self, reconnect=False): """ Loads queue settings Creates queues and attaches the same channel to each of them Binds queues to exchange Revives or creates consumers calls consume """ if self.queues: self.logger.debug("Init queues") for queue_name in self.queues.keys(): self.logger.debug(" + {}".format(queue_name)) qsettings = self.queues[queue_name] if not qsettings['queue']: self.logger.debug(" + Create queue") # copy list routing_keys = list(qsettings['routing_keys']) routing_key = None if len(routing_keys): routing_key = routing_keys[0] routing_keys = routing_keys[1:] exchange = self.get_exchange(qsettings['exchange_name']) if qsettings['exchange_name'] == "amq.direct" \ and not routing_key: routing_key = queue_name self.logger.debug("exchange: '{}', exclusive: {}," " auto_delete: {},no_ack: {}".format( qsettings['exchange_name'], qsettings['exclusive'], qsettings['auto_delete'], qsettings['no_ack'])) qsettings['queue'] = Queue( queue_name, exchange=exchange, routing_key=routing_key, exclusive=qsettings['exclusive'], auto_delete=qsettings['auto_delete'], no_ack=qsettings['no_ack'], channel=self.conn.channel()) qsettings['queue'].declare() if len(routing_keys): self.logger.debug(" + Bind on all routing keys") for routing_key in routing_keys: self.logger.debug( " + routing_key: '{}'".format(routing_key)) try: qsettings['queue'].bind_to( exchange=exchange, routing_key=routing_key) except Exception: self.logger.error( "You need upgrade your Kombu version ({})". format(__version__)) if qsettings['consumer'] and not reconnect: qsettings['consumer'].revive(self.chan) elif not qsettings['consumer'] or reconnect: self.logger.debug(" + Create Consumer") qsettings['consumer'] = self.conn.Consumer( qsettings['queue'], callbacks=[qsettings['callback']]) self.logger.debug(" + Consume queue") qsettings['consumer'].consume() if self.on_ready: self.on_ready() else: self.logger.info('Queue already inited') def add_queue(self, queue_name, routing_keys, callback, exchange_name=None, no_ack=True, exclusive=False, auto_delete=True): """ Initializes the queue configuration maps the callback on the queue """ c_routing_keys = [] if not isinstance(routing_keys, list): if isinstance(routing_keys, basestring): c_routing_keys = [routing_keys] else: c_routing_keys = routing_keys # aka if rk is nor a list nor a basetring, leave it empty if not exchange_name: exchange_name = self.exchange_name self.queues[queue_name] = { 'queue': False, 'consumer': False, 'queue_name': queue_name, 'routing_keys': c_routing_keys, 'callback': callback, 'exchange_name': exchange_name, 'no_ack': no_ack, 'exclusive': exclusive, 'auto_delete': auto_delete } def publish(self, msg, routing_key, exchange_name=None, serializer="json", compression=None, content_type=None, content_encoding=None): """ Tries to publish an event In case of failure, tries to reconnect and retry until (self.max_retries) :returns: operation success status :rtype: bool """ self.logger.warning("Publishing from old.rabbitmq.Amqp is deprecated") operation_success = False retries = 0 while not operation_success and retries < self.max_retries: retries += 1 if self.connected: if not exchange_name: exchange_name = self.exchange_name with self.producers[self.conn].acquire(block=True) as producer: try: _msg = msg.copy() Amqp._clean_msg_for_serialization(_msg) exchange = self.get_exchange( exchange_name.encode('utf-8')) producer.publish(_msg, serializer=serializer, compression=compression, routing_key=routing_key, exchange=exchange) self.logger.debug('publish {} in exchange {}'.format( routing_key, exchange_name)) operation_success = True except AmqpStructError: self.logger.warning( 'Malformed message: routing key is too long. ' 'Cancelling message') return False except Exception: self.logger.error(' + Impossible to send {}'.format( traceback.format_exc())) self.disconnect() self.connect() self.init_queue(reconnect=False) else: self.logger.error('Not connected ... try reconnecting') self.connect() if not operation_success: # Event and it's information are buffered until next send retry self.logger.info('Retry count {}'.format(retries)) if not operation_success: # Event and it's information are buffered until next send retry self.logger.error( 'Too much retries for event {}, give up'.format(routing_key)) return operation_success @staticmethod def _clean_msg_for_serialization(msg): from bson import objectid for key in msg: if isinstance(msg[key], objectid.ObjectId): msg[key] = str(msg[key]) def cancel_queues(self): if self.connected: for queue_name in self.queues.keys(): if self.queues[queue_name]['consumer']: self.logger.debug( " + Cancel consumer on {}".format(queue_name)) try: self.queues[queue_name]['consumer'].cancel() except Exception: pass del (self.queues[queue_name]['consumer']) self.queues[queue_name]['consumer'] = False del (self.queues[queue_name]['queue']) self.queues[queue_name]['queue'] = False def disconnect(self): if self.connected: self.logger.info("Disconnect from AMQP Broker") self.cancel_queues() # Force producers closing to permit a clean reconnect after # ... especially on timeout errors self.producers[self.conn].force_close_all() for exchange in self.exchanges: del exchange self.exchanges = {} try: pools.reset() except Exception as err: self.logger.error( "Impossible to reset kombu pools: {} ({})".format( err, type(err))) try: self.conn.release() del self.conn except Exception as err: self.logger.error( "Impossible to release connection: {} ({})".format( err, type(err))) self.connected = False def wait_connection(self, timeout=5): i = 0 while self.RUN and not self.connected and i < (timeout * 2): sleep(0.5) i += 1 def read_config(self, name): filename = join(root_path, 'etc', '{0}.conf'.format(name)) import ConfigParser self.config = ConfigParser.RawConfigParser() try: self.config.read(filename) section = 'master' self.host = self.config.get(section, "host") self.port = self.config.getint(section, "port") self.userid = self.config.get(section, "userid") self.password = self.config.get(section, "password") self.virtual_host = self.config.get(section, "virtual_host") self.exchange_name = self.config.get(section, "exchange_name") except Exception as err: self.logger.error( "Can't to load configurations ({}), use default ...".format( err)) def __del__(self): self.stop()
Queue_PassWord = "******" Queue_Server = '127.0.0.1' Queue_Port = 5672 Queue_Path = '/websocketserver' def callback(body, message): print body connection = Connection(hostname=Queue_Server, port=Queue_Port, userid=Queue_User, password=Queue_PassWord, virtual_host=Queue_Path) channel = connection.channel() smsExchange = Exchange("sys.sms", type='topic', channel=channel, durable=True, delivery_mode=2) task_queue = Queue('test_recv', exchange=smsExchange, routing_key='sms.code', durable=False, channel=channel) consumer = Consumer(channel, task_queue, no_ack=True, callbacks=[callback]) consumer.qos(prefetch_count=1) consumer.consume() while True: connection.drain_events()
class test_Consumer(Case): def setUp(self): self.connection = Connection(transport=Transport) self.connection.connect() self.assertTrue(self.connection.connection.connected) self.exchange = Exchange('foo', 'direct') def test_accept(self): a = Consumer(self.connection) self.assertIsNone(a.accept) b = Consumer(self.connection, accept=['json', 'pickle']) self.assertSetEqual( b.accept, set(['application/json', 'application/x-python-serialize']), ) c = Consumer(self.connection, accept=b.accept) self.assertSetEqual(b.accept, c.accept) def test_enter_exit_cancel_raises(self): c = Consumer(self.connection) c.cancel = Mock(name='Consumer.cancel') c.cancel.side_effect = KeyError('foo') with c: pass c.cancel.assert_called_with() def test_receive_callback_accept(self): message = Mock(name='Message') message.errors = [] callback = Mock(name='on_message') c = Consumer(self.connection, accept=['json'], on_message=callback) c.on_decode_error = None c.channel = Mock(name='channel') c.channel.message_to_python = None c._receive_callback(message) callback.assert_called_with(message) self.assertSetEqual(message.accept, c.accept) def test_accept__content_disallowed(self): conn = Connection('memory://') q = Queue('foo', exchange=self.exchange) p = conn.Producer() p.publish( {'complex': object()}, declare=[q], exchange=self.exchange, serializer='pickle', ) callback = Mock(name='callback') with conn.Consumer(queues=[q], callbacks=[callback]) as consumer: with self.assertRaises(consumer.ContentDisallowed): conn.drain_events(timeout=1) self.assertFalse(callback.called) def test_accept__content_allowed(self): conn = Connection('memory://') q = Queue('foo', exchange=self.exchange) p = conn.Producer() p.publish( {'complex': object()}, declare=[q], exchange=self.exchange, serializer='pickle', ) callback = Mock(name='callback') with conn.Consumer(queues=[q], accept=['pickle'], callbacks=[callback]): conn.drain_events(timeout=1) self.assertTrue(callback.called) body, message = callback.call_args[0] self.assertTrue(body['complex']) def test_set_no_channel(self): c = Consumer(None) self.assertIsNone(c.channel) c.revive(Mock()) self.assertTrue(c.channel) def test_set_no_ack(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') consumer = Consumer(channel, queue, auto_declare=True, no_ack=True) self.assertTrue(consumer.no_ack) def test_add_queue_when_auto_declare(self): consumer = self.connection.Consumer(auto_declare=True) q = Mock() q.return_value = q consumer.add_queue(q) self.assertIn(q, consumer.queues) q.declare.assert_called_with() def test_add_queue_when_not_auto_declare(self): consumer = self.connection.Consumer(auto_declare=False) q = Mock() q.return_value = q consumer.add_queue(q) self.assertIn(q, consumer.queues) self.assertFalse(q.declare.call_count) def test_consume_without_queues_returns(self): consumer = self.connection.Consumer() consumer.queues[:] = [] self.assertIsNone(consumer.consume()) def test_consuming_from(self): consumer = self.connection.Consumer() consumer.queues[:] = [Queue('a'), Queue('b'), Queue('d')] consumer._active_tags = {'a': 1, 'b': 2} self.assertFalse(consumer.consuming_from(Queue('c'))) self.assertFalse(consumer.consuming_from('c')) self.assertFalse(consumer.consuming_from(Queue('d'))) self.assertFalse(consumer.consuming_from('d')) self.assertTrue(consumer.consuming_from(Queue('a'))) self.assertTrue(consumer.consuming_from(Queue('b'))) self.assertTrue(consumer.consuming_from('b')) def test_receive_callback_without_m2p(self): channel = self.connection.channel() c = channel.Consumer() m2p = getattr(channel, 'message_to_python') channel.message_to_python = None try: message = Mock() message.errors = [] message.decode.return_value = 'Hello' recv = c.receive = Mock() c._receive_callback(message) recv.assert_called_with('Hello', message) finally: channel.message_to_python = m2p def test_set_callbacks(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') callbacks = [lambda x, y: x, lambda x, y: x] consumer = Consumer(channel, queue, auto_declare=True, callbacks=callbacks) self.assertEqual(consumer.callbacks, callbacks) def test_auto_declare(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') consumer = Consumer(channel, queue, auto_declare=True) consumer.consume() consumer.consume() # twice is a noop self.assertIsNot(consumer.queues[0], queue) self.assertTrue(consumer.queues[0].is_bound) self.assertTrue(consumer.queues[0].exchange.is_bound) self.assertIsNot(consumer.queues[0].exchange, self.exchange) for meth in ('exchange_declare', 'queue_declare', 'queue_bind', 'basic_consume'): self.assertIn(meth, channel) self.assertEqual(channel.called.count('basic_consume'), 1) self.assertTrue(consumer._active_tags) consumer.cancel_by_queue(queue.name) consumer.cancel_by_queue(queue.name) self.assertFalse(consumer._active_tags) def test_consumer_tag_prefix(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') consumer = Consumer(channel, queue, tag_prefix='consumer_') consumer.consume() self.assertTrue( consumer._active_tags[queue.name].startswith('consumer_'), ) def test_manual_declare(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') consumer = Consumer(channel, queue, auto_declare=False) self.assertIsNot(consumer.queues[0], queue) self.assertTrue(consumer.queues[0].is_bound) self.assertTrue(consumer.queues[0].exchange.is_bound) self.assertIsNot(consumer.queues[0].exchange, self.exchange) for meth in ('exchange_declare', 'queue_declare', 'basic_consume'): self.assertNotIn(meth, channel) consumer.declare() for meth in ('exchange_declare', 'queue_declare', 'queue_bind'): self.assertIn(meth, channel) self.assertNotIn('basic_consume', channel) consumer.consume() self.assertIn('basic_consume', channel) def test_consume__cancel(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') consumer = Consumer(channel, queue, auto_declare=True) consumer.consume() consumer.cancel() self.assertIn('basic_cancel', channel) self.assertFalse(consumer._active_tags) def test___enter____exit__(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') consumer = Consumer(channel, queue, auto_declare=True) context = consumer.__enter__() self.assertIs(context, consumer) self.assertTrue(consumer._active_tags) res = consumer.__exit__(None, None, None) self.assertFalse(res) self.assertIn('basic_cancel', channel) self.assertFalse(consumer._active_tags) def test_flow(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') consumer = Consumer(channel, queue, auto_declare=True) consumer.flow(False) self.assertIn('flow', channel) def test_qos(self): channel = self.connection.channel() queue = Queue('qname', self.exchange, 'rkey') consumer = Consumer(channel, queue, auto_declare=True) consumer.qos(30, 10, False) self.assertIn('basic_qos', channel) def test_purge(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') b2 = Queue('qname2', self.exchange, 'rkey') b3 = Queue('qname3', self.exchange, 'rkey') b4 = Queue('qname4', self.exchange, 'rkey') consumer = Consumer(channel, [b1, b2, b3, b4], auto_declare=True) consumer.purge() self.assertEqual(channel.called.count('queue_purge'), 4) def test_multiple_queues(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') b2 = Queue('qname2', self.exchange, 'rkey') b3 = Queue('qname3', self.exchange, 'rkey') b4 = Queue('qname4', self.exchange, 'rkey') consumer = Consumer(channel, [b1, b2, b3, b4]) consumer.consume() self.assertEqual(channel.called.count('exchange_declare'), 4) self.assertEqual(channel.called.count('queue_declare'), 4) self.assertEqual(channel.called.count('queue_bind'), 4) self.assertEqual(channel.called.count('basic_consume'), 4) self.assertEqual(len(consumer._active_tags), 4) consumer.cancel() self.assertEqual(channel.called.count('basic_cancel'), 4) self.assertFalse(len(consumer._active_tags)) def test_receive_callback(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) received = [] def callback(message_data, message): received.append(message_data) message.ack() message.payload # trigger cache consumer.register_callback(callback) consumer._receive_callback({'foo': 'bar'}) self.assertIn('basic_ack', channel) self.assertIn('message_to_python', channel) self.assertEqual(received[0], {'foo': 'bar'}) def test_basic_ack_twice(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) def callback(message_data, message): message.ack() message.ack() consumer.register_callback(callback) with self.assertRaises(MessageStateError): consumer._receive_callback({'foo': 'bar'}) def test_basic_reject(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) def callback(message_data, message): message.reject() consumer.register_callback(callback) consumer._receive_callback({'foo': 'bar'}) self.assertIn('basic_reject', channel) def test_basic_reject_twice(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) def callback(message_data, message): message.reject() message.reject() consumer.register_callback(callback) with self.assertRaises(MessageStateError): consumer._receive_callback({'foo': 'bar'}) self.assertIn('basic_reject', channel) def test_basic_reject__requeue(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) def callback(message_data, message): message.requeue() consumer.register_callback(callback) consumer._receive_callback({'foo': 'bar'}) self.assertIn('basic_reject:requeue', channel) def test_basic_reject__requeue_twice(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) def callback(message_data, message): message.requeue() message.requeue() consumer.register_callback(callback) with self.assertRaises(MessageStateError): consumer._receive_callback({'foo': 'bar'}) self.assertIn('basic_reject:requeue', channel) def test_receive_without_callbacks_raises(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) with self.assertRaises(NotImplementedError): consumer.receive(1, 2) def test_decode_error(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) consumer.channel.throw_decode_error = True with self.assertRaises(ValueError): consumer._receive_callback({'foo': 'bar'}) def test_on_decode_error_callback(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') thrown = [] def on_decode_error(msg, exc): thrown.append((msg.body, exc)) consumer = Consumer(channel, [b1], on_decode_error=on_decode_error) consumer.channel.throw_decode_error = True consumer._receive_callback({'foo': 'bar'}) self.assertTrue(thrown) m, exc = thrown[0] self.assertEqual(anyjson.loads(m), {'foo': 'bar'}) self.assertIsInstance(exc, ValueError) def test_recover(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) consumer.recover() self.assertIn('basic_recover', channel) def test_revive(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') consumer = Consumer(channel, [b1]) channel2 = self.connection.channel() consumer.revive(channel2) self.assertIs(consumer.channel, channel2) self.assertIs(consumer.queues[0].channel, channel2) self.assertIs(consumer.queues[0].exchange.channel, channel2) def test__repr__(self): channel = self.connection.channel() b1 = Queue('qname1', self.exchange, 'rkey') self.assertTrue(repr(Consumer(channel, [b1]))) def test_connection_property_handles_AttributeError(self): p = self.connection.Consumer() p.channel = object() self.assertIsNone(p.connection)
def _get_producer_kombu_objects(cls, url, exchange_name): connection = Connection(url) channel = connection.channel() return connection, channel
class test_MemoryTransport(TestCase): def setUp(self): self.c = Connection(transport='memory') self.e = Exchange('test_transport_memory') self.q = Queue('test_transport_memory', exchange=self.e, routing_key='test_transport_memory') self.q2 = Queue('test_transport_memory2', exchange=self.e, routing_key='test_transport_memory2') def test_produce_consume_noack(self): channel = self.c.channel() producer = Producer(channel, self.e) consumer = Consumer(channel, self.q, no_ack=True) for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory') _received = [] def callback(message_data, message): _received.append(message) consumer.register_callback(callback) consumer.consume() while 1: if len(_received) == 10: break self.c.drain_events() self.assertEqual(len(_received), 10) def test_produce_consume(self): channel = self.c.channel() producer = Producer(channel, self.e) consumer1 = Consumer(channel, self.q) consumer2 = Consumer(channel, self.q2) self.q2(channel).declare() for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory') for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory2') _received1 = [] _received2 = [] def callback1(message_data, message): _received1.append(message) message.ack() def callback2(message_data, message): _received2.append(message) message.ack() consumer1.register_callback(callback1) consumer2.register_callback(callback2) consumer1.consume() consumer2.consume() while 1: if len(_received1) + len(_received2) == 20: break self.c.drain_events() self.assertEqual(len(_received1) + len(_received2), 20) # compression producer.publish({'compressed': True}, routing_key='test_transport_memory', compression='zlib') m = self.q(channel).get() self.assertDictEqual(m.payload, {'compressed': True}) # queue.delete for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory') self.assertTrue(self.q(channel).get()) self.q(channel).delete() self.q(channel).declare() self.assertIsNone(self.q(channel).get()) # queue.purge for i in range(10): producer.publish({'foo': i}, routing_key='test_transport_memory2') self.assertTrue(self.q2(channel).get()) self.q2(channel).purge() self.assertIsNone(self.q2(channel).get()) def test_drain_events(self): with self.assertRaises(socket.timeout): self.c.drain_events(timeout=0.1) c1 = self.c.channel() c2 = self.c.channel() with self.assertRaises(socket.timeout): self.c.drain_events(timeout=0.1) del (c1) # so pyflakes doesn't complain. del (c2) def test_drain_events_unregistered_queue(self): c1 = self.c.channel() class Cycle(object): def get(self, timeout=None): return ('foo', 'foo'), c1 self.c.transport.cycle = Cycle() with self.assertRaises(KeyError): self.c.drain_events() def test_queue_for(self): chan = self.c.channel() chan.queues.clear() x = chan._queue_for('foo') self.assertTrue(x) self.assertIs(chan._queue_for('foo'), x)
class test_ConsumerSet(Case): def setup(self): self.connection = Connection(transport=Transport) def test_providing_channel(self): chan = Mock(name='channel') cs = compat.ConsumerSet(self.connection, channel=chan) self.assertTrue(cs._provided_channel) self.assertIs(cs.backend, chan) cs.cancel = Mock(name='cancel') cs.close() chan.close.assert_not_called() @patch('kombu.compat._iterconsume') def test_iterconsume(self, _iterconsume, n='test_iterconsume'): c = compat.Consumer(self.connection, queue=n, exchange=n) cs = compat.ConsumerSet(self.connection, consumers=[c]) cs.iterconsume(limit=10, no_ack=True) _iterconsume.assert_called_with(c.connection, cs, True, 10) def test_revive(self, n='test_revive'): c = compat.Consumer(self.connection, queue=n, exchange=n) cs = compat.ConsumerSet(self.connection, consumers=[c]) with self.connection.channel() as c2: cs.revive(c2) self.assertIs(cs.backend, c2) def test_constructor(self, prefix='0daf8h21'): dcon = { '%s.xyx' % prefix: { 'exchange': '%s.xyx' % prefix, 'routing_key': 'xyx' }, '%s.xyz' % prefix: { 'exchange': '%s.xyz' % prefix, 'routing_key': 'xyz' } } consumers = [ compat.Consumer(self.connection, queue=prefix + str(i), exchange=prefix + str(i)) for i in range(3) ] c = compat.ConsumerSet(self.connection, consumers=consumers) c2 = compat.ConsumerSet(self.connection, from_dict=dcon) self.assertEqual(len(c.queues), 3) self.assertEqual(len(c2.queues), 2) c.add_consumer( compat.Consumer(self.connection, queue=prefix + 'xaxxxa', exchange=prefix + 'xaxxxa')) self.assertEqual(len(c.queues), 4) for cq in c.queues: self.assertIs(cq.channel, c.channel) c2.add_consumer_from_dict( '%s.xxx' % prefix, exchange='%s.xxx' % prefix, routing_key='xxx', ) self.assertEqual(len(c2.queues), 3) for c2q in c2.queues: self.assertIs(c2q.channel, c2.channel) c.discard_all() self.assertEqual(c.channel.called.count('queue_purge'), 4) c.consume() c.close() c2.close() self.assertIn('basic_cancel', c.channel) self.assertIn('close', c.channel) self.assertIn('close', c2.channel)