class RabbitMQConnection:
def connection(self):
try:
logger.info("Connection to the AMQP broker '" + AMQP_IP + ":" + AMQP_PORT + "' ...")
connection = BrokerConnection(hostname=AMQP_IP,
port=AMQP_PORT,
userid=AMQP_USER,
password=AMQP_PASSWORD,
virtual_host="/",
transport=AMQP_TRANSPORT)
#channel = connection.channel()
logger.info("Connection to the AMQP broker established.")
return connection
except Exception as exep:
logger.warning("Could not connect to the AMQP broker '" + AMQP_IP + ":" + AMQP_PORT + "'. | " + str(exep))
def producer(self):
self.connection = BrokerConnection(hostname=AMQP_IP, port=AMQP_PORT,
userid=AMQP_USER, password=AMQP_PASSWORD,
virtual_host="/",
transport=AMQP_TRANSPORT)
self.channel = self.connection.channel()
# produce
self.media_exchange = Exchange("media", "direct", durable=True)
producer = Producer(self.channel, exchange=self.media_exchange, serializer="json")
producer.publish({"name": "/tmp/lolcat1.avi", "size": 1301013})
print self.connection.get_transport_cls()
def consumer(self):
self.connection = BrokerConnection(hostname=AMQP_IP, port=AMQP_PORT,
userid=AMQP_USER, password=AMQP_PASSWORD,
virtual_host="/")
self.channel = self.connection.channel()
# consume
self.media_exchange = Exchange("media", "direct", durable=True)
self.video_queue = Queue("video", exchange=self.media_exchange, key="video")
consumer = Consumer(self.channel, self.video_queue)
consumer.register_callback(self.process_media)
consumer.consume()
# Process messages on all channels
while True:
self.connection.drain_events()
def process_media(self, message_data, message):
feed_url = message_data["name"]
print ("Got feed import message for: %s" % feed_url)
# something importing this feed url
# import_feed(feed_url)
message.ack()
def init_rabbit_mq(self):
self.logger.info("Initializing RabbitMQ stuff")
try:
""""""
schedule_exchange = \
Exchange("airtime-pypo", "direct",
durable=True, auto_delete=True)
schedule_queue = \
Queue("pypo-fetch", exchange=schedule_exchange, key="foo")
connection = BrokerConnection(self.config["rabbitmq_host"], \
self.config["rabbitmq_user"], \
self.config["rabbitmq_password"], \
self.config["rabbitmq_vhost"])
channel = connection.channel()
self.simple_queue = SimpleQueue(channel, schedule_queue)
"""
connection = Connection('amqp://*****:*****@172.16.82.1:5672//pypox')
self.simple_queue = connection.SimpleQueue('pypo-fetch')
#message = simple_queue.get(block=True, timeout=1)
"""
except Exception, e:
self.logger.error(e)
return False
class PypoFetch(Thread):
def __init__(self, q):
Thread.__init__(self)
logger = logging.getLogger('fetch')
self.api_client = api_client.api_client_factory(config)
self.set_export_source('scheduler')
self.queue = q
logger.info("PypoFetch: init complete")
def init_rabbit_mq(self):
logger = logging.getLogger('fetch')
logger.info("Initializing RabbitMQ stuff")
try:
schedule_exchange = Exchange("airtime-schedule", "direct", durable=True, auto_delete=True)
schedule_queue = Queue("pypo-fetch", exchange=schedule_exchange, key="foo")
self.connection = BrokerConnection(config["rabbitmq_host"], config["rabbitmq_user"], config["rabbitmq_password"], "/")
channel = self.connection.channel()
consumer = Consumer(channel, schedule_queue)
consumer.register_callback(handle_message)
consumer.consume()
except Exception, e:
logger.error(e)
return False
return True
class AirtimeNotifier(Notifier):
def __init__(self, watch_manager, default_proc_fun=None, read_freq=0, threshold=0, timeout=None, airtime_config=None, api_client=None, bootstrap=None, mmc=None):
Notifier.__init__(self, watch_manager, default_proc_fun, read_freq, threshold, timeout)
self.logger = logging.getLogger()
self.config = airtime_config
self.api_client = api_client
self.bootstrap = bootstrap
self.md_manager = AirtimeMetadata()
self.import_processes = {}
self.watched_folders = []
self.mmc = mmc
self.wm = watch_manager
self.mask = pyinotify.ALL_EVENTS
while not self.init_rabbit_mq():
self.logger.error("Error connecting to RabbitMQ Server. Trying again in few seconds")
time.sleep(5)
def init_rabbit_mq(self):
self.logger.info("Initializing RabbitMQ stuff")
try:
schedule_exchange = Exchange("airtime-media-monitor", "direct", durable=True, auto_delete=True)
schedule_queue = Queue("media-monitor", exchange=schedule_exchange, key="filesystem")
self.connection = BrokerConnection(self.config.cfg["rabbitmq_host"], self.config.cfg["rabbitmq_user"], self.config.cfg["rabbitmq_password"], self.config.cfg["rabbitmq_vhost"])
channel = self.connection.channel()
consumer = Consumer(channel, schedule_queue)
consumer.register_callback(self.handle_message)
consumer.consume()
except Exception, e:
self.logger.error(e)
return False
return True
def _do_test():
conn = BrokerConnection(transport=Transport)
chan = conn.channel()
self.assertTrue(chan.Client)
self.assertTrue(chan.ResponseError)
self.assertTrue(conn.transport.connection_errors)
self.assertTrue(conn.transport.channel_errors)
class PypoFetch(Thread):
def __init__(self, q):
Thread.__init__(self)
logger = logging.getLogger('fetch')
self.api_client = api_client.api_client_factory(config)
self.set_export_source('scheduler')
self.queue = q
logger.info("PypoFetch: init complete")
def init_rabbit_mq(self):
logger = logging.getLogger('fetch')
logger.info("Initializing RabbitMQ stuff")
try:
schedule_exchange = Exchange("airtime-schedule",
"direct",
durable=True,
auto_delete=True)
schedule_queue = Queue("pypo-fetch",
exchange=schedule_exchange,
key="foo")
self.connection = BrokerConnection(config["rabbitmq_host"],
config["rabbitmq_user"],
config["rabbitmq_password"],
"/")
channel = self.connection.channel()
consumer = Consumer(channel, schedule_queue)
consumer.register_callback(handle_message)
consumer.consume()
except Exception, e:
logger.error(e)
return False
return True
def init_rabbit_mq(self):
self.logger.info("Initializing RabbitMQ stuff")
try:
self.logger.info("rabbitmq_host: " + self.config["rabbitmq_host"])
self.logger.info("rabbitmq_user: "******"rabbitmq_user"])
self.logger.info("rabbitmq_password: "******"rabbitmq_password"])
self.logger.info("rabbitmq_vhost: " +
self.config["rabbitmq_vhost"])
""""""
schedule_exchange = \
Exchange("airtime-pypo", "direct",
durable=True, auto_delete=True)
schedule_queue = \
Queue("pypo-fetch", exchange=schedule_exchange, key="foo")
connection = BrokerConnection(self.config["rabbitmq_host"], \
self.config["rabbitmq_user"], \
self.config["rabbitmq_password"], \
self.config["rabbitmq_vhost"])
channel = connection.channel()
self.simple_queue = SimpleQueue(channel, schedule_queue)
"""
connection = Connection('amqp://*****:*****@172.16.82.1:5672//pypox')
self.simple_queue = connection.SimpleQueue('pypo-fetch')
#message = simple_queue.get(block=True, timeout=1)
"""
except Exception, e:
self.logger.error(e)
return False
def test_publish__consume(self):
connection = BrokerConnection(transport=Transport)
channel = connection.channel()
producer = Producer(channel, self.exchange, routing_key="test_Redis")
consumer = Consumer(channel, self.queue)
producer.publish({"hello2": "world2"})
_received = []
def callback(message_data, message):
_received.append(message_data)
message.ack()
consumer.register_callback(callback)
consumer.consume()
self.assertIn(channel, channel.connection.cycle._channels)
try:
connection.drain_events(timeout=1)
self.assertTrue(_received)
self.assertRaises(socket.timeout,
connection.drain_events,
timeout=0.01)
finally:
channel.close()
def create_connection():
conn = BrokerConnection("localhost",
"fred",
"fred123",
"home")
channel = conn.channel()
return channel
def _do_test():
conn = BrokerConnection(transport=Transport)
chan = conn.channel()
self.assertTrue(chan.Client)
self.assertTrue(chan.ResponseError)
self.assertTrue(conn.transport.connection_errors)
self.assertTrue(conn.transport.channel_errors)
def test_publish__consume(self):
connection = BrokerConnection(transport=Transport)
channel = connection.channel()
producer = Producer(channel, self.exchange, routing_key="test_Redis")
consumer = Consumer(channel, self.queue)
producer.publish({"hello2": "world2"})
_received = []
def callback(message_data, message):
_received.append(message_data)
message.ack()
consumer.register_callback(callback)
consumer.consume()
self.assertTrue(channel._poller._can_start())
try:
connection.drain_events(timeout=1)
self.assertTrue(_received)
self.assertFalse(channel._poller._can_start())
self.assertRaises(socket.timeout,
connection.drain_events, timeout=0.01)
finally:
channel.close()
def setup_rabbit_mq_channel(self):
service_exchange = Exchange(self.acord_control_exchange, "topic", durable=False)
# connections/channels
connection = BrokerConnection(self.rabbit_host, self.rabbit_user, self.rabbit_password)
logging.info("Connection to RabbitMQ server successful")
channel = connection.channel()
# produce
self.producer = Producer(channel, exchange=service_exchange, routing_key='notifications.info')
def create_connection(config):
conn = BrokerConnection(config.get("connection", "hostname"),
config.get("connection", "userid"),
config.get("connection", "password"),
config.get("connection", "virtual_host"))
channel = conn.channel()
return channel
def setup_rabbit_mq_channel(self):
service_exchange = Exchange(self.acord_control_exchange, "topic", durable=False)
# connections/channels
connection = BrokerConnection(self.rabbit_host, self.rabbit_user, self.rabbit_password)
logging.info("Connection to RabbitMQ server successful")
channel = connection.channel()
# produce
self.producer = Producer(channel, exchange=service_exchange, routing_key='notifications.info')
self.publish = connection.ensure(self.producer, self.producer.publish, errback=self.errback, max_retries=3)
class AMQPWorker(Worker):
queues = [{
'routing_key': 'test',
'name': 'test',
'handler': 'handle_test'
}]
_connection = None
def handle_test(self, body, message):
log.debug("Handle message: %s" % body)
message.ack()
def handle(self):
log.debug("Start consuming")
exchange = Exchange('amqp.topic', type='direct', durable=True)
self._connection = BrokerConnection(*CONNECTION)
channel = self._connection.channel()
for entry in self.queues:
log.debug("prepare to consume %s" % entry['routing_key'])
queue = Queue(entry['name'],
exchange=exchange,
routing_key=entry['routing_key'])
consumer = Consumer(channel, queue)
consumer.register_callback(getattr(self, entry['handler']))
consumer.consume()
log.debug("start consuming...")
while True:
try:
self._connection.drain_events()
except socket.timeout:
log.debug("nothing to consume...")
break
self._connection.close()
def run(self):
while self.alive:
try:
self.handle()
except Exception:
self.alive = False
raise
def handle_quit(self, sig, frame):
if self._connection is not None:
self._connection.close()
self.alive = False
def handle_exit(self, sig, frame):
if self._connection is not None:
self._connection.close()
self.alive = False
sys.exit(0)
class AMQPWorker(Worker):
queues = [
{'routing_key': 'test',
'name': 'test',
'handler': 'handle_test'
}
]
_connection = None
def handle_test(self, body, message):
log.debug("Handle message: %s" % body)
message.ack()
def handle(self):
log.debug("Start consuming")
exchange = Exchange('amqp.topic', type='direct', durable=True)
self._connection = BrokerConnection(*CONNECTION)
channel = self._connection.channel()
for entry in self.queues:
log.debug("prepare to consume %s" % entry['routing_key'])
queue = Queue(entry['name'], exchange=exchange,
routing_key=entry['routing_key'])
consumer = Consumer(channel, queue)
consumer.register_callback(getattr(self, entry['handler']))
consumer.consume()
log.debug("start consuming...")
while True:
try:
self._connection.drain_events()
except socket.timeout:
log.debug("nothing to consume...")
break
self._connection.close()
def run(self):
while self.alive:
try:
self.handle()
except Exception:
self.alive = False
raise
def handle_quit(self, sig, frame):
if self._connection is not None:
self._connection.close()
self.alive = False
def handle_exit(self, sig, frame):
if self._connection is not None:
self._connection.close()
self.alive = False
sys.exit(0)
def setup_rabbit_mq_channel():
global producer
global rabbit_user, rabbit_password, rabbit_host, vcpeservice_rabbit_exchange,cpe_publisher_id
vcpeservice_exchange = Exchange(vcpeservice_rabbit_exchange, "topic", durable=False)
# connections/channels
connection = BrokerConnection(rabbit_host, rabbit_user, rabbit_password)
print 'Connection to RabbitMQ server successful'
channel = connection.channel()
# produce
producer = Producer(channel, exchange=vcpeservice_exchange, routing_key='notifications.info')
def init_rabbit_mq(self):
self.logger.info("Initializing RabbitMQ stuff")
try:
schedule_exchange = Exchange("airtime-pypo", "direct", durable=True, auto_delete=True)
schedule_queue = Queue("pypo-fetch", exchange=schedule_exchange, key="foo")
connection = BrokerConnection(config["rabbitmq_host"], config["rabbitmq_user"], config["rabbitmq_password"], config["rabbitmq_vhost"])
channel = connection.channel()
self.simple_queue = SimpleQueue(channel, schedule_queue)
except Exception, e:
self.logger.error(e)
return False
class AirtimeNotifier(Loggable):
"""
AirtimeNotifier is responsible for interecepting RabbitMQ messages and
feeding them to the event_handler object it was initialized with. The only
thing it does to the messages is parse them from json
"""
def __init__(self, cfg, message_receiver):
self.cfg = cfg
self.handler = message_receiver
while not self.init_rabbit_mq():
self.logger.error(
"Error connecting to RabbitMQ Server. Trying again in few seconds"
)
time.sleep(5)
def init_rabbit_mq(self):
try:
self.logger.info("Initializing RabbitMQ message consumer...")
schedule_exchange = Exchange("airtime-media-monitor",
"direct",
durable=True,
auto_delete=True)
schedule_queue = Queue("media-monitor",
exchange=schedule_exchange,
key="filesystem")
self.connection = BrokerConnection(self.cfg["rabbitmq_host"],
self.cfg["rabbitmq_user"],
self.cfg["rabbitmq_password"],
self.cfg["rabbitmq_vhost"])
channel = self.connection.channel()
self.simple_queue = SimpleQueue(channel, schedule_queue)
self.logger.info("Initialized RabbitMQ consumer.")
except Exception as e:
self.logger.info("Failed to initialize RabbitMQ consumer")
self.logger.error(e)
return False
return True
def handle_message(self, message):
"""
Messages received from RabbitMQ are handled here. These messages
instruct media-monitor of events such as a new directory being watched,
file metadata has been changed, or any other changes to the config of
media-monitor via the web UI.
"""
self.logger.info("Received md from RabbitMQ: %s" % str(message))
m = json.loads(message)
# TODO : normalize any other keys that could be used to pass
# directories
if 'directory' in m: m['directory'] = normpath(m['directory'])
self.handler.message(m)
def setup_rabbit_mq_channel(self):
service_exchange = Exchange(cfg.CONF.udpservice.acord_control_exchange, "topic", durable=False)
rabbit_host = cfg.CONF.udpservice.rabbit_hosts
rabbit_user = cfg.CONF.udpservice.rabbit_userid
rabbit_password = cfg.CONF.udpservice.rabbit_password
# connections/channels
connection = BrokerConnection(rabbit_host, rabbit_user, rabbit_password)
print 'Connection to RabbitMQ server successful'
channel = connection.channel()
# produce
self.producer = Producer(channel, exchange=service_exchange, routing_key='notifications.info')
def init_rabbit_mq(self):
self.logger.info("Initializing RabbitMQ stuff")
try:
schedule_exchange = Exchange("airtime-pypo", "direct", durable=True, auto_delete=True)
schedule_queue = Queue("pypo-fetch", exchange=schedule_exchange, key="foo")
connection = BrokerConnection(config["rabbitmq_host"], config["rabbitmq_user"], config["rabbitmq_password"], config["rabbitmq_vhost"])
channel = connection.channel()
self.simple_queue = SimpleQueue(channel, schedule_queue)
except Exception, e:
self.logger.error(e)
return False
def test_establish_connection(self):
conn = BrokerConnection(port=5672, backend_cls=Backend)
conn.connect()
self.assertTrue(conn.connection.connected)
self.assertEqual(conn.host, "localhost:5672")
channel = conn.channel()
self.assertTrue(channel.open)
self.assertEqual(conn.drain_events(), "event")
_connection = conn.connection
conn.close()
self.assertFalse(_connection.connected)
self.assertIsInstance(conn.backend, Backend)
class AirtimeNotifier(Loggable):
"""
AirtimeNotifier is responsible for interecepting RabbitMQ messages and
feeding them to the event_handler object it was initialized with. The only
thing it does to the messages is parse them from json
"""
def __init__(self, cfg, message_receiver):
self.cfg = cfg
self.handler = message_receiver
while not self.init_rabbit_mq():
self.logger.error("Error connecting to RabbitMQ Server. Trying again in few seconds")
time.sleep(5)
def init_rabbit_mq(self):
try:
self.logger.info("Initializing RabbitMQ message consumer...")
schedule_exchange = Exchange("airtime-media-monitor", "direct", durable=True, auto_delete=True)
schedule_queue = Queue("media-monitor", exchange=schedule_exchange, key="filesystem")
self.connection = BrokerConnection(
self.cfg["rabbitmq"]["host"],
self.cfg["rabbitmq"]["user"],
self.cfg["rabbitmq"]["password"],
self.cfg["rabbitmq"]["vhost"],
)
channel = self.connection.channel()
self.simple_queue = SimpleQueue(channel, schedule_queue)
self.logger.info("Initialized RabbitMQ consumer.")
except Exception as e:
self.logger.info("Failed to initialize RabbitMQ consumer")
self.logger.error(e)
return False
return True
def handle_message(self, message):
"""
Messages received from RabbitMQ are handled here. These messages
instruct media-monitor of events such as a new directory being watched,
file metadata has been changed, or any other changes to the config of
media-monitor via the web UI.
"""
self.logger.info("Received md from RabbitMQ: %s" % str(message))
m = json.loads(message)
# TODO : normalize any other keys that could be used to pass
# directories
if "directory" in m:
m["directory"] = normpath(m["directory"])
self.handler.message(m)
def setup_rabbit_mq_channel():
global producer
global rabbit_user, rabbit_password, rabbit_host, vcpeservice_rabbit_exchange,cpe_publisher_id
vcpeservice_exchange = Exchange(vcpeservice_rabbit_exchange, "topic", durable=False)
# connections/channels
connection = BrokerConnection(rabbit_host, rabbit_user, rabbit_password)
logger.info('Connection to RabbitMQ server successful')
channel = connection.channel()
# produce
producer = Producer(channel, exchange=vcpeservice_exchange, routing_key='notifications.info')
p = subprocess.Popen('hostname', shell=True, stdout=subprocess.PIPE)
(hostname, error) = p.communicate()
cpe_publisher_id = cpe_publisher_id + '_on_' + hostname
logger.info('cpe_publisher_id=%s',cpe_publisher_id)
def setup_rabbit_mq_channel(self):
ceilometer_exchange = Exchange(self.rabbit_exchange,
"topic",
durable=False)
# connections/channels
connection = BrokerConnection(self.rabbit_host, self.rabbit_user,
self.rabbit_password)
LOG.info(
"BroadViewPublisher: Connection to RabbitMQ server successful")
channel = connection.channel()
# produce
self._producer = Producer(channel,
exchange=ceilometer_exchange,
routing_key='notifications.info')
self._publish = connection.ensure(self._producer,
self._producer.publish,
errback=self.errback,
max_retries=3)
class AirtimeNotifier(Loggable):
"""
AirtimeNotifier is responsible for interecepting RabbitMQ messages and
feeding them to the event_handler object it was initialized with. The only
thing it does to the messages is parse them from json
"""
def __init__(self, cfg, message_receiver):
self.cfg = cfg
try:
self.handler = message_receiver
self.logger.info("Initializing RabbitMQ message consumer...")
schedule_exchange = Exchange("airtime-media-monitor", "direct",
durable=True, auto_delete=True)
schedule_queue = Queue("media-monitor", exchange=schedule_exchange,
key="filesystem")
self.connection = BrokerConnection(cfg["rabbitmq_host"],
cfg["rabbitmq_user"], cfg["rabbitmq_password"],
cfg["rabbitmq_vhost"])
channel = self.connection.channel()
consumer = Consumer(channel, schedule_queue)
consumer.register_callback(self.handle_message)
consumer.consume()
self.logger.info("Initialized RabbitMQ consumer.")
except Exception as e:
self.logger.info("Failed to initialize RabbitMQ consumer")
self.logger.error(e)
def handle_message(self, body, message):
"""
Messages received from RabbitMQ are handled here. These messages
instruct media-monitor of events such as a new directory being watched,
file metadata has been changed, or any other changes to the config of
media-monitor via the web UI.
"""
message.ack()
self.logger.info("Received md from RabbitMQ: %s" % str(body))
m = json.loads(message.body)
# TODO : normalize any other keys that could be used to pass
# directories
if 'directory' in m: m['directory'] = normpath(m['directory'])
self.handler.message(m)
class Sender():
""" Simple class to wrap the operations needed for an AMQP listener """
def __init__(self, hostname="127.0.0.1", userid="guest", password="******",
virtual_host="/", port=5672, name="", routing_key=""):
""" Setup a connection to the AMQP server, get a channel
and create an exchange.
A specific service listener implementation overrides the name
and routing_key
"""
if name == "":
raise Exception("Name must be non-empty string")
self.name = name
self.routing_key = routing_key
if routing_key == "":
exchange_type = "fanout"
elif "*" in routing_key or "#" in routing_key:
exchange_type = "topic"
else :
exchange_type = "direct"
self.connection = BrokerConnection(hostname=hostname,
userid=userid, password=password,
virtual_host=virtual_host, port=443,
insist=False, ssl=False)
self.channel = self.connection.channel()
self.exchange = Exchange(name=self.name, type=exchange_type, durable=False,
channel=self.channel)
self.connection.connect()
return
def send(self, msg):
""" Publishes a message to the AMQP server
on the initialized exchange
msg is a string, usually a JSON dump
"""
self.exchange.publish(self.exchange.Message(msg), routing_key=self.routing_key)
return
def main():
""" Query the master server and add the results to the database """
usage = 'usage: masterserver.py [options]'
parser = OptionParser(usage)
parser.add_option("-d", "--debug",
action="store_true", dest="debug", default=False,
help="enable debug messages")
(options, args) = parser.parse_args()
if len(args) != 0:
parser.error('incorrect number of arguments')
logging.basicConfig(level=logging.DEBUG)
if not options.debug:
logging.disable(logging.DEBUG)
# declare exchange
server_exchange = Exchange('servers', type='fanout')
# set up our amqp connection
connection = BrokerConnection(
hostname='localhost',
userid='gamelion',
password='******',
virtual_host='/'
)
channel = connection.channel()
producer = Producer(channel, server_exchange, serializer="pickle")
# run through all the master servers we know of and ask them for ips
for server_address in master_servers:
logging.debug('*' * 60)
logging.debug('NEW SERVER: %s', str(server_address))
logging.debug('*' * 60)
run_full_query(server_address, producer)
channel.close()
connection.release()
def run(self):
amqhost = self.config_parser.get(self.section,'amqhost')
amqport = int(self.config_parser.get(self.section,'amqport'))
amquser = self.config_parser.get(self.section,'amquser')
amqpass = self.config_parser.get(self.section,'amqpass')
amqvhost = self.config_parser.get(self.section,'amqvhost')
amqexchange = self.config_parser.get(self.section,'amqexchange')
routing_key = self.config_parser.get(self.section,'routing_key')
connection = BrokerConnection(hostname=amqhost, port=amqport,
userid=amquser, password=amqpass,
virtual_host=amqvhost)
channel = connection.channel()
exchange = Exchange(amqexchange,'topic',turable=True)
producer = Producer(channel,exchange=exchange,
routing_key=routing_key)
global entire_history
entire_history = self.config_parser.get(self.section,'entire_history')
for job in jobs():
if job['type']=='start':
job['completion_time']+=job['walltime']
producer.revive(channel)
producer.publish(job)
logging.debug(`job`)
class Listener():
""" Simple class to wrap the operations needed for an AMQP listener """
def __init__(self, hostname="127.0.0.1", userid="guest", password="******",
virtual_host="/", port=5672):
""" Setup a connection to the AMQP server, get a channel
Create a topic exchange, attach a bonded queue to it
and register a consumer callback.
A specific service listener implementation overrides the name
and routing_key
"""
self.connection = BrokerConnection(hostname=hostname,
userid=userid, password=password,
virtual_host=virtual_host, port=port,
insist=False, ssl=False)
self.channel = self.connection.channel()
self.exchange = Exchange(name=self.name, type="topic", durable=True,
channel=self.channel)
self.queue = Queue(self.name, exchange=self.exchange,
routing_key=self.routing_key)
self.queue = self.queue(self.channel)
self.queue.declare()
self.queue.consume(consumer_tag="", callback=self.callback, no_ack=True)
self.connection.connect()
return
def callback(self, msg):
""" This callback is run when a message is recieved """
return
def consume(self):
""" Event loop """
while True:
self.connection.drain_events()
return
class test_Channel(TestCase):
def setUp(self):
self.connection = BrokerConnection(transport=Transport)
self.channel = self.connection.channel()
def test_basic_consume_when_fanout_queue(self):
self.channel.exchange_declare(exchange="txconfan", type="fanout")
self.channel.queue_declare(queue="txconfanq")
self.channel.queue_bind(queue="txconfanq", exchange="txconfan")
self.assertIn("txconfanq", self.channel._fanout_queues)
self.channel.basic_consume("txconfanq", False, None, 1)
self.assertIn("txconfanq", self.channel.active_fanout_queues)
self.assertEqual(self.channel._fanout_to_queue.get("txconfan"),
"txconfanq")
def test_basic_cancel_unknown_delivery_tag(self):
self.assertIsNone(self.channel.basic_cancel("txaseqwewq"))
def test_subscribe_no_queues(self):
self.channel.subclient = Mock()
self.channel.active_fanout_queues.clear()
self.channel._subscribe()
self.assertFalse(self.channel.subclient.subscribe.called)
def test_subscribe(self):
self.channel.subclient = Mock()
self.channel.active_fanout_queues.add("a")
self.channel.active_fanout_queues.add("b")
self.channel._fanout_queues.update(a="a", b="b")
self.channel._subscribe()
self.assertTrue(self.channel.subclient.subscribe.called)
s_args, _ = self.channel.subclient.subscribe.call_args
self.assertItemsEqual(s_args[0], ["a", "b"])
self.channel.subclient.connection._sock = None
self.channel._subscribe()
self.channel.subclient.connection.connect.assert_called_with()
def test_handle_unsubscribe_message(self):
s = self.channel.subclient
s.subscribed = True
self.channel._handle_message(s, ["unsubscribe", "a", 0])
self.assertFalse(s.subscribed)
def test_handle_pmessage_message(self):
self.assertDictEqual(
self.channel._handle_message(
self.channel.subclient,
["pmessage", "pattern", "channel", "data"]), {
"type": "pmessage",
"pattern": "pattern",
"channel": "channel",
"data": "data"
})
def test_handle_message(self):
self.assertDictEqual(
self.channel._handle_message(self.channel.subclient,
["type", "channel", "data"]), {
"type": "type",
"pattern": None,
"channel": "channel",
"data": "data"
})
def test_brpop_start_but_no_queues(self):
self.channel.active_queues.clear()
self.assertIsNone(self.channel._brpop_start())
def test_receive(self):
s = self.channel.subclient = Mock()
self.channel._fanout_to_queue["a"] = "b"
s.parse_response.return_value = [
"message", "a", dumps({"hello": "world"})
]
payload, queue = self.channel._receive()
self.assertDictEqual(payload, {"hello": "world"})
self.assertEqual(queue, "b")
def test_receive_raises(self):
self.channel._in_listen = True
s = self.channel.subclient = Mock()
s.parse_response.side_effect = KeyError("foo")
with self.assertRaises(redis.Empty):
self.channel._receive()
self.assertFalse(self.channel._in_listen)
def test_receive_empty(self):
s = self.channel.subclient = Mock()
s.parse_response.return_value = None
with self.assertRaises(redis.Empty):
self.channel._receive()
def test_receive_different_message_Type(self):
s = self.channel.subclient = Mock()
s.parse_response.return_value = ["pmessage", "/foo/", 0, "data"]
with self.assertRaises(redis.Empty):
self.channel._receive()
def test_brpop_read_raises(self):
c = self.channel.client = Mock()
c.parse_response.side_effect = KeyError("foo")
with self.assertRaises(redis.Empty):
self.channel._brpop_read()
c.connection.disconnect.assert_called_with()
def test_brpop_read_gives_None(self):
c = self.channel.client = Mock()
c.parse_response.return_value = None
with self.assertRaises(redis.Empty):
self.channel._brpop_read()
def test_poll_error(self):
c = self.channel.client = Mock()
c.parse_response = Mock()
self.channel._poll_error("BRPOP")
c.parse_response.assert_called_with("BRPOP")
c.parse_response.side_effect = KeyError("foo")
self.assertIsNone(self.channel._poll_error("BRPOP"))
def test_put_fanout(self):
self.channel._in_poll = False
c = self.channel.client = Mock()
body = {"hello": "world"}
self.channel._put_fanout("exchange", body)
c.publish.assert_called_with("exchange", dumps(body))
def test_delete(self):
x = self.channel
self.channel._in_poll = False
c = x.client = Mock()
x._delete("queue", "exchange", "routing_key", None)
c.delete.assert_called_with("queue")
c.srem.assert_called_with(x.keyprefix_queue % ("exchange", ),
x.sep.join(["routing_key", "", "queue"]))
def test_has_queue(self):
self.channel._in_poll = False
c = self.channel.client = Mock()
c.exists.return_value = True
self.assertTrue(self.channel._has_queue("foo"))
c.exists.assert_called_with("foo")
c.exists.return_value = False
self.assertFalse(self.channel._has_queue("foo"))
def test_close_when_closed(self):
self.channel.closed = True
self.channel.close()
def test_close_client_close_raises(self):
c = self.channel.client = Mock()
c.connection.disconnect.side_effect = self.channel.ResponseError()
self.channel.close()
c.connection.disconnect.assert_called_with()
def test_invalid_database_raises_ValueError(self):
self.channel.connection.client.virtual_host = "xfeqwewkfk"
with self.assertRaises(ValueError):
self.channel._create_client()
@skip_if_not_module("redis")
def test_get_client(self):
import redis as R
KombuRedis = redis.Channel._get_client(self.channel)
self.assertTrue(KombuRedis)
Rv = getattr(R, "__version__")
try:
R.__version__ = "2.4.0"
with self.assertRaises(VersionMismatch):
redis.Channel._get_client(self.channel)
finally:
if Rv is not None:
R.__version__ = Rv
@skip_if_not_module("redis")
def test_get_response_error(self):
from redis.exceptions import ResponseError
self.assertIs(redis.Channel._get_response_error(self.channel),
ResponseError)
def test_avail_client_when_not_in_poll(self):
self.channel._in_poll = False
c = self.channel.client = Mock()
self.assertIs(self.channel._avail_client, c)
def test_avail_client_when_in_poll(self):
self.channel._in_poll = True
cc = self.channel._create_client = Mock()
self.assertTrue(self.channel._avail_client)
cc.assert_called_with()
@skip_if_not_module("redis")
def test_transport_get_errors(self):
self.assertTrue(redis.Transport._get_errors(self.connection.transport))
@skip_if_not_module("redis")
def test_transport_get_errors_when_InvalidData_used(self):
from redis import exceptions
class ID(Exception):
pass
DataError = getattr(exceptions, "DataError", None)
InvalidData = getattr(exceptions, "InvalidData", None)
exceptions.InvalidData = ID
exceptions.DataError = None
try:
errors = redis.Transport._get_errors(self.connection.transport)
self.assertTrue(errors)
self.assertIn(ID, errors[1])
finally:
if DataError is not None:
exceptions.DataError = DataError
if InvalidData is not None:
exceptions.InvalidData = InvalidData
def run(self):
print 'pypo Pusher'
if self.action == 'update_schedule':
print 'update_schedule!!'
credentials = pika.PlainCredentials(MQ_USER, MQ_PASS)
connection = pika.BlockingConnection(pika.ConnectionParameters(MQ_HOST,
5672,
'/airtime',
credentials))
channel = connection.channel()
channel.queue_declare(queue='pypo-fetch', durable=True)
message = {
'schedule': {
'media': {}
},
'event_type': 'update_schedule'
}
import json
message = json.dumps(message)
message = 'hallo'
channel.basic_publish(exchange='airtime-pypo',
routing_key='pypo-fetch',
body=message)
channel.close()
connection.close()
if self.action == 'update_schedule_kombu':
print 'update_schedule!!'
exchange = Exchange("airtime-pypo", "direct", durable=True, auto_delete=True)
queue = Queue("pypo-fetch", exchange=exchange, key="foo", durable=True)
connection = BrokerConnection(MQ_HOST, MQ_USER, MQ_PASS, MQ_VHOST)
channel = connection.channel()
simple_queue = SimpleQueue(channel, queue)
message = {
'schedule': {
'media': {}
},
'event_type': 'update_schedule'
}
print simple_queue.qsize()
print 'do:'
producer = Producer(channel, exchange=exchange, routing_key=None, serializer="json")
producer.publish(message, routing_key='pypo-fetch')
print simple_queue.qsize()
channel.close()
class test_Channel(TestCase):
def setUp(self):
self.connection = BrokerConnection(transport=Transport)
self.channel = self.connection.channel()
def test_basic_consume_when_fanout_queue(self):
self.channel.exchange_declare(exchange="txconfan", type="fanout")
self.channel.queue_declare(queue="txconfanq")
self.channel.queue_bind(queue="txconfanq", exchange="txconfan")
self.assertIn("txconfanq", self.channel._fanout_queues)
self.channel.basic_consume("txconfanq", False, None, 1)
self.assertIn("txconfanq", self.channel.active_fanout_queues)
self.assertEqual(self.channel._fanout_to_queue.get("txconfan"),
"txconfanq")
def test_basic_cancel_unknown_delivery_tag(self):
self.assertIsNone(self.channel.basic_cancel("txaseqwewq"))
def test_subscribe_no_queues(self):
self.channel.subclient = Mock()
self.channel.active_fanout_queues.clear()
self.channel._subscribe()
self.assertFalse(self.channel.subclient.subscribe.called)
def test_subscribe(self):
self.channel.subclient = Mock()
self.channel.active_fanout_queues.add("a")
self.channel.active_fanout_queues.add("b")
self.channel._fanout_queues.update(a="a", b="b")
self.channel._subscribe()
self.assertTrue(self.channel.subclient.subscribe.called)
s_args, _ = self.channel.subclient.subscribe.call_args
self.assertItemsEqual(s_args[0], ["a", "b"])
self.channel.subclient.connection._sock = None
self.channel._subscribe()
self.channel.subclient.connection.connect.assert_called_with()
def test_handle_unsubscribe_message(self):
s = self.channel.subclient
s.subscribed = True
self.channel._handle_message(s, ["unsubscribe", "a", 0])
self.assertFalse(s.subscribed)
def test_handle_pmessage_message(self):
self.assertDictEqual(self.channel._handle_message(
self.channel.subclient,
["pmessage", "pattern", "channel", "data"]),
{"type": "pmessage",
"pattern": "pattern",
"channel": "channel",
"data": "data"})
def test_handle_message(self):
self.assertDictEqual(self.channel._handle_message(
self.channel.subclient,
["type", "channel", "data"]),
{"type": "type",
"pattern": None,
"channel": "channel",
"data": "data"})
def test_brpop_start_but_no_queues(self):
self.channel.active_queues.clear()
self.assertIsNone(self.channel._brpop_start())
def test_receive(self):
s = self.channel.subclient = Mock()
self.channel._fanout_to_queue["a"] = "b"
s.parse_response.return_value = ["message", "a",
dumps({"hello": "world"})]
payload, queue = self.channel._receive()
self.assertDictEqual(payload, {"hello": "world"})
self.assertEqual(queue, "b")
def test_receive_raises(self):
self.channel._in_listen = True
s = self.channel.subclient = Mock()
s.parse_response.side_effect = KeyError("foo")
with self.assertRaises(redis.Empty):
self.channel._receive()
self.assertFalse(self.channel._in_listen)
def test_receive_empty(self):
s = self.channel.subclient = Mock()
s.parse_response.return_value = None
with self.assertRaises(redis.Empty):
self.channel._receive()
def test_receive_different_message_Type(self):
s = self.channel.subclient = Mock()
s.parse_response.return_value = ["pmessage", "/foo/", 0, "data"]
with self.assertRaises(redis.Empty):
self.channel._receive()
def test_brpop_read_raises(self):
c = self.channel.client = Mock()
c.parse_response.side_effect = KeyError("foo")
with self.assertRaises(redis.Empty):
self.channel._brpop_read()
c.connection.disconnect.assert_called_with()
def test_brpop_read_gives_None(self):
c = self.channel.client = Mock()
c.parse_response.return_value = None
with self.assertRaises(redis.Empty):
self.channel._brpop_read()
def test_poll_error(self):
c = self.channel.client = Mock()
c.parse_response = Mock()
self.channel._poll_error("BRPOP")
c.parse_response.assert_called_with("BRPOP")
c.parse_response.side_effect = KeyError("foo")
self.assertIsNone(self.channel._poll_error("BRPOP"))
def test_put_fanout(self):
self.channel._in_poll = False
c = self.channel.client = Mock()
body = {"hello": "world"}
self.channel._put_fanout("exchange", body)
c.publish.assert_called_with("exchange", dumps(body))
def test_delete(self):
x = self.channel
self.channel._in_poll = False
c = x.client = Mock()
x._delete("queue", "exchange", "routing_key", None)
c.delete.assert_called_with("queue")
c.srem.assert_called_with(x.keyprefix_queue % ("exchange", ),
x.sep.join(["routing_key", "", "queue"]))
def test_has_queue(self):
self.channel._in_poll = False
c = self.channel.client = Mock()
c.exists.return_value = True
self.assertTrue(self.channel._has_queue("foo"))
c.exists.assert_called_with("foo")
c.exists.return_value = False
self.assertFalse(self.channel._has_queue("foo"))
def test_close_when_closed(self):
self.channel.closed = True
self.channel.close()
def test_close_client_close_raises(self):
c = self.channel.client = Mock()
c.connection.disconnect.side_effect = self.channel.ResponseError()
self.channel.close()
c.connection.disconnect.assert_called_with()
def test_invalid_database_raises_ValueError(self):
self.channel.connection.client.virtual_host = "xfeqwewkfk"
with self.assertRaises(ValueError):
self.channel._create_client()
@skip_if_not_module("redis")
def test_get_client(self):
import redis as R
KombuRedis = redis.Channel._get_client(self.channel)
self.assertTrue(KombuRedis)
Rv = getattr(R, "__version__")
try:
R.__version__ = "2.4.0"
with self.assertRaises(VersionMismatch):
redis.Channel._get_client(self.channel)
finally:
if Rv is not None:
R.__version__ = Rv
@skip_if_not_module("redis")
def test_get_response_error(self):
from redis.exceptions import ResponseError
self.assertIs(redis.Channel._get_response_error(self.channel),
ResponseError)
def test_avail_client_when_not_in_poll(self):
self.channel._in_poll = False
c = self.channel.client = Mock()
self.assertIs(self.channel._avail_client, c)
def test_avail_client_when_in_poll(self):
self.channel._in_poll = True
cc = self.channel._create_client = Mock()
self.assertTrue(self.channel._avail_client)
cc.assert_called_with()
@skip_if_not_module("redis")
def test_transport_get_errors(self):
self.assertTrue(redis.Transport._get_errors(self.connection.transport))
@skip_if_not_module("redis")
def test_transport_get_errors_when_InvalidData_used(self):
from redis import exceptions
class ID(Exception):
pass
DataError = getattr(exceptions, "DataError", None)
InvalidData = getattr(exceptions, "InvalidData", None)
exceptions.InvalidData = ID
exceptions.DataError = None
try:
errors = redis.Transport._get_errors(self.connection.transport)
self.assertTrue(errors)
self.assertIn(ID, errors[1])
finally:
if DataError is not None:
exceptions.DataError = DataError
if InvalidData is not None:
exceptions.InvalidData = InvalidData
class test_Redis(TestCase):
def setUp(self):
self.connection = BrokerConnection(transport=Transport)
self.exchange = Exchange("test_Redis", type="direct")
self.queue = Queue("test_Redis", self.exchange, "test_Redis")
def tearDown(self):
self.connection.close()
def test_publish__get(self):
channel = self.connection.channel()
producer = Producer(channel, self.exchange, routing_key="test_Redis")
self.queue(channel).declare()
producer.publish({"hello": "world"})
self.assertDictEqual(self.queue(channel).get().payload,
{"hello": "world"})
self.assertIsNone(self.queue(channel).get())
self.assertIsNone(self.queue(channel).get())
self.assertIsNone(self.queue(channel).get())
def test_publish__consume(self):
connection = BrokerConnection(transport=Transport)
channel = connection.channel()
producer = Producer(channel, self.exchange, routing_key="test_Redis")
consumer = Consumer(channel, self.queue)
producer.publish({"hello2": "world2"})
_received = []
def callback(message_data, message):
_received.append(message_data)
message.ack()
consumer.register_callback(callback)
consumer.consume()
self.assertIn(channel, channel.connection.cycle._channels)
try:
connection.drain_events(timeout=1)
self.assertTrue(_received)
with self.assertRaises(socket.timeout):
connection.drain_events(timeout=0.01)
finally:
channel.close()
def test_purge(self):
channel = self.connection.channel()
producer = Producer(channel, self.exchange, routing_key="test_Redis")
self.queue(channel).declare()
for i in range(10):
producer.publish({"hello": "world-%s" % (i, )})
self.assertEqual(channel._size("test_Redis"), 10)
self.assertEqual(self.queue(channel).purge(), 10)
channel.close()
def test_db_values(self):
c1 = BrokerConnection(virtual_host=1,
transport=Transport).channel()
self.assertEqual(c1.client.db, 1)
c2 = BrokerConnection(virtual_host="1",
transport=Transport).channel()
self.assertEqual(c2.client.db, 1)
c3 = BrokerConnection(virtual_host="/1",
transport=Transport).channel()
self.assertEqual(c3.client.db, 1)
with self.assertRaises(Exception):
BrokerConnection(virtual_host="/foo",
transport=Transport).channel()
def test_db_port(self):
c1 = BrokerConnection(port=None, transport=Transport).channel()
self.assertEqual(c1.client.port, Transport.default_port)
c1.close()
c2 = BrokerConnection(port=9999, transport=Transport).channel()
self.assertEqual(c2.client.port, 9999)
c2.close()
def test_close_poller_not_active(self):
c = BrokerConnection(transport=Transport).channel()
cycle = c.connection.cycle
c.client.connection
c.close()
self.assertNotIn(c, cycle._channels)
def test_close_ResponseError(self):
c = BrokerConnection(transport=Transport).channel()
c.client.bgsave_raises_ResponseError = True
c.close()
def test_close_disconnects(self):
c = BrokerConnection(transport=Transport).channel()
conn1 = c.client.connection
conn2 = c.subclient.connection
c.close()
self.assertTrue(conn1.disconnected)
self.assertTrue(conn2.disconnected)
def test_get__Empty(self):
channel = self.connection.channel()
with self.assertRaises(Empty):
channel._get("does-not-exist")
channel.close()
def test_get_client(self):
myredis, exceptions = _redis_modules()
@module_exists(myredis, exceptions)
def _do_test():
conn = BrokerConnection(transport=Transport)
chan = conn.channel()
self.assertTrue(chan.Client)
self.assertTrue(chan.ResponseError)
self.assertTrue(conn.transport.connection_errors)
self.assertTrue(conn.transport.channel_errors)
_do_test()
class test_Consumer(unittest.TestCase):
def setUp(self):
self.connection = BrokerConnection(transport=Transport)
self.connection.connect()
self.assertTrue(self.connection.connection.connected)
self.exchange = Exchange("foo", "direct")
def test_set_no_ack(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True, no_ack=True)
self.assertTrue(consumer.no_ack)
def test_set_callbacks(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
callbacks = [lambda x, y: x, lambda x, y: x]
consumer = Consumer(channel,
queue,
auto_declare=True,
callbacks=callbacks)
self.assertEqual(consumer.callbacks, callbacks)
def test_auto_declare(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.consume() # twice is a noop
self.assertIsNot(consumer.queues[0], queue)
self.assertTrue(consumer.queues[0].is_bound)
self.assertTrue(consumer.queues[0].exchange.is_bound)
self.assertIsNot(consumer.queues[0].exchange, self.exchange)
for meth in ("exchange_declare", "queue_declare", "queue_bind",
"basic_consume"):
self.assertIn(meth, channel)
self.assertEqual(channel.called.count("basic_consume"), 1)
self.assertTrue(consumer._active_tags)
consumer.cancel_by_queue(queue.name)
consumer.cancel_by_queue(queue.name)
self.assertFalse(consumer._active_tags)
def test_manual_declare(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=False)
self.assertIsNot(consumer.queues[0], queue)
self.assertTrue(consumer.queues[0].is_bound)
self.assertTrue(consumer.queues[0].exchange.is_bound)
self.assertIsNot(consumer.queues[0].exchange, self.exchange)
for meth in ("exchange_declare", "queue_declare", "basic_consume"):
self.assertNotIn(meth, channel)
consumer.declare()
for meth in ("exchange_declare", "queue_declare", "queue_bind"):
self.assertIn(meth, channel)
self.assertNotIn("basic_consume", channel)
consumer.consume()
self.assertIn("basic_consume", channel)
def test_consume__cancel(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.cancel()
self.assertIn("basic_cancel", channel)
self.assertFalse(consumer._active_tags)
def test___enter____exit__(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
context = consumer.__enter__()
self.assertIs(context, consumer)
self.assertTrue(consumer._active_tags)
res = consumer.__exit__(None, None, None)
self.assertFalse(res)
self.assertIn("basic_cancel", channel)
self.assertFalse(consumer._active_tags)
def test_flow(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
consumer.flow(False)
self.assertIn("flow", channel)
def test_qos(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
consumer.qos(30, 10, False)
self.assertIn("basic_qos", channel)
def test_purge(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
b2 = Queue("qname2", self.exchange, "rkey")
b3 = Queue("qname3", self.exchange, "rkey")
b4 = Queue("qname4", self.exchange, "rkey")
consumer = Consumer(channel, [b1, b2, b3, b4], auto_declare=True)
consumer.purge()
self.assertEqual(channel.called.count("queue_purge"), 4)
def test_multiple_queues(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
b2 = Queue("qname2", self.exchange, "rkey")
b3 = Queue("qname3", self.exchange, "rkey")
b4 = Queue("qname4", self.exchange, "rkey")
consumer = Consumer(channel, [b1, b2, b3, b4])
consumer.consume()
self.assertEqual(channel.called.count("exchange_declare"), 4)
self.assertEqual(channel.called.count("queue_declare"), 4)
self.assertEqual(channel.called.count("queue_bind"), 4)
self.assertEqual(channel.called.count("basic_consume"), 4)
self.assertEqual(len(consumer._active_tags), 4)
consumer.cancel()
self.assertEqual(channel.called.count("basic_cancel"), 4)
self.assertFalse(len(consumer._active_tags))
def test_receive_callback(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
received = []
def callback(message_data, message):
received.append(message_data)
message.ack()
message.payload # trigger cache
consumer.register_callback(callback)
consumer._receive_callback({u"foo": u"bar"})
self.assertIn("basic_ack", channel)
self.assertIn("message_to_python", channel)
self.assertEqual(received[0], {u"foo": u"bar"})
def test_basic_ack_twice(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.ack()
message.ack()
consumer.register_callback(callback)
self.assertRaises(MessageStateError, consumer._receive_callback,
{"foo": "bar"})
def test_basic_reject(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
consumer.register_callback(callback)
consumer._receive_callback({"foo": "bar"})
self.assertIn("basic_reject", channel)
def test_basic_reject_twice(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
message.reject()
consumer.register_callback(callback)
self.assertRaises(MessageStateError, consumer._receive_callback,
{"foo": "bar"})
self.assertIn("basic_reject", channel)
def test_basic_reject__requeue(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
consumer.register_callback(callback)
consumer._receive_callback({"foo": "bar"})
self.assertIn("basic_reject:requeue", channel)
def test_basic_reject__requeue_twice(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
message.requeue()
consumer.register_callback(callback)
self.assertRaises(MessageStateError, consumer._receive_callback,
{"foo": "bar"})
self.assertIn("basic_reject:requeue", channel)
def test_receive_without_callbacks_raises(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
self.assertRaises(NotImplementedError, consumer.receive, 1, 2)
def test_decode_error(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
consumer.channel.throw_decode_error = True
self.assertRaises(ValueError, consumer._receive_callback,
{"foo": "bar"})
def test_on_decode_error_callback(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
thrown = []
def on_decode_error(msg, exc):
thrown.append((msg.body, exc))
consumer = Consumer(channel, [b1], on_decode_error=on_decode_error)
consumer.channel.throw_decode_error = True
consumer._receive_callback({"foo": "bar"})
self.assertTrue(thrown)
m, exc = thrown[0]
self.assertEqual(anyjson.deserialize(m), {"foo": "bar"})
self.assertIsInstance(exc, ValueError)
def test_recover(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
consumer.recover()
self.assertIn("basic_recover", channel)
def test_revive(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
channel2 = self.connection.channel()
consumer.revive(channel2)
self.assertIs(consumer.channel, channel2)
self.assertIs(consumer.queues[0].channel, channel2)
self.assertIs(consumer.queues[0].exchange.channel, channel2)
def test__repr__(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
self.assertTrue(repr(Consumer(channel, [b1])))
class test_Producer(unittest.TestCase):
def setUp(self):
self.exchange = Exchange("foo", "direct")
self.connection = BrokerConnection(transport=Transport)
self.connection.connect()
self.assertTrue(self.connection.connection.connected)
self.assertFalse(self.exchange.is_bound)
def test_auto_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=True)
self.assertIsNot(p.exchange, self.exchange,
"creates Exchange clone at bind")
self.assertTrue(p.exchange.is_bound)
self.assertIn("exchange_declare", channel,
"auto_declare declares exchange")
def test_manual_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=False)
self.assertTrue(p.exchange.is_bound)
self.assertNotIn("exchange_declare", channel,
"auto_declare=False does not declare exchange")
p.declare()
self.assertIn("exchange_declare", channel,
"p.declare() declares exchange")
def test_prepare(self):
message = {u"the quick brown fox": u"jumps over the lazy dog"}
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
m, ctype, cencoding = p._prepare(message, headers={})
self.assertDictEqual(message, anyjson.deserialize(m))
self.assertEqual(ctype, "application/json")
self.assertEqual(cencoding, "utf-8")
def test_prepare_compression(self):
message = {u"the quick brown fox": u"jumps over the lazy dog"}
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
headers = {}
m, ctype, cencoding = p._prepare(message,
compression="zlib",
headers=headers)
self.assertEqual(ctype, "application/json")
self.assertEqual(cencoding, "utf-8")
self.assertEqual(headers["compression"], "application/x-gzip")
import zlib
self.assertEqual(
anyjson.deserialize(zlib.decompress(m).decode("utf-8")), message)
def test_prepare_custom_content_type(self):
message = "the quick brown fox".encode("utf-8")
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
m, ctype, cencoding = p._prepare(message, content_type="custom")
self.assertEqual(m, message)
self.assertEqual(ctype, "custom")
self.assertEqual(cencoding, "binary")
m, ctype, cencoding = p._prepare(message,
content_type="custom",
content_encoding="alien")
self.assertEqual(m, message)
self.assertEqual(ctype, "custom")
self.assertEqual(cencoding, "alien")
def test_prepare_is_already_unicode(self):
message = u"the quick brown fox"
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
m, ctype, cencoding = p._prepare(message, content_type="text/plain")
self.assertEqual(m, message.encode("utf-8"))
self.assertEqual(ctype, "text/plain")
self.assertEqual(cencoding, "utf-8")
m, ctype, cencoding = p._prepare(message,
content_type="text/plain",
content_encoding="utf-8")
self.assertEqual(m, message.encode("utf-8"))
self.assertEqual(ctype, "text/plain")
self.assertEqual(cencoding, "utf-8")
def test_publish(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
message = {u"the quick brown fox": u"jumps over the lazy dog"}
ret = p.publish(message, routing_key="process")
self.assertIn("prepare_message", channel)
self.assertIn("basic_publish", channel)
m, exc, rkey = ret
self.assertDictEqual(message, anyjson.deserialize(m["body"]))
self.assertDictContainsSubset(
{
"content_type": "application/json",
"content_encoding": "utf-8",
"priority": 0
}, m)
self.assertDictContainsSubset({"delivery_mode": 2}, m["properties"])
self.assertEqual(exc, p.exchange.name)
self.assertEqual(rkey, "process")
def test_no_exchange(self):
chan = self.connection.channel()
p = Producer(chan)
self.assertFalse(p.exchange.name)
def test_revive(self):
chan = self.connection.channel()
p = Producer(chan)
chan2 = self.connection.channel()
p.revive(chan2)
self.assertIs(p.channel, chan2)
self.assertIs(p.exchange.channel, chan2)
def test_on_return(self):
chan = self.connection.channel()
def on_return(exception, exchange, routing_key, message):
pass
p = Producer(chan, on_return=on_return)
self.assertTrue(on_return in chan.events["basic_return"])
self.assertTrue(p.on_return)
class test_Producer(TestCase):
def setUp(self):
self.exchange = Exchange("foo", "direct")
self.connection = BrokerConnection(transport=Transport)
self.connection.connect()
self.assertTrue(self.connection.connection.connected)
self.assertFalse(self.exchange.is_bound)
@patch("kombu.common.maybe_declare")
def test_maybe_declare(self, maybe_declare):
p = self.connection.Producer()
q = Queue("foo")
p.maybe_declare(q)
maybe_declare.assert_called_with(q, p.channel, False)
@patch("kombu.common.maybe_declare")
def test_maybe_declare_when_entity_false(self, maybe_declare):
p = self.connection.Producer()
p.maybe_declare(None)
self.assertFalse(maybe_declare.called)
def test_auto_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=True)
self.assertIsNot(p.exchange, self.exchange,
"creates Exchange clone at bind")
self.assertTrue(p.exchange.is_bound)
self.assertIn("exchange_declare", channel,
"auto_declare declares exchange")
def test_manual_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=False)
self.assertTrue(p.exchange.is_bound)
self.assertNotIn("exchange_declare", channel,
"auto_declare=False does not declare exchange")
p.declare()
self.assertIn("exchange_declare", channel,
"p.declare() declares exchange")
def test_prepare(self):
message = {u"the quick brown fox": u"jumps over the lazy dog"}
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
m, ctype, cencoding = p._prepare(message, headers={})
self.assertDictEqual(message, anyjson.loads(m))
self.assertEqual(ctype, "application/json")
self.assertEqual(cencoding, "utf-8")
def test_prepare_compression(self):
message = {u"the quick brown fox": u"jumps over the lazy dog"}
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
headers = {}
m, ctype, cencoding = p._prepare(message,
compression="zlib",
headers=headers)
self.assertEqual(ctype, "application/json")
self.assertEqual(cencoding, "utf-8")
self.assertEqual(headers["compression"], "application/x-gzip")
import zlib
self.assertEqual(anyjson.loads(zlib.decompress(m).decode("utf-8")),
message)
def test_prepare_custom_content_type(self):
message = "the quick brown fox".encode("utf-8")
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
m, ctype, cencoding = p._prepare(message, content_type="custom")
self.assertEqual(m, message)
self.assertEqual(ctype, "custom")
self.assertEqual(cencoding, "binary")
m, ctype, cencoding = p._prepare(message,
content_type="custom",
content_encoding="alien")
self.assertEqual(m, message)
self.assertEqual(ctype, "custom")
self.assertEqual(cencoding, "alien")
def test_prepare_is_already_unicode(self):
message = u"the quick brown fox"
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
m, ctype, cencoding = p._prepare(message, content_type="text/plain")
self.assertEqual(m, message.encode("utf-8"))
self.assertEqual(ctype, "text/plain")
self.assertEqual(cencoding, "utf-8")
m, ctype, cencoding = p._prepare(message,
content_type="text/plain",
content_encoding="utf-8")
self.assertEqual(m, message.encode("utf-8"))
self.assertEqual(ctype, "text/plain")
self.assertEqual(cencoding, "utf-8")
def test_publish_with_Exchange_instance(self):
p = self.connection.Producer()
p.exchange.publish = Mock()
p.publish("hello", exchange=Exchange("foo"))
self.assertEqual(p.exchange.publish.call_args[0][4], "foo")
def test_publish_retry_with_declare(self):
p = self.connection.Producer()
p.maybe_declare = Mock()
ensure = p.connection.ensure = Mock()
ex = Exchange("foo")
p.publish("hello",
exchange=ex,
declare=[ex],
retry=True,
retry_policy={"step": 4})
p.maybe_declare.assert_called_with(ex, True, step=4)
ensure.assert_called_with(p, p.exchange.publish, step=4)
def test_revive_when_channel_is_connection(self):
p = self.connection.Producer()
p.exchange = Mock()
new_conn = BrokerConnection("memory://")
defchan = new_conn.default_channel
p.revive(new_conn)
self.assertIs(p.channel, defchan)
p.exchange.revive.assert_called_with(defchan)
def test_enter_exit(self):
p = self.connection.Producer()
p.release = Mock()
self.assertIs(p.__enter__(), p)
p.__exit__()
p.release.assert_called_with()
def test_connection_property_handles_AttributeError(self):
p = self.connection.Producer()
p.channel = object()
self.assertIsNone(p.connection)
def test_publish(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
message = {u"the quick brown fox": u"jumps over the lazy dog"}
ret = p.publish(message, routing_key="process")
self.assertIn("prepare_message", channel)
self.assertIn("basic_publish", channel)
m, exc, rkey = ret
self.assertDictEqual(message, anyjson.loads(m["body"]))
self.assertDictContainsSubset(
{
"content_type": "application/json",
"content_encoding": "utf-8",
"priority": 0
}, m)
self.assertDictContainsSubset({"delivery_mode": 2}, m["properties"])
self.assertEqual(exc, p.exchange.name)
self.assertEqual(rkey, "process")
def test_no_exchange(self):
chan = self.connection.channel()
p = Producer(chan)
self.assertFalse(p.exchange.name)
def test_revive(self):
chan = self.connection.channel()
p = Producer(chan)
chan2 = self.connection.channel()
p.revive(chan2)
self.assertIs(p.channel, chan2)
self.assertIs(p.exchange.channel, chan2)
def test_on_return(self):
chan = self.connection.channel()
def on_return(exception, exchange, routing_key, message):
pass
p = Producer(chan, on_return=on_return)
self.assertTrue(on_return in chan.events["basic_return"])
self.assertTrue(p.on_return)
process_servers()
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
server_exchange = Exchange('servers', type='fanout')
server_queue = Queue('servers', exchange=server_exchange)
connection = BrokerConnection(
hostname='localhost',
userid='gamelion',
password='******',
virtual_host='/'
)
channel = connection.channel()
consumer = Consumer(channel, server_queue)
consumer.register_callback(receive_message)
consumer.consume(no_ack=True)
while True:
try:
connection.drain_events(timeout=CONSUME_TIMEOUT)
except socket.timeout:
# flush the server buffer if we haven't gotten
# any messages in a while
if len(server_buffer) > 0:
logging.debug('QUEUE TIMED OUT, FLUSHING BUFFER')
process_servers()
elif display_waiting:
class test_Consumer(TestCase):
def setUp(self):
self.connection = BrokerConnection(transport=Transport)
self.connection.connect()
self.assertTrue(self.connection.connection.connected)
self.exchange = Exchange("foo", "direct")
def test_set_no_ack(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True, no_ack=True)
self.assertTrue(consumer.no_ack)
def test_add_queue_when_auto_declare(self):
consumer = self.connection.Consumer(auto_declare=True)
q = Mock()
q.return_value = q
consumer.add_queue(q)
self.assertIn(q, consumer.queues)
q.declare.assert_called_with()
def test_add_queue_when_not_auto_declare(self):
consumer = self.connection.Consumer(auto_declare=False)
q = Mock()
q.return_value = q
consumer.add_queue(q)
self.assertIn(q, consumer.queues)
self.assertFalse(q.declare.call_count)
def test_consume_without_queues_returns(self):
consumer = self.connection.Consumer()
consumer.queues[:] = []
self.assertIsNone(consumer.consume())
def test_consuming_from(self):
consumer = self.connection.Consumer()
consumer.queues[:] = [Queue("a"), Queue("b")]
self.assertFalse(consumer.consuming_from(Queue("c")))
self.assertFalse(consumer.consuming_from("c"))
self.assertTrue(consumer.consuming_from(Queue("a")))
self.assertTrue(consumer.consuming_from(Queue("b")))
self.assertTrue(consumer.consuming_from("b"))
def test_receive_callback_without_m2p(self):
channel = self.connection.channel()
c = channel.Consumer()
m2p = getattr(channel, "message_to_python")
channel.message_to_python = None
try:
message = Mock()
message.decode.return_value = "Hello"
recv = c.receive = Mock()
c._receive_callback(message)
recv.assert_called_with("Hello", message)
finally:
channel.message_to_python = m2p
def test_set_callbacks(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
callbacks = [lambda x, y: x,
lambda x, y: x]
consumer = Consumer(channel, queue, auto_declare=True,
callbacks=callbacks)
self.assertEqual(consumer.callbacks, callbacks)
def test_auto_declare(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.consume() # twice is a noop
self.assertIsNot(consumer.queues[0], queue)
self.assertTrue(consumer.queues[0].is_bound)
self.assertTrue(consumer.queues[0].exchange.is_bound)
self.assertIsNot(consumer.queues[0].exchange, self.exchange)
for meth in ("exchange_declare",
"queue_declare",
"queue_bind",
"basic_consume"):
self.assertIn(meth, channel)
self.assertEqual(channel.called.count("basic_consume"), 1)
self.assertTrue(consumer._active_tags)
consumer.cancel_by_queue(queue.name)
consumer.cancel_by_queue(queue.name)
self.assertFalse(consumer._active_tags)
def test_manual_declare(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=False)
self.assertIsNot(consumer.queues[0], queue)
self.assertTrue(consumer.queues[0].is_bound)
self.assertTrue(consumer.queues[0].exchange.is_bound)
self.assertIsNot(consumer.queues[0].exchange, self.exchange)
for meth in ("exchange_declare",
"queue_declare",
"basic_consume"):
self.assertNotIn(meth, channel)
consumer.declare()
for meth in ("exchange_declare",
"queue_declare",
"queue_bind"):
self.assertIn(meth, channel)
self.assertNotIn("basic_consume", channel)
consumer.consume()
self.assertIn("basic_consume", channel)
def test_consume__cancel(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.cancel()
self.assertIn("basic_cancel", channel)
self.assertFalse(consumer._active_tags)
def test___enter____exit__(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
context = consumer.__enter__()
self.assertIs(context, consumer)
self.assertTrue(consumer._active_tags)
res = consumer.__exit__(None, None, None)
self.assertFalse(res)
self.assertIn("basic_cancel", channel)
self.assertFalse(consumer._active_tags)
def test_flow(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
consumer.flow(False)
self.assertIn("flow", channel)
def test_qos(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
consumer.qos(30, 10, False)
self.assertIn("basic_qos", channel)
def test_purge(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
b2 = Queue("qname2", self.exchange, "rkey")
b3 = Queue("qname3", self.exchange, "rkey")
b4 = Queue("qname4", self.exchange, "rkey")
consumer = Consumer(channel, [b1, b2, b3, b4], auto_declare=True)
consumer.purge()
self.assertEqual(channel.called.count("queue_purge"), 4)
def test_multiple_queues(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
b2 = Queue("qname2", self.exchange, "rkey")
b3 = Queue("qname3", self.exchange, "rkey")
b4 = Queue("qname4", self.exchange, "rkey")
consumer = Consumer(channel, [b1, b2, b3, b4])
consumer.consume()
self.assertEqual(channel.called.count("exchange_declare"), 4)
self.assertEqual(channel.called.count("queue_declare"), 4)
self.assertEqual(channel.called.count("queue_bind"), 4)
self.assertEqual(channel.called.count("basic_consume"), 4)
self.assertEqual(len(consumer._active_tags), 4)
consumer.cancel()
self.assertEqual(channel.called.count("basic_cancel"), 4)
self.assertFalse(len(consumer._active_tags))
def test_receive_callback(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
received = []
def callback(message_data, message):
received.append(message_data)
message.ack()
message.payload # trigger cache
consumer.register_callback(callback)
consumer._receive_callback({u"foo": u"bar"})
self.assertIn("basic_ack", channel)
self.assertIn("message_to_python", channel)
self.assertEqual(received[0], {u"foo": u"bar"})
def test_basic_ack_twice(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.ack()
message.ack()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({"foo": "bar"})
def test_basic_reject(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
consumer.register_callback(callback)
consumer._receive_callback({"foo": "bar"})
self.assertIn("basic_reject", channel)
def test_basic_reject_twice(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
message.reject()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({"foo": "bar"})
self.assertIn("basic_reject", channel)
def test_basic_reject__requeue(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
consumer.register_callback(callback)
consumer._receive_callback({"foo": "bar"})
self.assertIn("basic_reject:requeue", channel)
def test_basic_reject__requeue_twice(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
message.requeue()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({"foo": "bar"})
self.assertIn("basic_reject:requeue", channel)
def test_receive_without_callbacks_raises(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
with self.assertRaises(NotImplementedError):
consumer.receive(1, 2)
def test_decode_error(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
consumer.channel.throw_decode_error = True
with self.assertRaises(ValueError):
consumer._receive_callback({"foo": "bar"})
def test_on_decode_error_callback(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
thrown = []
def on_decode_error(msg, exc):
thrown.append((msg.body, exc))
consumer = Consumer(channel, [b1], on_decode_error=on_decode_error)
consumer.channel.throw_decode_error = True
consumer._receive_callback({"foo": "bar"})
self.assertTrue(thrown)
m, exc = thrown[0]
self.assertEqual(anyjson.loads(m), {"foo": "bar"})
self.assertIsInstance(exc, ValueError)
def test_recover(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
consumer.recover()
self.assertIn("basic_recover", channel)
def test_revive(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
channel2 = self.connection.channel()
consumer.revive(channel2)
self.assertIs(consumer.channel, channel2)
self.assertIs(consumer.queues[0].channel, channel2)
self.assertIs(consumer.queues[0].exchange.channel, channel2)
def test__repr__(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
self.assertTrue(repr(Consumer(channel, [b1])))
def test_connection_property_handles_AttributeError(self):
p = self.connection.Consumer()
p.channel = object()
self.assertIsNone(p.connection)
class test_Mailbox(unittest.TestCase):
def _handler(self, state):
return self.stats["var"]
def setUp(self):
class Mailbox(pidbox.Mailbox):
def _collect(self, *args, **kwargs):
return "COLLECTED"
self.mailbox = Mailbox("test_pidbox")
self.connection = BrokerConnection(transport="memory")
self.state = {"var": 1}
self.handlers = {"mymethod": self._handler}
self.bound = self.mailbox(self.connection)
self.default_chan = self.connection.channel()
self.node = self.bound.Node("test_pidbox", state=self.state,
handlers=self.handlers,
channel=self.default_chan)
def test_reply__collect(self):
mailbox = pidbox.Mailbox("test_reply__collect")(self.connection)
exchange = mailbox.reply_exchange.name
ticket = gen_unique_id()
mailbox.get_reply_queue(ticket)(self.connection.channel()).declare()
mailbox._publish_reply({"foo": "bar"}, exchange, ticket)
_callback_called = [False]
def callback(body):
_callback_called[0] = True
channel = self.connection.channel()
reply = mailbox._collect(ticket, limit=1, callback=callback,
channel=channel)
self.assertEqual(reply, [{"foo": "bar"}])
self.assertTrue(_callback_called[0])
ticket = gen_unique_id()
mailbox.get_reply_queue(ticket)(self.connection.channel()).declare()
mailbox._publish_reply({"biz": "boz"}, exchange, ticket)
reply = mailbox._collect(ticket, limit=1, channel=channel)
self.assertEqual(reply, [{"biz": "boz"}])
def test_constructor(self):
self.assertIsNone(self.mailbox.connection)
self.assertTrue(self.mailbox.exchange.name)
self.assertTrue(self.mailbox.reply_exchange.name)
def test_bound(self):
bound = self.mailbox(self.connection)
self.assertIs(bound.connection, self.connection)
def test_Node(self):
self.assertTrue(self.node.hostname)
self.assertTrue(self.node.state)
self.assertIs(self.node.mailbox, self.bound)
self.assertTrue(self.handlers)
# No initial handlers
node2 = self.bound.Node("test_pidbox2", state=self.state)
self.assertDictEqual(node2.handlers, {})
def test_Node_consumer(self):
consumer1 = self.node.Consumer()
self.assertIs(consumer1.channel, self.default_chan)
self.assertTrue(consumer1.no_ack)
chan2 = self.connection.channel()
consumer2 = self.node.Consumer(channel=chan2, no_ack=False)
self.assertIs(consumer2.channel, chan2)
self.assertFalse(consumer2.no_ack)
def test_handler(self):
node = self.bound.Node("test_handler", state=self.state)
@node.handler
def my_handler_name(state):
return 42
self.assertIn("my_handler_name", node.handlers)
def test_dispatch(self):
node = self.bound.Node("test_dispatch", state=self.state)
@node.handler
def my_handler_name(state, x=None, y=None):
return x + y
self.assertEqual(node.dispatch("my_handler_name",
arguments={"x": 10, "y": 10}), 20)
def test_dispatch_raising_SystemExit(self):
node = self.bound.Node("test_dispatch_raising_SystemExit",
state=self.state)
@node.handler
def my_handler_name(state):
raise SystemExit
self.assertRaises(SystemExit,
node.dispatch, "my_handler_name")
def test_dispatch_raising(self):
node = self.bound.Node("test_dispatch_raising", state=self.state)
@node.handler
def my_handler_name(state):
raise KeyError("foo")
res = node.dispatch("my_handler_name")
self.assertIn("error", res)
self.assertIn("KeyError", res["error"])
def test_dispatch_replies(self):
_replied = [False]
def reply(data, **options):
_replied[0] = True
node = self.bound.Node("test_dispatch", state=self.state)
node.reply = reply
@node.handler
def my_handler_name(state, x=None, y=None):
return x + y
node.dispatch("my_handler_name",
arguments={"x": 10, "y": 10},
reply_to={"exchange": "foo", "routing_key": "bar"})
self.assertTrue(_replied[0])
def test_reply(self):
_replied = [(None, None, None)]
def publish_reply(data, exchange, routing_key, **kwargs):
_replied[0] = (data, exchange, routing_key)
mailbox = self.mailbox(self.connection)
mailbox._publish_reply = publish_reply
node = mailbox.Node("test_reply")
@node.handler
def my_handler_name(state):
return 42
node.dispatch("my_handler_name",
reply_to={"exchange": "exchange",
"routing_key": "rkey"})
data, exchange, routing_key = _replied[0]
self.assertEqual(data, {"test_reply": 42})
self.assertEqual(exchange, "exchange")
self.assertEqual(routing_key, "rkey")
def test_handle_message(self):
node = self.bound.Node("test_dispatch_from_message")
@node.handler
def my_handler_name(state, x=None, y=None):
return x * y
body = {"method": "my_handler_name",
"arguments": {"x": 64, "y": 64}}
self.assertEqual(node.handle_message(body, None), 64 * 64)
# message not for me should not be processed.
body["destination"] = ["some_other_node"]
self.assertIsNone(node.handle_message(body, None))
def test_listen(self):
consumer = self.node.listen()
self.assertEqual(consumer.callbacks[0],
self.node.handle_message)
self.assertEqual(consumer.channel, self.default_chan)
def test_cast(self):
self.bound.cast(["somenode"], "mymethod")
consumer = self.node.Consumer()
self.assertIsCast(self.get_next(consumer))
def test_abcast(self):
self.bound.abcast("mymethod")
consumer = self.node.Consumer()
self.assertIsCast(self.get_next(consumer))
def test_call_destination_must_be_sequence(self):
self.assertRaises(ValueError,
self.bound.call, "some_node", "mymethod")
def test_call(self):
self.assertEqual(self.bound.call(["some_node"], "mymethod"),
"COLLECTED")
consumer = self.node.Consumer()
self.assertIsCall(self.get_next(consumer))
def test_multi_call(self):
self.assertEqual(self.bound.multi_call("mymethod"), "COLLECTED")
consumer = self.node.Consumer()
self.assertIsCall(self.get_next(consumer))
def get_next(self, consumer):
m = consumer.queues[0].get()
if m:
return m.payload
def assertIsCast(self, message):
self.assertTrue(message["method"])
def assertIsCall(self, message):
self.assertTrue(message["method"])
self.assertTrue(message["reply_to"])
class test_Producer(TestCase):
def setUp(self):
self.exchange = Exchange("foo", "direct")
self.connection = BrokerConnection(transport=Transport)
self.connection.connect()
self.assertTrue(self.connection.connection.connected)
self.assertFalse(self.exchange.is_bound)
@patch("kombu.common.maybe_declare")
def test_maybe_declare(self, maybe_declare):
p = self.connection.Producer()
q = Queue("foo")
p.maybe_declare(q)
maybe_declare.assert_called_with(q, p.channel, False)
@patch("kombu.common.maybe_declare")
def test_maybe_declare_when_entity_false(self, maybe_declare):
p = self.connection.Producer()
p.maybe_declare(None)
self.assertFalse(maybe_declare.called)
def test_auto_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=True)
self.assertIsNot(p.exchange, self.exchange,
"creates Exchange clone at bind")
self.assertTrue(p.exchange.is_bound)
self.assertIn("exchange_declare", channel,
"auto_declare declares exchange")
def test_manual_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=False)
self.assertTrue(p.exchange.is_bound)
self.assertNotIn("exchange_declare", channel,
"auto_declare=False does not declare exchange")
p.declare()
self.assertIn("exchange_declare", channel,
"p.declare() declares exchange")
def test_prepare(self):
message = {u"the quick brown fox": u"jumps over the lazy dog"}
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
m, ctype, cencoding = p._prepare(message, headers={})
self.assertDictEqual(message, anyjson.loads(m))
self.assertEqual(ctype, "application/json")
self.assertEqual(cencoding, "utf-8")
def test_prepare_compression(self):
message = {u"the quick brown fox": u"jumps over the lazy dog"}
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
headers = {}
m, ctype, cencoding = p._prepare(message, compression="zlib",
headers=headers)
self.assertEqual(ctype, "application/json")
self.assertEqual(cencoding, "utf-8")
self.assertEqual(headers["compression"], "application/x-gzip")
import zlib
self.assertEqual(anyjson.loads(
zlib.decompress(m).decode("utf-8")), message)
def test_prepare_custom_content_type(self):
message = "the quick brown fox".encode("utf-8")
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
m, ctype, cencoding = p._prepare(message, content_type="custom")
self.assertEqual(m, message)
self.assertEqual(ctype, "custom")
self.assertEqual(cencoding, "binary")
m, ctype, cencoding = p._prepare(message, content_type="custom",
content_encoding="alien")
self.assertEqual(m, message)
self.assertEqual(ctype, "custom")
self.assertEqual(cencoding, "alien")
def test_prepare_is_already_unicode(self):
message = u"the quick brown fox"
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
m, ctype, cencoding = p._prepare(message, content_type="text/plain")
self.assertEqual(m, message.encode("utf-8"))
self.assertEqual(ctype, "text/plain")
self.assertEqual(cencoding, "utf-8")
m, ctype, cencoding = p._prepare(message, content_type="text/plain",
content_encoding="utf-8")
self.assertEqual(m, message.encode("utf-8"))
self.assertEqual(ctype, "text/plain")
self.assertEqual(cencoding, "utf-8")
def test_publish_with_Exchange_instance(self):
p = self.connection.Producer()
p.exchange.publish = Mock()
p.publish("hello", exchange=Exchange("foo"))
self.assertEqual(p.exchange.publish.call_args[0][4], "foo")
def test_publish_retry_with_declare(self):
p = self.connection.Producer()
p.maybe_declare = Mock()
ensure = p.connection.ensure = Mock()
ex = Exchange("foo")
p.publish("hello", exchange=ex, declare=[ex], retry=True,
retry_policy={"step": 4})
p.maybe_declare.assert_called_with(ex, True, step=4)
ensure.assert_called_with(p, p.exchange.publish, step=4)
def test_revive_when_channel_is_connection(self):
p = self.connection.Producer()
p.exchange = Mock()
new_conn = BrokerConnection("memory://")
defchan = new_conn.default_channel
p.revive(new_conn)
self.assertIs(p.channel, defchan)
p.exchange.revive.assert_called_with(defchan)
def test_enter_exit(self):
p = self.connection.Producer()
p.release = Mock()
self.assertIs(p.__enter__(), p)
p.__exit__()
p.release.assert_called_with()
def test_connection_property_handles_AttributeError(self):
p = self.connection.Producer()
p.channel = object()
self.assertIsNone(p.connection)
def test_publish(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
message = {u"the quick brown fox": u"jumps over the lazy dog"}
ret = p.publish(message, routing_key="process")
self.assertIn("prepare_message", channel)
self.assertIn("basic_publish", channel)
m, exc, rkey = ret
self.assertDictEqual(message, anyjson.loads(m["body"]))
self.assertDictContainsSubset({"content_type": "application/json",
"content_encoding": "utf-8",
"priority": 0}, m)
self.assertDictContainsSubset({"delivery_mode": 2}, m["properties"])
self.assertEqual(exc, p.exchange.name)
self.assertEqual(rkey, "process")
def test_no_exchange(self):
chan = self.connection.channel()
p = Producer(chan)
self.assertFalse(p.exchange.name)
def test_revive(self):
chan = self.connection.channel()
p = Producer(chan)
chan2 = self.connection.channel()
p.revive(chan2)
self.assertIs(p.channel, chan2)
self.assertIs(p.exchange.channel, chan2)
def test_on_return(self):
chan = self.connection.channel()
def on_return(exception, exchange, routing_key, message):
pass
p = Producer(chan, on_return=on_return)
self.assertTrue(on_return in chan.events["basic_return"])
self.assertTrue(p.on_return)
class test_Producer(unittest.TestCase):
def setUp(self):
self.exchange = Exchange("foo", "direct")
self.connection = BrokerConnection(transport=Transport)
self.connection.connect()
self.assertTrue(self.connection.connection.connected)
self.assertFalse(self.exchange.is_bound)
def test_auto_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=True)
self.assertIsNot(p.exchange, self.exchange,
"creates Exchange clone at bind")
self.assertTrue(p.exchange.is_bound)
self.assertIn("exchange_declare", channel,
"auto_declare declares exchange")
def test_manual_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=False)
self.assertTrue(p.exchange.is_bound)
self.assertNotIn("exchange_declare", channel,
"auto_declare=False does not declare exchange")
p.declare()
self.assertIn("exchange_declare", channel,
"p.declare() declares exchange")
def test_prepare(self):
message = {u"the quick brown fox": u"jumps over the lazy dog"}
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
m, ctype, cencoding = p._prepare(message, headers={})
self.assertDictEqual(message, anyjson.deserialize(m))
self.assertEqual(ctype, "application/json")
self.assertEqual(cencoding, "utf-8")
def test_prepare_compression(self):
message = {u"the quick brown fox": u"jumps over the lazy dog"}
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
headers = {}
m, ctype, cencoding = p._prepare(message, compression="zlib",
headers=headers)
self.assertEqual(ctype, "application/json")
self.assertEqual(cencoding, "utf-8")
self.assertEqual(headers["compression"], "application/x-gzip")
import zlib
self.assertEqual(anyjson.deserialize(
zlib.decompress(m).decode("utf-8")), message)
def test_prepare_custom_content_type(self):
message = "the quick brown fox".encode("utf-8")
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
m, ctype, cencoding = p._prepare(message, content_type="custom")
self.assertEqual(m, message)
self.assertEqual(ctype, "custom")
self.assertEqual(cencoding, "binary")
m, ctype, cencoding = p._prepare(message, content_type="custom",
content_encoding="alien")
self.assertEqual(m, message)
self.assertEqual(ctype, "custom")
self.assertEqual(cencoding, "alien")
def test_prepare_is_already_unicode(self):
message = u"the quick brown fox"
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
m, ctype, cencoding = p._prepare(message, content_type="text/plain")
self.assertEqual(m, message.encode("utf-8"))
self.assertEqual(ctype, "text/plain")
self.assertEqual(cencoding, "utf-8")
m, ctype, cencoding = p._prepare(message, content_type="text/plain",
content_encoding="utf-8")
self.assertEqual(m, message.encode("utf-8"))
self.assertEqual(ctype, "text/plain")
self.assertEqual(cencoding, "utf-8")
def test_publish(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, serializer="json")
message = {u"the quick brown fox": u"jumps over the lazy dog"}
ret = p.publish(message, routing_key="process")
self.assertIn("prepare_message", channel)
self.assertIn("basic_publish", channel)
m, exc, rkey = ret
self.assertDictEqual(message, anyjson.deserialize(m["body"]))
self.assertDictContainsSubset({"content_type": "application/json",
"content_encoding": "utf-8",
"priority": 0}, m)
self.assertDictContainsSubset({"delivery_mode": 2}, m["properties"])
self.assertEqual(exc, p.exchange.name)
self.assertEqual(rkey, "process")
def test_no_exchange(self):
chan = self.connection.channel()
p = Producer(chan)
self.assertFalse(p.exchange.name)
def test_revive(self):
chan = self.connection.channel()
p = Producer(chan)
chan2 = self.connection.channel()
p.revive(chan2)
self.assertIs(p.channel, chan2)
self.assertIs(p.exchange.channel, chan2)
def test_on_return(self):
chan = self.connection.channel()
def on_return(exception, exchange, routing_key, message):
pass
p = Producer(chan, on_return=on_return)
self.assertTrue(on_return in chan.events["basic_return"])
self.assertTrue(p.on_return)
class test_Consumer(unittest.TestCase):
def setUp(self):
self.connection = BrokerConnection(transport=Transport)
self.connection.connect()
self.assertTrue(self.connection.connection.connected)
self.exchange = Exchange("foo", "direct")
def test_set_no_ack(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True, no_ack=True)
self.assertTrue(consumer.no_ack)
def test_set_callbacks(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
callbacks = [lambda x, y: x,
lambda x, y: x]
consumer = Consumer(channel, queue, auto_declare=True,
callbacks=callbacks)
self.assertEqual(consumer.callbacks, callbacks)
def test_auto_declare(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.consume() # twice is a noop
self.assertIsNot(consumer.queues[0], queue)
self.assertTrue(consumer.queues[0].is_bound)
self.assertTrue(consumer.queues[0].exchange.is_bound)
self.assertIsNot(consumer.queues[0].exchange, self.exchange)
for meth in ("exchange_declare",
"queue_declare",
"queue_bind",
"basic_consume"):
self.assertIn(meth, channel)
self.assertEqual(channel.called.count("basic_consume"), 1)
self.assertTrue(consumer._active_tags)
consumer.cancel_by_queue(queue.name)
consumer.cancel_by_queue(queue.name)
self.assertFalse(consumer._active_tags)
def test_manual_declare(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=False)
self.assertIsNot(consumer.queues[0], queue)
self.assertTrue(consumer.queues[0].is_bound)
self.assertTrue(consumer.queues[0].exchange.is_bound)
self.assertIsNot(consumer.queues[0].exchange, self.exchange)
for meth in ("exchange_declare",
"queue_declare",
"basic_consume"):
self.assertNotIn(meth, channel)
consumer.declare()
for meth in ("exchange_declare",
"queue_declare",
"queue_bind"):
self.assertIn(meth, channel)
self.assertNotIn("basic_consume", channel)
consumer.consume()
self.assertIn("basic_consume", channel)
def test_consume__cancel(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.cancel()
self.assertIn("basic_cancel", channel)
self.assertFalse(consumer._active_tags)
def test___enter____exit__(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
context = consumer.__enter__()
self.assertIs(context, consumer)
self.assertTrue(consumer._active_tags)
consumer.__exit__()
self.assertIn("basic_cancel", channel)
self.assertFalse(consumer._active_tags)
def test_flow(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
consumer.flow(False)
self.assertIn("flow", channel)
def test_qos(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
consumer.qos(30, 10, False)
self.assertIn("basic_qos", channel)
def test_purge(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
b2 = Queue("qname2", self.exchange, "rkey")
b3 = Queue("qname3", self.exchange, "rkey")
b4 = Queue("qname4", self.exchange, "rkey")
consumer = Consumer(channel, [b1, b2, b3, b4], auto_declare=True)
consumer.purge()
self.assertEqual(channel.called.count("queue_purge"), 4)
def test_multiple_queues(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
b2 = Queue("qname2", self.exchange, "rkey")
b3 = Queue("qname3", self.exchange, "rkey")
b4 = Queue("qname4", self.exchange, "rkey")
consumer = Consumer(channel, [b1, b2, b3, b4])
consumer.consume()
self.assertEqual(channel.called.count("exchange_declare"), 4)
self.assertEqual(channel.called.count("queue_declare"), 4)
self.assertEqual(channel.called.count("queue_bind"), 4)
self.assertEqual(channel.called.count("basic_consume"), 4)
self.assertEqual(len(consumer._active_tags), 4)
consumer.cancel()
self.assertEqual(channel.called.count("basic_cancel"), 4)
self.assertFalse(len(consumer._active_tags))
def test_receive_callback(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
received = []
def callback(message_data, message):
received.append(message_data)
message.ack()
message.payload # trigger cache
consumer.register_callback(callback)
consumer._receive_callback({u"foo": u"bar"})
self.assertIn("basic_ack", channel)
self.assertIn("message_to_python", channel)
self.assertEqual(received[0], {u"foo": u"bar"})
def test_basic_ack_twice(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.ack()
message.ack()
consumer.register_callback(callback)
self.assertRaises(MessageStateError,
consumer._receive_callback, {"foo": "bar"})
def test_basic_reject(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
consumer.register_callback(callback)
consumer._receive_callback({"foo": "bar"})
self.assertIn("basic_reject", channel)
def test_basic_reject_twice(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
message.reject()
consumer.register_callback(callback)
self.assertRaises(MessageStateError,
consumer._receive_callback, {"foo": "bar"})
self.assertIn("basic_reject", channel)
def test_basic_reject__requeue(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
consumer.register_callback(callback)
consumer._receive_callback({"foo": "bar"})
self.assertIn("basic_reject:requeue", channel)
def test_basic_reject__requeue_twice(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
message.requeue()
consumer.register_callback(callback)
self.assertRaises(MessageStateError,
consumer._receive_callback, {"foo": "bar"})
self.assertIn("basic_reject:requeue", channel)
def test_receive_without_callbacks_raises(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
self.assertRaises(NotImplementedError, consumer.receive, 1, 2)
def test_decode_error(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
consumer.channel.throw_decode_error = True
self.assertRaises(ValueError,
consumer._receive_callback, {"foo": "bar"})
def test_on_decode_error_callback(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
thrown = []
def on_decode_error(msg, exc):
thrown.append((msg.body, exc))
consumer = Consumer(channel, [b1], on_decode_error=on_decode_error)
consumer.channel.throw_decode_error = True
consumer._receive_callback({"foo": "bar"})
self.assertTrue(thrown)
m, exc = thrown[0]
self.assertEqual(anyjson.deserialize(m), {"foo": "bar"})
self.assertIsInstance(exc, ValueError)
def test_recover(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
consumer.recover()
self.assertIn("basic_recover", channel)
def test_revive(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
channel2 = self.connection.channel()
consumer.revive(channel2)
self.assertIs(consumer.channel, channel2)
self.assertIs(consumer.queues[0].channel, channel2)
self.assertIs(consumer.queues[0].exchange.channel, channel2)
def test__repr__(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
self.assertTrue(repr(Consumer(channel, [b1])))
class test_MemoryTransport(unittest.TestCase):
def setUp(self):
self.c = BrokerConnection(transport="memory")
self.e = Exchange("test_transport_memory")
self.q = Queue("test_transport_memory",
exchange=self.e,
routing_key="test_transport_memory")
self.q2 = Queue("test_transport_memory2",
exchange=self.e,
routing_key="test_transport_memory2")
def test_produce_consume_noack(self):
channel = self.c.channel()
producer = Producer(channel, self.e)
consumer = Consumer(channel, self.q, no_ack=True)
for i in range(10):
producer.publish({"foo": i}, routing_key="test_transport_memory")
_received = []
def callback(message_data, message):
_received.append(message)
consumer.register_callback(callback)
consumer.consume()
while 1:
if len(_received) == 10:
break
self.c.drain_events()
self.assertEqual(len(_received), 10)
def test_produce_consume(self):
channel = self.c.channel()
producer = Producer(channel, self.e)
consumer1 = Consumer(channel, self.q)
consumer2 = Consumer(channel, self.q2)
self.q2(channel).declare()
for i in range(10):
producer.publish({"foo": i}, routing_key="test_transport_memory")
for i in range(10):
producer.publish({"foo": i}, routing_key="test_transport_memory2")
_received1 = []
_received2 = []
def callback1(message_data, message):
_received1.append(message)
message.ack()
def callback2(message_data, message):
_received2.append(message)
message.ack()
consumer1.register_callback(callback1)
consumer2.register_callback(callback2)
consumer1.consume()
consumer2.consume()
while 1:
if len(_received1) + len(_received2) == 20:
break
self.c.drain_events()
self.assertEqual(len(_received1) + len(_received2), 20)
# compression
producer.publish({"compressed": True},
routing_key="test_transport_memory",
compression="zlib")
m = self.q(channel).get()
self.assertDictEqual(m.payload, {"compressed": True})
# queue.delete
for i in range(10):
producer.publish({"foo": i}, routing_key="test_transport_memory")
self.assertTrue(self.q(channel).get())
self.q(channel).delete()
self.q(channel).declare()
self.assertIsNone(self.q(channel).get())
# queue.purge
for i in range(10):
producer.publish({"foo": i}, routing_key="test_transport_memory2")
self.assertTrue(self.q2(channel).get())
self.q2(channel).purge()
self.assertIsNone(self.q2(channel).get())
def test_drain_events(self):
self.assertRaises(ValueError, self.c.drain_events, timeout=0.1)
c1 = self.c.channel()
c2 = self.c.channel()
self.assertRaises(socket.timeout, self.c.drain_events, timeout=0.1)
del(c1) # so pyflakes doesn't complain.
del(c2)
def test_drain_events_unregistered_queue(self):
c1 = self.c.channel()
class Cycle(object):
def get(self):
return ("foo", "foo"), c1
self.c.transport.cycle = Cycle()
self.assertRaises(KeyError, self.c.drain_events)
class test_Consumer(TestCase):
def setUp(self):
self.connection = BrokerConnection(transport=Transport)
self.connection.connect()
self.assertTrue(self.connection.connection.connected)
self.exchange = Exchange("foo", "direct")
def test_set_no_ack(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True, no_ack=True)
self.assertTrue(consumer.no_ack)
def test_add_queue_when_auto_declare(self):
consumer = self.connection.Consumer(auto_declare=True)
q = Mock()
q.return_value = q
consumer.add_queue(q)
self.assertIn(q, consumer.queues)
q.declare.assert_called_with()
def test_add_queue_when_not_auto_declare(self):
consumer = self.connection.Consumer(auto_declare=False)
q = Mock()
q.return_value = q
consumer.add_queue(q)
self.assertIn(q, consumer.queues)
self.assertFalse(q.declare.call_count)
def test_consume_without_queues_returns(self):
consumer = self.connection.Consumer()
consumer.queues[:] = []
self.assertIsNone(consumer.consume())
def test_consuming_from(self):
consumer = self.connection.Consumer()
consumer.queues[:] = [Queue("a"), Queue("b")]
self.assertFalse(consumer.consuming_from(Queue("c")))
self.assertFalse(consumer.consuming_from("c"))
self.assertTrue(consumer.consuming_from(Queue("a")))
self.assertTrue(consumer.consuming_from(Queue("b")))
self.assertTrue(consumer.consuming_from("b"))
def test_receive_callback_without_m2p(self):
channel = self.connection.channel()
c = channel.Consumer()
m2p = getattr(channel, "message_to_python")
channel.message_to_python = None
try:
message = Mock()
message.decode.return_value = "Hello"
recv = c.receive = Mock()
c._receive_callback(message)
recv.assert_called_with("Hello", message)
finally:
channel.message_to_python = m2p
def test_set_callbacks(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
callbacks = [lambda x, y: x, lambda x, y: x]
consumer = Consumer(channel,
queue,
auto_declare=True,
callbacks=callbacks)
self.assertEqual(consumer.callbacks, callbacks)
def test_auto_declare(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.consume() # twice is a noop
self.assertIsNot(consumer.queues[0], queue)
self.assertTrue(consumer.queues[0].is_bound)
self.assertTrue(consumer.queues[0].exchange.is_bound)
self.assertIsNot(consumer.queues[0].exchange, self.exchange)
for meth in ("exchange_declare", "queue_declare", "queue_bind",
"basic_consume"):
self.assertIn(meth, channel)
self.assertEqual(channel.called.count("basic_consume"), 1)
self.assertTrue(consumer._active_tags)
consumer.cancel_by_queue(queue.name)
consumer.cancel_by_queue(queue.name)
self.assertFalse(consumer._active_tags)
def test_manual_declare(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=False)
self.assertIsNot(consumer.queues[0], queue)
self.assertTrue(consumer.queues[0].is_bound)
self.assertTrue(consumer.queues[0].exchange.is_bound)
self.assertIsNot(consumer.queues[0].exchange, self.exchange)
for meth in ("exchange_declare", "queue_declare", "basic_consume"):
self.assertNotIn(meth, channel)
consumer.declare()
for meth in ("exchange_declare", "queue_declare", "queue_bind"):
self.assertIn(meth, channel)
self.assertNotIn("basic_consume", channel)
consumer.consume()
self.assertIn("basic_consume", channel)
def test_consume__cancel(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
consumer.consume()
consumer.cancel()
self.assertIn("basic_cancel", channel)
self.assertFalse(consumer._active_tags)
def test___enter____exit__(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
context = consumer.__enter__()
self.assertIs(context, consumer)
self.assertTrue(consumer._active_tags)
res = consumer.__exit__(None, None, None)
self.assertFalse(res)
self.assertIn("basic_cancel", channel)
self.assertFalse(consumer._active_tags)
def test_flow(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
consumer.flow(False)
self.assertIn("flow", channel)
def test_qos(self):
channel = self.connection.channel()
queue = Queue("qname", self.exchange, "rkey")
consumer = Consumer(channel, queue, auto_declare=True)
consumer.qos(30, 10, False)
self.assertIn("basic_qos", channel)
def test_purge(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
b2 = Queue("qname2", self.exchange, "rkey")
b3 = Queue("qname3", self.exchange, "rkey")
b4 = Queue("qname4", self.exchange, "rkey")
consumer = Consumer(channel, [b1, b2, b3, b4], auto_declare=True)
consumer.purge()
self.assertEqual(channel.called.count("queue_purge"), 4)
def test_multiple_queues(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
b2 = Queue("qname2", self.exchange, "rkey")
b3 = Queue("qname3", self.exchange, "rkey")
b4 = Queue("qname4", self.exchange, "rkey")
consumer = Consumer(channel, [b1, b2, b3, b4])
consumer.consume()
self.assertEqual(channel.called.count("exchange_declare"), 4)
self.assertEqual(channel.called.count("queue_declare"), 4)
self.assertEqual(channel.called.count("queue_bind"), 4)
self.assertEqual(channel.called.count("basic_consume"), 4)
self.assertEqual(len(consumer._active_tags), 4)
consumer.cancel()
self.assertEqual(channel.called.count("basic_cancel"), 4)
self.assertFalse(len(consumer._active_tags))
def test_receive_callback(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
received = []
def callback(message_data, message):
received.append(message_data)
message.ack()
message.payload # trigger cache
consumer.register_callback(callback)
consumer._receive_callback({u"foo": u"bar"})
self.assertIn("basic_ack", channel)
self.assertIn("message_to_python", channel)
self.assertEqual(received[0], {u"foo": u"bar"})
def test_basic_ack_twice(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.ack()
message.ack()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({"foo": "bar"})
def test_basic_reject(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
consumer.register_callback(callback)
consumer._receive_callback({"foo": "bar"})
self.assertIn("basic_reject", channel)
def test_basic_reject_twice(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.reject()
message.reject()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({"foo": "bar"})
self.assertIn("basic_reject", channel)
def test_basic_reject__requeue(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
consumer.register_callback(callback)
consumer._receive_callback({"foo": "bar"})
self.assertIn("basic_reject:requeue", channel)
def test_basic_reject__requeue_twice(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
def callback(message_data, message):
message.requeue()
message.requeue()
consumer.register_callback(callback)
with self.assertRaises(MessageStateError):
consumer._receive_callback({"foo": "bar"})
self.assertIn("basic_reject:requeue", channel)
def test_receive_without_callbacks_raises(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
with self.assertRaises(NotImplementedError):
consumer.receive(1, 2)
def test_decode_error(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
consumer.channel.throw_decode_error = True
with self.assertRaises(ValueError):
consumer._receive_callback({"foo": "bar"})
def test_on_decode_error_callback(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
thrown = []
def on_decode_error(msg, exc):
thrown.append((msg.body, exc))
consumer = Consumer(channel, [b1], on_decode_error=on_decode_error)
consumer.channel.throw_decode_error = True
consumer._receive_callback({"foo": "bar"})
self.assertTrue(thrown)
m, exc = thrown[0]
self.assertEqual(anyjson.loads(m), {"foo": "bar"})
self.assertIsInstance(exc, ValueError)
def test_recover(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
consumer.recover()
self.assertIn("basic_recover", channel)
def test_revive(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
consumer = Consumer(channel, [b1])
channel2 = self.connection.channel()
consumer.revive(channel2)
self.assertIs(consumer.channel, channel2)
self.assertIs(consumer.queues[0].channel, channel2)
self.assertIs(consumer.queues[0].exchange.channel, channel2)
def test__repr__(self):
channel = self.connection.channel()
b1 = Queue("qname1", self.exchange, "rkey")
self.assertTrue(repr(Consumer(channel, [b1])))
def test_connection_property_handles_AttributeError(self):
p = self.connection.Consumer()
p.channel = object()
self.assertIsNone(p.connection)
class test_Redis(TestCase):
def setUp(self):
self.connection = BrokerConnection(transport=Transport)
self.exchange = Exchange("test_Redis", type="direct")
self.queue = Queue("test_Redis", self.exchange, "test_Redis")
def tearDown(self):
self.connection.close()
def test_publish__get(self):
channel = self.connection.channel()
producer = Producer(channel, self.exchange, routing_key="test_Redis")
self.queue(channel).declare()
producer.publish({"hello": "world"})
self.assertDictEqual(
self.queue(channel).get().payload, {"hello": "world"})
self.assertIsNone(self.queue(channel).get())
self.assertIsNone(self.queue(channel).get())
self.assertIsNone(self.queue(channel).get())
def test_publish__consume(self):
connection = BrokerConnection(transport=Transport)
channel = connection.channel()
producer = Producer(channel, self.exchange, routing_key="test_Redis")
consumer = Consumer(channel, self.queue)
producer.publish({"hello2": "world2"})
_received = []
def callback(message_data, message):
_received.append(message_data)
message.ack()
consumer.register_callback(callback)
consumer.consume()
self.assertIn(channel, channel.connection.cycle._channels)
try:
connection.drain_events(timeout=1)
self.assertTrue(_received)
with self.assertRaises(socket.timeout):
connection.drain_events(timeout=0.01)
finally:
channel.close()
def test_purge(self):
channel = self.connection.channel()
producer = Producer(channel, self.exchange, routing_key="test_Redis")
self.queue(channel).declare()
for i in range(10):
producer.publish({"hello": "world-%s" % (i, )})
self.assertEqual(channel._size("test_Redis"), 10)
self.assertEqual(self.queue(channel).purge(), 10)
channel.close()
def test_db_values(self):
c1 = BrokerConnection(virtual_host=1, transport=Transport).channel()
self.assertEqual(c1.client.db, 1)
c2 = BrokerConnection(virtual_host="1", transport=Transport).channel()
self.assertEqual(c2.client.db, 1)
c3 = BrokerConnection(virtual_host="/1", transport=Transport).channel()
self.assertEqual(c3.client.db, 1)
with self.assertRaises(Exception):
BrokerConnection(virtual_host="/foo",
transport=Transport).channel()
def test_db_port(self):
c1 = BrokerConnection(port=None, transport=Transport).channel()
self.assertEqual(c1.client.port, Transport.default_port)
c1.close()
c2 = BrokerConnection(port=9999, transport=Transport).channel()
self.assertEqual(c2.client.port, 9999)
c2.close()
def test_close_poller_not_active(self):
c = BrokerConnection(transport=Transport).channel()
cycle = c.connection.cycle
c.client.connection
c.close()
self.assertNotIn(c, cycle._channels)
def test_close_ResponseError(self):
c = BrokerConnection(transport=Transport).channel()
c.client.bgsave_raises_ResponseError = True
c.close()
def test_close_disconnects(self):
c = BrokerConnection(transport=Transport).channel()
conn1 = c.client.connection
conn2 = c.subclient.connection
c.close()
self.assertTrue(conn1.disconnected)
self.assertTrue(conn2.disconnected)
def test_get__Empty(self):
channel = self.connection.channel()
with self.assertRaises(Empty):
channel._get("does-not-exist")
channel.close()
def test_get_client(self):
myredis, exceptions = _redis_modules()
@module_exists(myredis, exceptions)
def _do_test():
conn = BrokerConnection(transport=Transport)
chan = conn.channel()
self.assertTrue(chan.Client)
self.assertTrue(chan.ResponseError)
self.assertTrue(conn.transport.connection_errors)
self.assertTrue(conn.transport.channel_errors)
_do_test()
class test_Mailbox(TestCase):
def _handler(self, state):
return self.stats["var"]
def setUp(self):
class Mailbox(pidbox.Mailbox):
def _collect(self, *args, **kwargs):
return "COLLECTED"
self.mailbox = Mailbox("test_pidbox")
self.connection = BrokerConnection(transport="memory")
self.state = {"var": 1}
self.handlers = {"mymethod": self._handler}
self.bound = self.mailbox(self.connection)
self.default_chan = self.connection.channel()
self.node = self.bound.Node("test_pidbox",
state=self.state,
handlers=self.handlers,
channel=self.default_chan)
def test_reply__collect(self):
mailbox = pidbox.Mailbox("test_reply__collect")(self.connection)
exchange = mailbox.reply_exchange.name
ticket = uuid()
mailbox.get_reply_queue(ticket)(self.connection.channel()).declare()
mailbox._publish_reply({"foo": "bar"}, exchange, ticket)
_callback_called = [False]
def callback(body):
_callback_called[0] = True
channel = self.connection.channel()
reply = mailbox._collect(ticket,
limit=1,
callback=callback,
channel=channel)
self.assertEqual(reply, [{"foo": "bar"}])
self.assertTrue(_callback_called[0])
ticket = uuid()
mailbox.get_reply_queue(ticket)(self.connection.channel()).declare()
mailbox._publish_reply({"biz": "boz"}, exchange, ticket)
reply = mailbox._collect(ticket, limit=1, channel=channel)
self.assertEqual(reply, [{"biz": "boz"}])
de = mailbox.connection.drain_events = Mock()
de.side_effect = socket.timeout
mailbox._collect(ticket, limit=1, channel=channel)
def test_constructor(self):
self.assertIsNone(self.mailbox.connection)
self.assertTrue(self.mailbox.exchange.name)
self.assertTrue(self.mailbox.reply_exchange.name)
def test_bound(self):
bound = self.mailbox(self.connection)
self.assertIs(bound.connection, self.connection)
def test_Node(self):
self.assertTrue(self.node.hostname)
self.assertTrue(self.node.state)
self.assertIs(self.node.mailbox, self.bound)
self.assertTrue(self.handlers)
# No initial handlers
node2 = self.bound.Node("test_pidbox2", state=self.state)
self.assertDictEqual(node2.handlers, {})
def test_Node_consumer(self):
consumer1 = self.node.Consumer()
self.assertIs(consumer1.channel, self.default_chan)
self.assertTrue(consumer1.no_ack)
chan2 = self.connection.channel()
consumer2 = self.node.Consumer(channel=chan2, no_ack=False)
self.assertIs(consumer2.channel, chan2)
self.assertFalse(consumer2.no_ack)
def test_handler(self):
node = self.bound.Node("test_handler", state=self.state)
@node.handler
def my_handler_name(state):
return 42
self.assertIn("my_handler_name", node.handlers)
def test_dispatch(self):
node = self.bound.Node("test_dispatch", state=self.state)
@node.handler
def my_handler_name(state, x=None, y=None):
return x + y
self.assertEqual(
node.dispatch("my_handler_name", arguments={
"x": 10,
"y": 10
}), 20)
def test_dispatch_raising_SystemExit(self):
node = self.bound.Node("test_dispatch_raising_SystemExit",
state=self.state)
@node.handler
def my_handler_name(state):
raise SystemExit
with self.assertRaises(SystemExit):
node.dispatch("my_handler_name")
def test_dispatch_raising(self):
node = self.bound.Node("test_dispatch_raising", state=self.state)
@node.handler
def my_handler_name(state):
raise KeyError("foo")
res = node.dispatch("my_handler_name")
self.assertIn("error", res)
self.assertIn("KeyError", res["error"])
def test_dispatch_replies(self):
_replied = [False]
def reply(data, **options):
_replied[0] = True
node = self.bound.Node("test_dispatch", state=self.state)
node.reply = reply
@node.handler
def my_handler_name(state, x=None, y=None):
return x + y
node.dispatch("my_handler_name",
arguments={
"x": 10,
"y": 10
},
reply_to={
"exchange": "foo",
"routing_key": "bar"
})
self.assertTrue(_replied[0])
def test_reply(self):
_replied = [(None, None, None)]
def publish_reply(data, exchange, routing_key, **kwargs):
_replied[0] = (data, exchange, routing_key)
mailbox = self.mailbox(self.connection)
mailbox._publish_reply = publish_reply
node = mailbox.Node("test_reply")
@node.handler
def my_handler_name(state):
return 42
node.dispatch("my_handler_name",
reply_to={
"exchange": "exchange",
"routing_key": "rkey"
})
data, exchange, routing_key = _replied[0]
self.assertEqual(data, {"test_reply": 42})
self.assertEqual(exchange, "exchange")
self.assertEqual(routing_key, "rkey")
def test_handle_message(self):
node = self.bound.Node("test_dispatch_from_message")
@node.handler
def my_handler_name(state, x=None, y=None):
return x * y
body = {"method": "my_handler_name", "arguments": {"x": 64, "y": 64}}
self.assertEqual(node.handle_message(body, None), 64 * 64)
# message not for me should not be processed.
body["destination"] = ["some_other_node"]
self.assertIsNone(node.handle_message(body, None))
def test_listen(self):
consumer = self.node.listen()
self.assertEqual(consumer.callbacks[0], self.node.handle_message)
self.assertEqual(consumer.channel, self.default_chan)
def test_cast(self):
self.bound.cast(["somenode"], "mymethod")
consumer = self.node.Consumer()
self.assertIsCast(self.get_next(consumer))
def test_abcast(self):
self.bound.abcast("mymethod")
consumer = self.node.Consumer()
self.assertIsCast(self.get_next(consumer))
def test_call_destination_must_be_sequence(self):
with self.assertRaises(ValueError):
self.bound.call("some_node", "mymethod")
def test_call(self):
self.assertEqual(self.bound.call(["some_node"], "mymethod"),
"COLLECTED")
consumer = self.node.Consumer()
self.assertIsCall(self.get_next(consumer))
def test_multi_call(self):
self.assertEqual(self.bound.multi_call("mymethod"), "COLLECTED")
consumer = self.node.Consumer()
self.assertIsCall(self.get_next(consumer))
def get_next(self, consumer):
m = consumer.queues[0].get()
if m:
return m.payload
def assertIsCast(self, message):
self.assertTrue(message["method"])
def assertIsCall(self, message):
self.assertTrue(message["method"])
self.assertTrue(message["reply_to"])