def setUp(self):
global pulse_cfg
self.proc = None
self.publisher = None
self.guardian = PulseGuardian(warn_queue_size=TEST_WARN_SIZE,
del_queue_size=TEST_DELETE_SIZE,
emails=False)
dbinit.init_and_clear_db()
self.consumer_cfg = pulse_cfg.copy()
self.consumer_cfg['applabel'] = str(uuid.uuid1())
# Configure/create the test user to be used for message consumption.
self.consumer_cfg['user'] = CONSUMER_USER
self.consumer_cfg['password'] = CONSUMER_PASSWORD
self.user = User.new_user(email=CONSUMER_EMAIL, admin=False)
# As a default owner for rabbitmq_account in some tests where an owner
# is not provided.
self.admin = User.new_user(email=ADMIN_EMAIL, admin=True)
db_session.add(self.user)
db_session.commit()
self.rabbitmq_account = RabbitMQAccount.new_user(
username=CONSUMER_USER,
password=CONSUMER_PASSWORD,
owners=self.user)
db_session.add(self.rabbitmq_account)
db_session.commit()
def setUp(self):
global pulse_cfg
self.proc = None
self.publisher = None
self.guardian = PulseGuardian(warn_queue_size=TEST_WARN_SIZE,
del_queue_size=TEST_DELETE_SIZE,
emails=False)
dbinit.init_and_clear_db()
self.consumer_cfg = pulse_cfg.copy()
self.consumer_cfg['applabel'] = str(uuid.uuid1())
# Configure/create the test user to be used for message consumption.
self.consumer_cfg['user'] = CONSUMER_USER
self.consumer_cfg['password'] = CONSUMER_PASSWORD
self.user = User.new_user(email=CONSUMER_EMAIL, admin=False)
db_session.add(self.user)
db_session.commit()
self.pulse_user = PulseUser.new_user(username=CONSUMER_USER,
password=CONSUMER_PASSWORD,
owner=self.user)
db_session.add(self.pulse_user)
db_session.commit()
def setUp(self):
global pulse_cfg
self.proc = None
self.publisher = None
self.guardian = PulseGuardian(warn_queue_size=TEST_WARN_SIZE,
del_queue_size=TEST_DELETE_SIZE,
emails=False)
dbinit.init_and_clear_db()
self.consumer_cfg = pulse_cfg.copy()
self.consumer_cfg['applabel'] = str(uuid.uuid1())
# Configure/create the test user to be used for message consumption.
self.consumer_cfg['user'] = CONSUMER_USER
self.consumer_cfg['password'] = CONSUMER_PASSWORD
self.user = User.new_user(email=CONSUMER_EMAIL, admin=False)
# As a default owner for pulse_user in some tests where an owner is not
# provided.
self.admin = User.new_user(email=ADMIN_EMAIL, admin=True)
db_session.add(self.user)
db_session.commit()
self.pulse_user = PulseUser.new_user(
username=CONSUMER_USER,
password=CONSUMER_PASSWORD,
owners=self.user)
db_session.add(self.pulse_user)
db_session.commit()
def setUp(self):
global pulse_cfg
self.proc = None
self.publisher = None
self.management_api = PulseManagementAPI(
management_url='http://{}:{}/api'.format(
pulse_cfg['host'], pulse_cfg['management_port']),
user=pulse_cfg['user'],
password=pulse_cfg['password']
)
self.guardian = PulseGuardian(self.management_api,
warn_queue_size=TEST_WARN_SIZE,
del_queue_size=TEST_DELETE_SIZE,
emails=False)
# Hack in a test config.
dbinit.pulse_management = self.management_api
dbinit.init_and_clear_db()
self.consumer_cfg = pulse_cfg.copy()
self.consumer_cfg['applabel'] = str(uuid.uuid1())
# Configure/create the test user to be used for message consumption.
self.consumer_cfg['user'] = CONSUMER_USER
self.consumer_cfg['password'] = CONSUMER_PASSWORD
self.user = User.new_user(email=CONSUMER_EMAIL, admin=False)
db_session.add(self.user)
db_session.commit()
self.pulse_user = PulseUser.new_user(
username=CONSUMER_USER,
password=CONSUMER_PASSWORD,
owner=self.user,
management_api=self.management_api)
db_session.add(self.pulse_user)
db_session.commit()
class GuardianTest(unittest.TestCase):
"""Launches a consumer process that creates a queue then disconnects,
and then floods the exchange with messages and checks that PulseGuardian
warns the queue's owner and deletes the queue if it gets over the maximum
size.
"""
# Defaults; can be overridden for particular tests.
consumer_class = consumers.PulseTestConsumer
publisher_class = publishers.PulseTestPublisher
proc = None
QUEUE_CHECK_PERIOD = 0.05
QUEUE_CHECK_ATTEMPTS = 4000
QUEUE_RECORD_CHECK_PERIOD = 0.1
QUEUE_RECORD_CHECK_ATTEMPTS = 50
PUBLISHER_CONNECT_ATTEMPTS = 50
def setUp(self):
global pulse_cfg
self.proc = None
self.publisher = None
self.guardian = PulseGuardian(warn_queue_size=TEST_WARN_SIZE,
del_queue_size=TEST_DELETE_SIZE,
emails=False)
dbinit.init_and_clear_db()
self.consumer_cfg = pulse_cfg.copy()
self.consumer_cfg['applabel'] = str(uuid.uuid1())
# Configure/create the test user to be used for message consumption.
self.consumer_cfg['user'] = CONSUMER_USER
self.consumer_cfg['password'] = CONSUMER_PASSWORD
self.user = User.new_user(email=CONSUMER_EMAIL, admin=False)
# As a default owner for pulse_user in some tests where an owner is not
# provided.
self.admin = User.new_user(email=ADMIN_EMAIL, admin=True)
db_session.add(self.user)
db_session.commit()
self.pulse_user = PulseUser.new_user(
username=CONSUMER_USER,
password=CONSUMER_PASSWORD,
owners=self.user)
db_session.add(self.pulse_user)
db_session.commit()
def tearDown(self):
self._terminate_consumer_proc() # Just in case.
for queue in Queue.query.all():
pulse_management.delete_queue(vhost=DEFAULT_RABBIT_VHOST,
queue=queue.name)
def _setup_queue(self):
"""Setup a publisher, consumer and Queue, then terminate consumer"""
self._create_publisher()
self._create_consumer_proc(durable=True)
self._wait_for_queue()
self._wait_for_queue_record()
self._terminate_consumer_proc()
def _build_message(self, msg_id):
msg = TestMessage()
msg.set_data('id', msg_id)
return msg
def _create_publisher(self, create_exchange=True):
self.publisher = self.publisher_class(**pulse_cfg)
if create_exchange:
attempts = 0
exc = None
while attempts < self.PUBLISHER_CONNECT_ATTEMPTS:
attempts += 1
if attempts > 1:
time.sleep(0.1)
try:
self.publisher.publish(self._build_message(0))
except socket.error as e:
exc = e
else:
exc = None
break
if exc:
raise exc
def _create_consumer_proc(self, durable=False):
self.proc = ConsumerSubprocess(self.consumer_class, self.consumer_cfg,
durable)
self.proc.start()
def _terminate_consumer_proc(self):
if self.proc:
self.proc.terminate()
self.proc.join()
self.proc = None
def _create_passive_consumer(self):
cfg = self.consumer_cfg.copy()
cfg['connect'] = False
consumer = self.consumer_class(**self.consumer_cfg)
consumer.configure(topic='#', callback=lambda x, y: None)
return consumer
def _wait_for_queue(self, queue_should_exist=True):
'''Wait until queue has been created by consumer process.'''
consumer = self._create_passive_consumer()
attempts = 0
while attempts < self.QUEUE_CHECK_ATTEMPTS:
attempts += 1
if attempts > 1:
time.sleep(self.QUEUE_CHECK_PERIOD)
if consumer.queue_exists() == queue_should_exist:
break
self.assertEqual(consumer.queue_exists(), queue_should_exist)
def _wait_for_queue_record(self):
'''Wait until one or more queues have been added to the database.'''
consumer = self._create_passive_consumer()
attempts = 0
while attempts < self.QUEUE_RECORD_CHECK_ATTEMPTS:
attempts += 1
if attempts > 1:
time.sleep(self.QUEUE_RECORD_CHECK_PERIOD)
self.guardian.monitor_queues(pulse_management.queues(),
pulse_management.bindings())
if Queue.query.filter(Queue.name == consumer.queue_name).first():
break
def _wait_for_binding_record(self, queue_name, exchange_name, routing_key):
"""Wait until a binding has been added to the database"""
consumer = self._create_passive_consumer()
attempts = 0
while attempts < self.QUEUE_RECORD_CHECK_ATTEMPTS:
attempts += 1
if attempts > 1:
time.sleep(self.QUEUE_RECORD_CHECK_PERIOD)
self.guardian.monitor_queues(pulse_management.queues(),
pulse_management.bindings())
if Binding.query.filter(
Binding.queue_name == queue_name,
Binding.exchange == exchange_name,
Binding.routing_key == routing_key).first():
break
def _wait_for_binding_delete(self, queue_name, exchange_name, routing_key):
"""Wait until a binding has been removed from the database"""
consumer = self._create_passive_consumer()
attempts = 0
while attempts < self.QUEUE_RECORD_CHECK_ATTEMPTS:
attempts += 1
if attempts > 1:
time.sleep(self.QUEUE_RECORD_CHECK_PERIOD)
self.guardian.clear_deleted_queues(pulse_management.queues(),
pulse_management.bindings())
if not Binding.query.filter(
Binding.queue_name == queue_name,
Binding.exchange == exchange_name,
Binding.routing_key == routing_key).first():
break
def test_abnormal_queue_name(self):
self.consumer_class = AbnormalQueueConsumer
# Use account with full permissions.
self.consumer_cfg['user'] = pulse_cfg['user']
self.consumer_cfg['password'] = pulse_cfg['password']
self._create_publisher()
self._create_consumer_proc()
self._wait_for_queue()
self._wait_for_queue_record()
queue = Queue.query.filter(Queue.name ==
AbnormalQueueConsumer.QUEUE_NAME).first()
owner = queue.owner
# Queue is not durable and will be cleaned up when consumer process
# exits; delete it from the queue to avoid assertion failure in
# tearDown().
self._terminate_consumer_proc()
self._wait_for_queue(False)
db_session.delete(queue)
db_session.commit()
self.assertEqual(owner, None)
def test_warning(self):
self._setup_queue()
# Queue should still exist.
self._wait_for_queue()
# Get the queue's object.
db_session.refresh(self.pulse_user)
# Queue multiple messages while no consumer exists.
for i in xrange(self.guardian.warn_queue_size + 1):
msg = self._build_message(i)
self.publisher.publish(msg)
# Wait for messages to be taken into account and get the warned
# messages, if any.
for i in xrange(10):
time.sleep(0.3)
queues_to_warn = set(q_data['name'] for q_data
in pulse_management.queues()
if self.guardian.warn_queue_size
< q_data['messages_ready']
<= self.guardian.del_queue_size)
if queues_to_warn:
break
# Test that no queue has been warned at the beginning of the process.
self.assertTrue(not any(q.warned for q in self.pulse_user.queues))
# ... but some queues should be now.
self.assertGreater(len(queues_to_warn), 0)
# Monitor the queues; this should detect queues that should be warned.
self.guardian.monitor_queues(pulse_management.queues(),
pulse_management.bindings())
# Refresh the user's queues state.
db_session.refresh(self.pulse_user)
# Test that the queues that had to be "warned" were.
self.assertTrue(all(q.warned for q in self.pulse_user.queues
if q in queues_to_warn))
# The queues that needed to be warned haven't been deleted.
queues_to_warn_bis = set(q_data['name'] for q_data
in pulse_management.queues()
if self.guardian.warn_queue_size
< q_data['messages_ready']
<= self.guardian.del_queue_size)
self.assertEqual(queues_to_warn_bis, queues_to_warn)
def test_delete(self):
self._setup_queue()
# Queue should still exist.
self._wait_for_queue()
# Get the queue's object
db_session.refresh(self.pulse_user)
self.assertGreater(len(self.pulse_user.queues), 0)
# Queue multiple messages while no consumer exists.
for i in xrange(self.guardian.del_queue_size + 1):
msg = self._build_message(i)
self.publisher.publish(msg)
# Wait some time for published messages to be taken into account.
for i in xrange(10):
time.sleep(0.3)
queues_to_delete = [q_data['name'] for q_data
in pulse_management.queues()
if q_data['messages_ready']
> self.guardian.del_queue_size]
if queues_to_delete:
break
# Test that there are some queues that should be deleted.
self.assertGreater(len(queues_to_delete), 0)
# Setting up a callback to capture deleted queues
deleted_queues = []
def on_delete(queue):
deleted_queues.append(queue)
self.guardian.on_delete = on_delete
# Monitor the queues; this should delete overgrown queues
self.guardian.monitor_queues(pulse_management.queues(),
pulse_management.bindings())
# Test that the queues that had to be deleted were deleted...
self.assertTrue(not any(q in queues_to_delete for q
in pulse_management.queues()))
# And that they were deleted by guardian...
self.assertEqual(sorted(queues_to_delete), sorted(deleted_queues))
# And that no queue has overgrown.
queues_to_delete = [q_data['name'] for q_data
in pulse_management.queues()
if q_data['messages_ready'] >
self.guardian.del_queue_size]
self.assertEqual(len(queues_to_delete), 0)
def test_delete_skip_unbounded(self):
self._setup_queue()
# Queue should still exist.
self._wait_for_queue()
# Get the queue's object
db_session.refresh(self.pulse_user)
self.assertGreater(len(self.pulse_user.queues), 0)
# set queues as unbound so they won't be deleted
for queue in self.pulse_user.queues:
queue.unbound = 1
# Queue multiple messages while no consumer exists.
for i in xrange(self.guardian.del_queue_size + 1):
msg = self._build_message(i)
self.publisher.publish(msg)
# Wait some time for published messages to be taken into account.
for i in xrange(10):
time.sleep(0.3)
queues_to_delete = [q_data['name'] for q_data
in pulse_management.queues()
if q_data['messages_ready']
> self.guardian.del_queue_size]
if queues_to_delete:
break
# Test that there are some queues that should be deleted.
self.assertGreater(len(queues_to_delete), 0)
# Setting up a callback to capture deleted queues
deleted_queues = []
def on_delete(queue):
deleted_queues.append(queue)
self.guardian.on_delete = on_delete
# Run through the code that decides whether to delete a queue
# that has grown too large.
# In this case, it should run the check and decide to not delete
# any queues.
self.guardian.monitor_queues(pulse_management.queues(),
pulse_management.bindings())
# Test that none of the queues were deleted...
self.assertTrue(all(q in queues_to_delete for q
in pulse_management.queues()))
# And that they were not deleted by guardian...
self.assertGreater(len(queues_to_delete), 0)
def test_binding(self):
"""Test that you can get the bindings for a queue"""
self._setup_queue()
# Get the queue's object
db_session.refresh(self.pulse_user)
self.assertEqual(len(self.pulse_user.queues), 1)
# check queue bindings in the DB
queues = Queue.query.all()
self.assertEqual(len(queues), 1)
queue = queues[0]
bindings = queue.bindings
self.assertEqual(len(bindings), 1)
self.assertEqual(bindings[0].routing_key, "#")
self.assertEqual(bindings[0].exchange, "exchange/pulse/test")
def test_create_binding_different_queue_same_exchange_routing_key(self):
"""
Test create bindings for two queues with same exchange and routing key
"""
self.consumer_class = consumers.PulseTestConsumer
self._setup_queue()
# create a second queue with the same binding
self.consumer_class = SecondaryQueueConsumer
self._setup_queue()
# check queue bindings in the DB
queues = Queue.query.all()
self.assertEqual(len(queues), 2)
for queue in queues:
bindings = queue.bindings
self.assertEqual(len(bindings), 1)
self.assertEqual(bindings[0].routing_key, "#")
self.assertEqual(bindings[0].exchange, "exchange/pulse/test")
def test_add_delete_binding(self):
"""Test adding and removing a binding from a queue"""
self._setup_queue()
consumer = self._create_passive_consumer()
exchange = Exchange(consumer.exchange, type='topic')
queue = consumer._create_queue(exchange)
bound = queue.bind(consumer.connection.channel())
routing_key = "foo"
bound.bind_to(exchange=consumer.exchange, routing_key=routing_key)
self._wait_for_binding_record(queue.name, exchange.name, routing_key)
def test_bindings(exp_routing_keys):
queues = Queue.query.all()
self.assertEqual(len(queues), 1)
self.assertEqual(len(queues[0].bindings), len(exp_routing_keys))
db_queue = queues[0]
mgmt_bindings = pulse_management.queue_bindings('/', db_queue.name)
mgmt_routing_keys = {x["routing_key"] for x in mgmt_bindings}
self.assertEqual(mgmt_routing_keys, exp_routing_keys)
db_routing_keys = {x.routing_key for x in db_queue.bindings}
self.assertEqual(db_routing_keys, exp_routing_keys)
test_bindings({"#", "foo"})
# test deleting one of the bindings
bound.unbind_from(exchange=exchange, routing_key=routing_key)
self._wait_for_binding_delete(queue.name, exchange.name, routing_key)
test_bindings({"#"})
class GuardianTest(unittest.TestCase):
"""Launches a consumer process that creates a queue then disconnects,
and then floods the exchange with messages and checks that PulseGuardian
warns the queue's owner and deletes the queue if it gets over the maximum
size.
"""
# Defaults; can be overridden for particular tests.
consumer_class = consumers.PulseTestConsumer
publisher_class = publishers.PulseTestPublisher
proc = None
QUEUE_CHECK_PERIOD = 0.05
QUEUE_CHECK_ATTEMPTS = 4000
QUEUE_RECORD_CHECK_PERIOD = 0.1
QUEUE_RECORD_CHECK_ATTEMPTS = 50
PUBLISHER_CONNECT_ATTEMPTS = 50
def setUp(self):
global pulse_cfg
self.proc = None
self.publisher = None
self.guardian = PulseGuardian(warn_queue_size=TEST_WARN_SIZE,
del_queue_size=TEST_DELETE_SIZE,
emails=False)
dbinit.init_and_clear_db()
self.consumer_cfg = pulse_cfg.copy()
self.consumer_cfg['applabel'] = str(uuid.uuid1())
# Configure/create the test user to be used for message consumption.
self.consumer_cfg['user'] = CONSUMER_USER
self.consumer_cfg['password'] = CONSUMER_PASSWORD
self.user = User.new_user(email=CONSUMER_EMAIL, admin=False)
# As a default owner for rabbitmq_account in some tests where an owner
# is not provided.
self.admin = User.new_user(email=ADMIN_EMAIL, admin=True)
db_session.add(self.user)
db_session.commit()
self.rabbitmq_account = RabbitMQAccount.new_user(
username=CONSUMER_USER,
password=CONSUMER_PASSWORD,
owners=self.user)
db_session.add(self.rabbitmq_account)
db_session.commit()
def tearDown(self):
self._terminate_consumer_proc() # Just in case.
for queue in Queue.query.all():
pulse_management.delete_queue(vhost=DEFAULT_RABBIT_VHOST,
queue=queue.name)
def _setup_queue(self):
"""Setup a publisher, consumer and Queue, then terminate consumer"""
self._create_publisher()
self._create_consumer_proc(durable=True)
self._wait_for_queue()
self._wait_for_queue_record()
self._terminate_consumer_proc()
def _build_message(self, msg_id):
msg = TestMessage()
msg.set_data('id', msg_id)
return msg
def _create_publisher(self, create_exchange=True):
self.publisher = self.publisher_class(**pulse_cfg)
if create_exchange:
attempts = 0
exc = None
while attempts < self.PUBLISHER_CONNECT_ATTEMPTS:
attempts += 1
if attempts > 1:
time.sleep(0.1)
try:
self.publisher.publish(self._build_message(0))
except socket.error as e:
exc = e
else:
exc = None
break
if exc:
raise exc
def _create_consumer_proc(self, durable=False):
self.proc = ConsumerSubprocess(self.consumer_class, self.consumer_cfg,
durable)
self.proc.start()
def _terminate_consumer_proc(self):
if self.proc:
self.proc.terminate()
self.proc.join()
self.proc = None
def _create_passive_consumer(self):
cfg = self.consumer_cfg.copy()
cfg['connect'] = False
consumer = self.consumer_class(**self.consumer_cfg)
consumer.configure(topic='#', callback=lambda x, y: None)
return consumer
def _wait_for_queue(self, queue_should_exist=True):
'''Wait until queue has been created by consumer process.'''
consumer = self._create_passive_consumer()
attempts = 0
while attempts < self.QUEUE_CHECK_ATTEMPTS:
attempts += 1
if attempts > 1:
time.sleep(self.QUEUE_CHECK_PERIOD)
if consumer.queue_exists() == queue_should_exist:
break
self.assertEqual(consumer.queue_exists(), queue_should_exist)
def _wait_for_queue_record(self):
'''Wait until one or more queues have been added to the database.'''
consumer = self._create_passive_consumer()
attempts = 0
while attempts < self.QUEUE_RECORD_CHECK_ATTEMPTS:
attempts += 1
if attempts > 1:
time.sleep(self.QUEUE_RECORD_CHECK_PERIOD)
self.guardian.monitor_queues(pulse_management.queues(vhost='/'),
pulse_management.bindings(vhost='/'))
if Queue.query.filter(Queue.name == consumer.queue_name).first():
break
def _wait_for_binding_record(self, queue_name, exchange_name, routing_key):
"""Wait until a binding has been added to the database"""
consumer = self._create_passive_consumer()
attempts = 0
while attempts < self.QUEUE_RECORD_CHECK_ATTEMPTS:
attempts += 1
if attempts > 1:
time.sleep(self.QUEUE_RECORD_CHECK_PERIOD)
self.guardian.monitor_queues(pulse_management.queues(vhost='/'),
pulse_management.bindings(vhost='/'))
if Binding.query.filter(
Binding.queue_name == queue_name,
Binding.exchange == exchange_name,
Binding.routing_key == routing_key).first():
break
def _wait_for_binding_delete(self, queue_name, exchange_name, routing_key):
"""Wait until a binding has been removed from the database"""
consumer = self._create_passive_consumer()
attempts = 0
while attempts < self.QUEUE_RECORD_CHECK_ATTEMPTS:
attempts += 1
if attempts > 1:
time.sleep(self.QUEUE_RECORD_CHECK_PERIOD)
self.guardian.clear_deleted_queues(
pulse_management.queues(vhost='/'),
pulse_management.bindings(vhost='/'))
if not Binding.query.filter(Binding.queue_name == queue_name,
Binding.exchange == exchange_name,
Binding.routing_key
== routing_key).first():
break
def test_abnormal_queue_name(self):
self.consumer_class = AbnormalQueueConsumer
# Use account with full permissions.
self.consumer_cfg['user'] = pulse_cfg['user']
self.consumer_cfg['password'] = pulse_cfg['password']
self._create_publisher()
self._create_consumer_proc()
self._wait_for_queue()
self._wait_for_queue_record()
queue = Queue.query.filter(
Queue.name == AbnormalQueueConsumer.QUEUE_NAME).first()
owner = queue.owner
# Queue is not durable and will be cleaned up when consumer process
# exits; delete it from the queue to avoid assertion failure in
# tearDown().
self._terminate_consumer_proc()
self._wait_for_queue(False)
db_session.delete(queue)
db_session.commit()
self.assertEqual(owner, None)
def test_reserved_queue_name(self):
self.consumer_class = ReservedQueueConsumer
# Use account with full permissions.
self.consumer_cfg['user'] = pulse_cfg['user']
self.consumer_cfg['password'] = pulse_cfg['password']
config.reserved_users_regex = 'reserved-.*'
try:
self._create_publisher()
self._create_consumer_proc()
self._wait_for_queue()
self._wait_for_queue_record()
finally:
config.reserved_users_regex = None
queue = Queue.query.filter(
Queue.name == ReservedQueueConsumer.QUEUE_NAME).first()
# Queue is not durable and will be cleaned up when consumer process
# exits; delete it from the queue to avoid assertion failure in
# tearDown().
self._terminate_consumer_proc()
self._wait_for_queue(False)
self.assertEqual(queue, None)
def test_warning(self):
self._setup_queue()
# Queue should still exist.
self._wait_for_queue()
# Get the queue's object.
db_session.refresh(self.rabbitmq_account)
# Queue multiple messages while no consumer exists.
for i in range(self.guardian.warn_queue_size + 1):
msg = self._build_message(i)
self.publisher.publish(msg)
# Wait for messages to be taken into account and get the warned
# messages, if any.
for i in range(100):
time.sleep(0.3)
queues_to_warn = set(
q_data['name'] for q_data in pulse_management.queues()
if self.guardian.warn_queue_size < q_data['messages_ready'] <=
self.guardian.del_queue_size)
if queues_to_warn:
break
# Test that no queue has been warned at the beginning of the process.
self.assertTrue(not any(q.warned
for q in self.rabbitmq_account.queues))
# ... but some queues should be now.
self.assertGreater(len(queues_to_warn), 0)
# Monitor the queues; this should detect queues that should be warned.
self.guardian.monitor_queues(pulse_management.queues(vhost='/'),
pulse_management.bindings(vhost='/'))
# Refresh the user's queues state.
db_session.refresh(self.rabbitmq_account)
# Test that the queues that had to be "warned" were.
self.assertTrue(
all(q.warned for q in self.rabbitmq_account.queues
if q in queues_to_warn))
# The queues that needed to be warned haven't been deleted.
queues_to_warn_bis = set(
q_data['name'] for q_data in pulse_management.queues()
if self.guardian.warn_queue_size < q_data['messages_ready'] <=
self.guardian.del_queue_size)
self.assertEqual(queues_to_warn_bis, queues_to_warn)
def test_delete(self):
self._setup_queue()
# Queue should still exist.
self._wait_for_queue()
# Get the queue's object
db_session.refresh(self.rabbitmq_account)
self.assertGreater(len(self.rabbitmq_account.queues), 0)
# Queue multiple messages while no consumer exists.
for i in range(self.guardian.del_queue_size + 1):
msg = self._build_message(i)
self.publisher.publish(msg)
# Wait some time for published messages to be taken into account.
for i in range(100):
time.sleep(0.3)
queues_to_delete = [
q_data['name'] for q_data in pulse_management.queues()
if q_data['messages_ready'] > self.guardian.del_queue_size
]
if queues_to_delete:
break
# Test that there are some queues that should be deleted.
self.assertGreater(len(queues_to_delete), 0)
# Setting up a callback to capture deleted queues
deleted_queues = []
def on_delete(queue):
deleted_queues.append(queue)
self.guardian.on_delete = on_delete
# Monitor the queues; this should delete overgrown queues
self.guardian.monitor_queues(pulse_management.queues(vhost='/'),
pulse_management.bindings(vhost='/'))
# Test that the queues that had to be deleted were deleted...
self.assertTrue(not any(q in queues_to_delete
for q in pulse_management.queues()))
# And that they were deleted by guardian...
self.assertEqual(sorted(queues_to_delete), sorted(deleted_queues))
# And that no queue has overgrown.
queues_to_delete = [
q_data['name'] for q_data in pulse_management.queues()
if q_data['messages_ready'] > self.guardian.del_queue_size
]
self.assertEqual(len(queues_to_delete), 0)
def test_delete_skip_unbounded(self):
self._setup_queue()
# Queue should still exist.
self._wait_for_queue()
# Get the queue's object
db_session.refresh(self.rabbitmq_account)
self.assertGreater(len(self.rabbitmq_account.queues), 0)
# set queues as unbound so they won't be deleted
for queue in self.rabbitmq_account.queues:
queue.unbound = 1
# Queue multiple messages while no consumer exists.
for i in range(self.guardian.del_queue_size + 1):
msg = self._build_message(i)
self.publisher.publish(msg)
# Wait some time for published messages to be taken into account.
for i in range(100):
time.sleep(0.3)
queues_to_delete = [
q_data['name'] for q_data in pulse_management.queues()
if q_data['messages_ready'] > self.guardian.del_queue_size
]
if queues_to_delete:
break
# Test that there are some queues that should be deleted.
self.assertGreater(len(queues_to_delete), 0)
# Setting up a callback to capture deleted queues
deleted_queues = []
def on_delete(queue):
deleted_queues.append(queue)
self.guardian.on_delete = on_delete
# Run through the code that decides whether to delete a queue
# that has grown too large.
# In this case, it should run the check and decide to not delete
# any queues.
self.guardian.monitor_queues(pulse_management.queues(vhost='/'),
pulse_management.bindings(vhost='/'))
# Test that none of the queues were deleted...
self.assertTrue(
all(q in queues_to_delete for q in pulse_management.queues()))
# And that they were not deleted by guardian...
self.assertGreater(len(queues_to_delete), 0)
def test_binding(self):
"""Test that you can get the bindings for a queue"""
self._setup_queue()
# Get the queue's object
db_session.refresh(self.rabbitmq_account)
self.assertEqual(len(self.rabbitmq_account.queues), 1)
# check queue bindings in the DB
queues = Queue.query.all()
self.assertEqual(len(queues), 1)
queue = queues[0]
bindings = queue.bindings
self.assertEqual(len(bindings), 1)
self.assertEqual(bindings[0].routing_key, "#")
self.assertEqual(bindings[0].exchange, "exchange/pulse/test")
def test_create_binding_different_queue_same_exchange_routing_key(self):
"""
Test create bindings for two queues with same exchange and routing key
"""
self.consumer_class = consumers.PulseTestConsumer
self._setup_queue()
# create a second queue with the same binding
self.consumer_class = SecondaryQueueConsumer
self._setup_queue()
# check queue bindings in the DB
queues = Queue.query.all()
self.assertEqual(len(queues), 2)
for queue in queues:
bindings = queue.bindings
self.assertEqual(len(bindings), 1)
self.assertEqual(bindings[0].routing_key, "#")
self.assertEqual(bindings[0].exchange, "exchange/pulse/test")
def test_add_delete_binding(self):
"""Test adding and removing a binding from a queue"""
self._setup_queue()
consumer = self._create_passive_consumer()
exchange = Exchange(consumer.exchange, type='topic')
queue = consumer._create_queue(exchange)
bound = queue.bind(consumer.connection.channel())
routing_key = "foo"
bound.bind_to(exchange=consumer.exchange, routing_key=routing_key)
self._wait_for_binding_record(queue.name, exchange.name, routing_key)
def test_bindings(exp_routing_keys):
queues = Queue.query.all()
self.assertEqual(len(queues), 1)
self.assertEqual(len(queues[0].bindings), len(exp_routing_keys))
db_queue = queues[0]
mgmt_bindings = pulse_management.queue_bindings('/', db_queue.name)
mgmt_routing_keys = {x["routing_key"] for x in mgmt_bindings}
self.assertEqual(mgmt_routing_keys, exp_routing_keys)
db_routing_keys = {x.routing_key for x in db_queue.bindings}
self.assertEqual(db_routing_keys, exp_routing_keys)
test_bindings({"#", "foo"})
# test deleting one of the bindings
bound.unbind_from(exchange=exchange, routing_key=routing_key)
self._wait_for_binding_delete(queue.name, exchange.name, routing_key)
test_bindings({"#"})
class GuardianTest(unittest.TestCase):
"""Launches a consumer process that creates a queue then disconnects,
and then floods the exchange with messages and checks that PulseGuardian
warns the queue's owner and deletes the queue if it gets over the maximum
size.
"""
# Defaults; can be overridden for particular tests.
consumer_class = consumers.PulseTestConsumer
publisher_class = publishers.PulseTestPublisher
proc = None
QUEUE_CHECK_PERIOD = 0.05
QUEUE_CHECK_ATTEMPTS = 4000
QUEUE_RECORD_CHECK_PERIOD = 0.1
QUEUE_RECORD_CHECK_ATTEMPTS = 50
PUBLISHER_CONNECT_ATTEMPTS = 50
def setUp(self):
global pulse_cfg
self.proc = None
self.publisher = None
self.management_api = PulseManagementAPI(
management_url='http://{}:{}/api'.format(
pulse_cfg['host'], pulse_cfg['management_port']),
user=pulse_cfg['user'],
password=pulse_cfg['password']
)
self.guardian = PulseGuardian(self.management_api,
warn_queue_size=TEST_WARN_SIZE,
del_queue_size=TEST_DELETE_SIZE,
emails=False)
# Hack in a test config.
dbinit.pulse_management = self.management_api
dbinit.init_and_clear_db()
self.consumer_cfg = pulse_cfg.copy()
self.consumer_cfg['applabel'] = str(uuid.uuid1())
# Configure/create the test user to be used for message consumption.
self.consumer_cfg['user'] = CONSUMER_USER
self.consumer_cfg['password'] = CONSUMER_PASSWORD
self.user = User.new_user(email=CONSUMER_EMAIL, admin=False)
db_session.add(self.user)
db_session.commit()
self.pulse_user = PulseUser.new_user(
username=CONSUMER_USER,
password=CONSUMER_PASSWORD,
owner=self.user,
management_api=self.management_api)
db_session.add(self.pulse_user)
db_session.commit()
def tearDown(self):
self._terminate_consumer_proc() # Just in case.
for queue in Queue.query.all():
self.management_api.delete_queue(vhost=DEFAULT_RABBIT_VHOST,
queue=queue.name)
def _build_message(self, msg_id):
msg = TestMessage()
msg.set_data('id', msg_id)
return msg
def _create_publisher(self, create_exchange=True):
self.publisher = self.publisher_class(**pulse_cfg)
if create_exchange:
attempts = 0
exc = None
while attempts < self.PUBLISHER_CONNECT_ATTEMPTS:
attempts += 1
if attempts > 1:
time.sleep(0.1)
try:
self.publisher.publish(self._build_message(0))
except socket.error as e:
exc = e
else:
exc = None
break
if exc:
raise exc
def _create_consumer_proc(self, durable=False):
self.proc = ConsumerSubprocess(self.consumer_class, self.consumer_cfg,
durable)
self.proc.start()
def _terminate_consumer_proc(self):
if self.proc:
self.proc.terminate()
self.proc.join()
self.proc = None
def _create_passive_consumer(self):
cfg = self.consumer_cfg.copy()
cfg['connect'] = False
consumer = self.consumer_class(**self.consumer_cfg)
consumer.configure(topic='#', callback=lambda x, y: None)
return consumer
def _wait_for_queue(self, queue_should_exist=True):
'''Wait until queue has been created by consumer process.'''
consumer = self._create_passive_consumer()
attempts = 0
while attempts < self.QUEUE_CHECK_ATTEMPTS:
attempts += 1
if attempts > 1:
time.sleep(self.QUEUE_CHECK_PERIOD)
if consumer.queue_exists() == queue_should_exist:
break
self.assertEqual(consumer.queue_exists(), queue_should_exist)
def _wait_for_queue_record(self):
'''Wait until one or more queues have been added to the database.'''
consumer = self._create_passive_consumer()
attempts = 0
while attempts < self.QUEUE_RECORD_CHECK_ATTEMPTS:
attempts += 1
if attempts > 1:
time.sleep(self.QUEUE_RECORD_CHECK_PERIOD)
self.guardian.monitor_queues(self.management_api.queues())
if Queue.query.filter(Queue.name == consumer.queue_name).first():
break
def test_abnormal_queue_name(self):
self.consumer_class = AbnormalQueueConsumer
# Use account with full permissions.
self.consumer_cfg['user'] = pulse_cfg['user']
self.consumer_cfg['password'] = pulse_cfg['password']
self._create_publisher()
self._create_consumer_proc()
self._wait_for_queue()
self._wait_for_queue_record()
queue = Queue.query.filter(Queue.name ==
AbnormalQueueConsumer.QUEUE_NAME).first()
owner = queue.owner
# Queue is not durable and will be cleaned up when consumer process
# exits; delete it from the queue to avoid assertion failure in
# tearDown().
self._terminate_consumer_proc()
self._wait_for_queue(False)
db_session.delete(queue)
db_session.commit()
self.assertEqual(owner, None)
def test_warning(self):
self._create_publisher()
self._create_consumer_proc(durable=True)
self._wait_for_queue()
self._wait_for_queue_record()
self._terminate_consumer_proc()
# Queue should still exist.
self._wait_for_queue()
# Get the queue's object.
db_session.refresh(self.pulse_user)
# Queue multiple messages while no consumer exists.
for i in xrange(self.guardian.warn_queue_size + 1):
msg = self._build_message(i)
self.publisher.publish(msg)
# Wait for messages to be taken into account and get the warned
# messages, if any.
for i in xrange(10):
time.sleep(0.3)
queues_to_warn = set(q_data['name'] for q_data
in self.management_api.queues()
if self.guardian.warn_queue_size
< q_data['messages_ready']
<= self.guardian.del_queue_size)
if queues_to_warn:
break
# Test that no queue has been warned at the beginning of the process.
self.assertTrue(not any(q.warned for q in self.pulse_user.queues))
# ... but some queues should be now.
self.assertTrue(len(queues_to_warn) > 0)
# Monitor the queues; this should detect queues that should be warned.
self.guardian.monitor_queues(self.management_api.queues())
# Refresh the user's queues state.
db_session.refresh(self.pulse_user)
# Test that the queues that had to be "warned" were.
self.assertTrue(all(q.warned for q in self.pulse_user.queues
if q in queues_to_warn))
# The queues that needed to be warned haven't been deleted.
queues_to_warn_bis = set(q_data['name'] for q_data
in self.management_api.queues()
if self.guardian.warn_queue_size
< q_data['messages_ready']
<= self.guardian.del_queue_size)
self.assertEqual(queues_to_warn_bis, queues_to_warn)
def test_delete(self):
self._create_publisher()
self._create_consumer_proc(durable=True)
self._wait_for_queue()
self._wait_for_queue_record()
self._terminate_consumer_proc()
# Queue should still exist.
self._wait_for_queue()
# Get the queue's object
db_session.refresh(self.pulse_user)
self.assertTrue(len(self.pulse_user.queues) > 0)
# Queue multiple messages while no consumer exists.
for i in xrange(self.guardian.del_queue_size + 1):
msg = self._build_message(i)
self.publisher.publish(msg)
# Wait some time for published messages to be taken into account.
for i in xrange(10):
time.sleep(0.3)
queues_to_delete = [q_data['name'] for q_data
in self.management_api.queues()
if q_data['messages_ready']
> self.guardian.del_queue_size]
if queues_to_delete:
break
# Test that there are some queues that should be deleted.
self.assertTrue(len(queues_to_delete) > 0)
# Setting up a callback to capture deleted queues
deleted_queues = []
def on_delete(queue):
deleted_queues.append(queue)
self.guardian.on_delete = on_delete
# Monitor the queues; this should create the queue object and assign
# it to the user.
for i in xrange(20):
self.guardian.monitor_queues(self.management_api.queues())
time.sleep(0.2)
# Test that the queues that had to be deleted were deleted...
self.assertTrue(not any(q in queues_to_delete for q
in self.management_api.queues()))
# And that they were deleted by guardian...
self.assertEqual(sorted(queues_to_delete), sorted(deleted_queues))
# And that no queue has overgrown.
queues_to_delete = [q_data['name'] for q_data
in self.management_api.queues()
if q_data['messages_ready'] >
self.guardian.del_queue_size]
self.assertTrue(len(queues_to_delete) == 0)
