def test_expired_claim(self):
meta = {'ttl': 0, 'grace': 60}
claim_id, messages = self.controller.create(self.queue_name, meta,
project=self.project)
with testing.expect(storage.errors.DoesNotExist):
self.controller.get(self.queue_name, claim_id,
project=self.project)
with testing.expect(storage.errors.DoesNotExist):
self.controller.update(self.queue_name, claim_id,
meta, project=self.project)
def test_expired_messages(self):
messages = [{'body': 3.14, 'ttl': 0}, {'body': 0.618, 'ttl': 600}]
client_uuid = uuid.uuid4()
[msgid_expired, msgid] = self.controller.post(self.queue_name,
messages,
project=self.project,
client_uuid=client_uuid)
time.sleep(self.gc_interval)
with testing.expect(storage.errors.DoesNotExist):
self.controller.get(self.queue_name, msgid_expired,
project=self.project)
stats = self.queue_controller.stats(self.queue_name,
project=self.project)
self.assertEqual(stats['messages']['free'], 1)
# Make sure expired messages not return when listing
interaction = self.controller.list(self.queue_name,
project=self.project)
messages = list(next(interaction))
self.assertEqual(len(messages), 1)
self.assertEqual(msgid, messages[0]['id'])
# Make sure expired messages not return when popping
messages = self.controller.pop(self.queue_name,
limit=10,
project=self.project)
self.assertEqual(len(messages), 1)
self.assertEqual(msgid, messages[0]['id'])
def test_delete_pool_used_by_flavor(self):
self.flavors_controller.create('durable', self.pool_group,
project=self.project,
capabilities={})
with testing.expect(errors.PoolInUseByFlavor):
self.pools_controller.delete(self.pool)
def test_message_lifecycle(self):
queue_name = self.queue_name
message = {
'ttl': 60,
'body': {
'event': 'BackupStarted',
'backupId': 'c378813c-3f0b-11e2-ad92-7823d2b0f3ce'
}
}
# Test Message Creation
created = list(self.controller.post(queue_name, [message],
project=self.project,
client_uuid=uuid.uuid4()))
self.assertEqual(len(created), 1)
message_id = created[0]
# Test Message Get
message_out = self.controller.get(queue_name, message_id,
project=self.project)
self.assertEqual(set(message_out), set(('id', 'body', 'ttl', 'age')))
self.assertEqual(message_out['id'], message_id)
self.assertEqual(message_out['body'], message['body'])
self.assertEqual(message_out['ttl'], message['ttl'])
# Test Message Deletion
self.controller.delete(queue_name, message_id, project=self.project)
# Test does not exist
with testing.expect(storage.errors.DoesNotExist):
self.controller.get(queue_name, message_id, project=self.project)
def test_delete_pool_used_by_flavor(self):
self.flavors_controller.create('durable', self.pool_group,
project=self.project,
capabilities={})
with testing.expect(errors.PoolInUseByFlavor):
self.pools_controller.delete(self.pool)
def test_expired_messages(self):
messages = [{'body': 3.14, 'ttl': 0}, {'body': 0.618, 'ttl': 600}]
client_uuid = uuid.uuid4()
[msgid_expired, msgid] = self.controller.post(self.queue_name,
messages,
project=self.project,
client_uuid=client_uuid)
time.sleep(self.gc_interval)
with testing.expect(storage.errors.DoesNotExist):
self.controller.get(self.queue_name,
msgid_expired,
project=self.project)
stats = self.queue_controller.stats(self.queue_name,
project=self.project)
self.assertEqual(stats['messages']['free'], 1)
# Make sure expired messages not return when listing
interaction = self.controller.list(self.queue_name,
project=self.project)
messages = list(next(interaction))
self.assertEqual(len(messages), 1)
self.assertEqual(msgid, messages[0]['id'])
# Make sure expired messages not return when popping
messages = self.controller.pop(self.queue_name,
limit=10,
project=self.project)
self.assertEqual(len(messages), 1)
self.assertEqual(msgid, messages[0]['id'])
def test_message_lifecycle(self):
queue_name = self.queue_name
message = {
'ttl': 60,
'body': {
'event': 'BackupStarted',
'backupId': 'c378813c-3f0b-11e2-ad92-7823d2b0f3ce'
}
}
# Test Message Creation
created = list(
self.controller.post(queue_name, [message],
project=self.project,
client_uuid=uuid.uuid4()))
self.assertEqual(len(created), 1)
message_id = created[0]
# Test Message Get
message_out = self.controller.get(queue_name,
message_id,
project=self.project)
self.assertEqual(set(message_out), set(('id', 'body', 'ttl', 'age')))
self.assertEqual(message_out['id'], message_id)
self.assertEqual(message_out['body'], message['body'])
self.assertEqual(message_out['ttl'], message['ttl'])
# Test Message Deletion
self.controller.delete(queue_name, message_id, project=self.project)
# Test does not exist
with testing.expect(storage.errors.DoesNotExist):
self.controller.get(queue_name, message_id, project=self.project)
def test_mismatching_capabilities_fifo(self):
with testing.expect(errors.PoolCapabilitiesMismatch):
self.pools_controller.create(str(uuid.uuid1()),
100,
'mongodb.fifo://localhost',
group=self.pool_group,
options={})
def test_illformed_id(self):
# any ill-formed IDs should be regarded as non-existing ones.
self.controller.delete(self.queue_name,
'illformed',
project=self.project)
with testing.expect(errors.DoesNotExist):
self.controller.get(self.queue_name,
'illformed',
project=self.project)
with testing.expect(errors.DoesNotExist):
self.controller.update(self.queue_name,
'illformed', {'ttl': 40},
project=self.project)
def test_expired_claim(self):
meta = {'ttl': 0, 'grace': 60}
claim_id, messages = self.controller.create(self.queue_name,
meta,
project=self.project)
with testing.expect(storage.errors.DoesNotExist):
self.controller.get(self.queue_name,
claim_id,
project=self.project)
with testing.expect(storage.errors.DoesNotExist):
self.controller.update(self.queue_name,
claim_id,
meta,
project=self.project)
def test_mismatching_capabilities1(self):
# NOTE(gengchc2): This test is used for testing mismatchming
# capabilities in pool with flavor
with testing.expect(storage.errors.PoolCapabilitiesMismatch):
self.pools_controller.create(str(uuid.uuid1()),
100, 'redis://localhost',
flavor=self.flavor,
options={})
def test_mismatching_capabilities1(self):
# NOTE(gengchc2): This test is used for testing mismatchming
# capabilities in pool with flavor
with testing.expect(storage.errors.PoolCapabilitiesMismatch):
self.pools_controller.create(str(uuid.uuid1()),
100, 'mongodb://localhost',
flavor=self.flavor,
options={})
def test_delete_pool_used_by_flavor1(self):
self.flavors_controller.create(self.flavor,
project=self.project,
capabilities={})
self.pools_controller.update(self.pool1, flavor=self.flavor)
with testing.expect(errors.PoolInUseByFlavor):
self.pools_controller.delete(self.pool1)
def test_illformed_id(self):
# any ill-formed IDs should be regarded as non-existing ones.
self.controller.delete(self.queue_name,
'illformed',
project=self.project)
with testing.expect(errors.DoesNotExist):
self.controller.get(self.queue_name,
'illformed',
project=self.project)
with testing.expect(errors.DoesNotExist):
self.controller.update(self.queue_name,
'illformed',
{'ttl': 40},
project=self.project)
def test_delete_pool_used_by_flavor1(self):
self.flavors_controller.create(self.flavor,
project=self.project,
capabilities={})
self.pools_controller.update(self.pool1, flavor=self.flavor)
with testing.expect(errors.PoolInUseByFlavor):
self.pools_controller.delete(self.pool1)
def test_duplicate_uri(self):
with testing.expect(errors.PoolAlreadyExists):
# The url 'localhost' is used in setUp(). So reusing the uri
# 'localhost' here will raise PoolAlreadyExists.
self.pools_controller.create(str(uuid.uuid1()),
100,
'localhost',
group=str(uuid.uuid1()),
options={})
def test_mismatching_capabilities(self):
# NOTE(gengchc2): This test is used for testing mismatchming
# capabilities in pool with group
with testing.expect(errors.PoolCapabilitiesMismatch):
self.pools_controller.create(str(uuid.uuid1()),
100,
'redis://localhost',
group=self.pool_group,
options={})
def test_bad_id(self):
# NOTE(cpp-cabrera): A malformed ID should result in an empty
# query. Raising an exception for validating IDs makes the
# implementation more verbose instead of taking advantage of
# the Maybe/Optional protocol, particularly when dealing with
# bulk operations.
bad_message_id = 'xyz'
self.controller.delete(self.queue_name,
bad_message_id,
project=self.project)
with testing.expect(errors.MessageDoesNotExist):
self.controller.get(self.queue_name,
bad_message_id,
project=self.project)
def test_bad_id(self):
# NOTE(cpp-cabrera): A malformed ID should result in an empty
# query. Raising an exception for validating IDs makes the
# implementation more verbose instead of taking advantage of
# the Maybe/Optional protocol, particularly when dealing with
# bulk operations.
bad_message_id = 'xyz'
self.controller.delete(self.queue_name,
bad_message_id,
project=self.project)
with testing.expect(errors.MessageDoesNotExist):
self.controller.get(self.queue_name,
bad_message_id,
project=self.project)
def test_expired_messages_be_deleted(self):
messages = [{'body': 3.14, 'ttl': 0}, {'body': 0.618, 'ttl': 600}]
client_uuid = uuid.uuid4()
[msgid_expired, msgid] = self.controller.post(self.queue_name,
messages,
project=self.project,
client_uuid=client_uuid)
mid = utils.msgid_decode(msgid_expired)
def _get(count=False):
j = sa.join(tables.Messages, tables.Queues,
tables.Messages.c.qid == tables.Queues.c.id)
sel = sa.sql.select([
tables.Messages.c.body, tables.Messages.c.ttl,
tables.Messages.c.created
])
if count:
sel = sa.sql.select([sfunc.count(tables.Messages.c.id)])
sel = sel.select_from(j)
and_stmt = [
tables.Messages.c.id == mid,
tables.Queues.c.name == self.queue_name,
tables.Queues.c.project == self.project
]
sel = sel.where(sa.and_(*and_stmt))
return self.driver.get(sel)
[count] = _get(count=True)
self.assertEqual(count, 1)
# Expired messages will be removed from db until next Post
message = [{'body': 3.14, 'ttl': 300}]
self.controller.post(self.queue_name,
message,
project=self.project,
client_uuid=client_uuid)
with testing.expect(utils.NoResult):
_get()
def test_expired_messages_be_deleted(self):
messages = [{'body': 3.14, 'ttl': 0}, {'body': 0.618, 'ttl': 600}]
client_uuid = uuid.uuid4()
[msgid_expired, msgid] = self.controller.post(self.queue_name,
messages,
project=self.project,
client_uuid=client_uuid)
mid = utils.msgid_decode(msgid_expired)
def _get(count=False):
j = sa.join(tables.Messages, tables.Queues,
tables.Messages.c.qid == tables.Queues.c.id)
sel = sa.sql.select([tables.Messages.c.body,
tables.Messages.c.ttl,
tables.Messages.c.created])
if count:
sel = sa.sql.select([sfunc.count(tables.Messages.c.id)])
sel = sel.select_from(j)
and_stmt = [tables.Messages.c.id == mid,
tables.Queues.c.name == self.queue_name,
tables.Queues.c.project == self.project]
sel = sel.where(sa.and_(*and_stmt))
return self.driver.get(sel)
[count] = _get(count=True)
self.assertEqual(count, 1)
# Expired messages will be removed from db until next Post
message = [{'body': 3.14, 'ttl': 300}]
self.controller.post(self.queue_name,
message,
project=self.project,
client_uuid=client_uuid)
with testing.expect(utils.NoResult):
_get()
def test_multi_ids(self):
messages_in = [{'ttl': 120, 'body': 0}, {'ttl': 240, 'body': 1}]
ids = self.controller.post(self.queue_name, messages_in,
project=self.project,
client_uuid=uuid.uuid4())
messages_out = self.controller.bulk_get(self.queue_name, ids,
project=self.project)
for idx, message in enumerate(messages_out):
self.assertEqual(set(message), set(('id', 'body', 'ttl', 'age')))
self.assertEqual(message['body'], idx)
self.controller.bulk_delete(self.queue_name, ids,
project=self.project)
with testing.expect(StopIteration):
result = self.controller.bulk_get(self.queue_name, ids,
project=self.project)
next(result)
def test_multi_ids(self):
messages_in = [{'ttl': 120, 'body': 0}, {'ttl': 240, 'body': 1}]
ids = self.controller.post(self.queue_name,
messages_in,
project=self.project,
client_uuid=uuid.uuid4())
messages_out = self.controller.bulk_get(self.queue_name,
ids,
project=self.project)
for idx, message in enumerate(messages_out):
self.assertEqual(set(message), set(('id', 'body', 'ttl', 'age')))
self.assertEqual(message['body'], idx)
self.controller.bulk_delete(self.queue_name, ids, project=self.project)
with testing.expect(StopIteration):
result = self.controller.bulk_get(self.queue_name,
ids,
project=self.project)
next(result)
def test_race_condition_on_post(self):
queue_name = self.queue_name
expected_messages = [
{
'ttl': 60,
'body': {
'event': 'BackupStarted',
'backupId': 'c378813c-3f0b-11e2-ad92-7823d2b0f3ce',
},
},
{
'ttl': 60,
'body': {
'event': 'BackupStarted',
'backupId': 'd378813c-3f0b-11e2-ad92-7823d2b0f3ce',
},
},
{
'ttl': 60,
'body': {
'event': 'BackupStarted',
'backupId': 'e378813c-3f0b-11e2-ad92-7823d2b0f3ce',
},
},
]
uuid = '97b64000-2526-11e3-b088-d85c1300734c'
# NOTE(kgriffs): Patch _inc_counter so it is a noop, so that
# the second time we post, we will get a collision. This simulates
# what happens when we have parallel requests and the "winning"
# requests hasn't gotten around to calling _inc_counter before the
# "losing" request attempts to insert it's batch of messages.
with mock.patch.object(mongodb.queues.QueueController,
'_inc_counter', autospec=True) as method:
method.return_value = 2
messages = expected_messages[:1]
created = list(self.controller.post(queue_name, messages,
uuid, project=self.project))
self.assertEqual(len(created), 1)
# Force infinite retries
method.return_value = None
with testing.expect(errors.MessageConflict):
self.controller.post(queue_name, messages,
uuid, project=self.project)
created = list(self.controller.post(queue_name,
expected_messages[1:],
uuid, project=self.project))
self.assertEqual(len(created), 2)
expected_ids = [m['body']['backupId'] for m in expected_messages]
interaction = self.controller.list(queue_name, client_uuid=uuid,
echo=True, project=self.project)
actual_messages = list(next(interaction))
self.assertEqual(len(actual_messages), len(expected_messages))
actual_ids = [m['body']['backupId'] for m in actual_messages]
self.assertEqual(actual_ids, expected_ids)
def test_delete_pool_used_by_flavor(self):
with testing.expect(storage.errors.PoolInUseByFlavor):
self.pools_controller.delete(self.pool1)
def test_delete_pool_used_by_flavor(self):
with testing.expect(storage.errors.PoolInUseByFlavor):
self.pools_controller.delete(self.pool1)
def test_mismatching_capabilities_fifo(self):
with testing.expect(errors.PoolCapabilitiesMismatch):
self.pools_controller.create(str(uuid.uuid1()),
100, 'mongodb.fifo://localhost',
group=self.pool_group,
options={})
def test_claim_effects(self):
client_uuid = uuid.uuid4()
_insert_fixtures(self.controller,
self.queue_name,
project=self.project,
client_uuid=client_uuid,
num=12)
def list_messages(include_claimed=None):
kwargs = {
'project': self.project,
'client_uuid': client_uuid,
'echo': True,
}
# Properly test default value
if include_claimed is not None:
kwargs['include_claimed'] = include_claimed
interaction = self.controller.list(self.queue_name, **kwargs)
messages = next(interaction)
return [msg['id'] for msg in messages]
messages_before = list_messages(True)
meta = {'ttl': 70, 'grace': 60}
another_cid, _ = self.claim_controller.create(self.queue_name,
meta,
project=self.project)
messages_after = list_messages(True)
self.assertEqual(messages_before, messages_after)
messages_excluding_claimed = list_messages()
self.assertNotEqual(messages_before, messages_excluding_claimed)
self.assertEqual(2, len(messages_excluding_claimed))
cid, msgs = self.claim_controller.create(self.queue_name,
meta,
project=self.project)
[msg1, msg2] = msgs
# A wrong claim does not ensure the message deletion
with testing.expect(storage.errors.NotPermitted):
self.controller.delete(self.queue_name,
msg1['id'],
project=self.project,
claim=another_cid)
# Make sure a message can be deleted with a claim
self.controller.delete(self.queue_name,
msg1['id'],
project=self.project,
claim=cid)
with testing.expect(storage.errors.DoesNotExist):
self.controller.get(self.queue_name,
msg1['id'],
project=self.project)
# Make sure such a deletion is idempotent
self.controller.delete(self.queue_name,
msg1['id'],
project=self.project,
claim=cid)
# A non-existing claim does not ensure the message deletion
self.claim_controller.delete(self.queue_name,
cid,
project=self.project)
with testing.expect(storage.errors.NotPermitted):
self.controller.delete(self.queue_name,
msg2['id'],
project=self.project,
claim=cid)
def test_race_condition_on_post(self):
queue_name = self.queue_name
expected_messages = [
{
'ttl': 60,
'body': {
'event': 'BackupStarted',
'backupId': 'c378813c-3f0b-11e2-ad92-7823d2b0f3ce',
},
},
{
'ttl': 60,
'body': {
'event': 'BackupStarted',
'backupId': 'd378813c-3f0b-11e2-ad92-7823d2b0f3ce',
},
},
{
'ttl': 60,
'body': {
'event': 'BackupStarted',
'backupId': 'e378813c-3f0b-11e2-ad92-7823d2b0f3ce',
},
},
]
uuid = '97b64000-2526-11e3-b088-d85c1300734c'
# NOTE(kgriffs): Patch _inc_counter so it is a noop, so that
# the second time we post, we will get a collision. This simulates
# what happens when we have parallel requests and the "winning"
# requests hasn't gotten around to calling _inc_counter before the
# "losing" request attempts to insert it's batch of messages.
with mock.patch.object(mongodb.messages.MessageController,
'_inc_counter', autospec=True) as ic:
ic.return_value = 2
messages = expected_messages[:1]
created = list(self.controller.post(queue_name,
messages, uuid,
project=self.project))
self.assertEqual(1, len(created))
# Force infinite retries
ic.return_value = None
with testing.expect(errors.MessageConflict):
self.controller.post(queue_name, messages,
uuid, project=self.project)
created = list(self.controller.post(queue_name,
expected_messages[1:],
uuid, project=self.project))
self.assertEqual(2, len(created))
expected_ids = [m['body']['backupId'] for m in expected_messages]
interaction = self.controller.list(queue_name, client_uuid=uuid,
echo=True, project=self.project)
actual_messages = list(next(interaction))
self.assertEqual(len(expected_messages), len(actual_messages))
actual_ids = [m['body']['backupId'] for m in actual_messages]
self.assertEqual(expected_ids, actual_ids)
def test_duplicate_uri1(self):
with testing.expect(errors.PoolAlreadyExists):
# The url 'localhost' is used in setUp(). So reusing the uri
# 'localhost' here will raise PoolAlreadyExists.
self.pools_controller.create(str(uuid.uuid1()), 100, self.uri,
flavor=str(uuid.uuid1()), options={})
def test_claim_effects(self):
client_uuid = uuid.uuid4()
_insert_fixtures(self.controller, self.queue_name,
project=self.project, client_uuid=client_uuid, num=12)
def list_messages(include_claimed=None):
kwargs = {
'project': self.project,
'client_uuid': client_uuid,
'echo': True,
}
# Properly test default value
if include_claimed is not None:
kwargs['include_claimed'] = include_claimed
interaction = self.controller.list(self.queue_name, **kwargs)
messages = next(interaction)
return [msg['id'] for msg in messages]
messages_before = list_messages(True)
meta = {'ttl': 70, 'grace': 60}
another_cid, _ = self.claim_controller.create(self.queue_name, meta,
project=self.project)
messages_after = list_messages(True)
self.assertEqual(messages_before, messages_after)
messages_excluding_claimed = list_messages()
self.assertNotEqual(messages_before, messages_excluding_claimed)
self.assertEqual(2, len(messages_excluding_claimed))
cid, msgs = self.claim_controller.create(self.queue_name, meta,
project=self.project)
[msg1, msg2] = msgs
# A wrong claim does not ensure the message deletion
with testing.expect(storage.errors.NotPermitted):
self.controller.delete(self.queue_name, msg1['id'],
project=self.project,
claim=another_cid)
# Make sure a message can be deleted with a claim
self.controller.delete(self.queue_name, msg1['id'],
project=self.project,
claim=cid)
with testing.expect(storage.errors.DoesNotExist):
self.controller.get(self.queue_name, msg1['id'],
project=self.project)
# Make sure such a deletion is idempotent
self.controller.delete(self.queue_name, msg1['id'],
project=self.project,
claim=cid)
# A non-existing claim does not ensure the message deletion
self.claim_controller.delete(self.queue_name, cid,
project=self.project)
with testing.expect(storage.errors.NotPermitted):
self.controller.delete(self.queue_name, msg2['id'],
project=self.project,
claim=cid)
