def _test_post(self, sample_messages):
sample_doc = json.dumps(sample_messages)
result = self.simulate_post(self.messages_path, self.project_id,
body=sample_doc, headers=self.headers)
self.assertEqual(self.srmock.status, falcon.HTTP_201)
result_doc = json.loads(result[0])
msg_ids = self._get_msg_ids(self.srmock.headers_dict)
self.assertEqual(len(msg_ids), len(sample_messages))
expected_resources = [unicode(self.messages_path + '/' + id)
for id in msg_ids]
self.assertEqual(expected_resources, result_doc['resources'])
self.assertFalse(result_doc['partial'])
self.assertEqual(len(msg_ids), len(sample_messages))
lookup = dict([(m['ttl'], m['body']) for m in sample_messages])
# Test GET on the message resource directly
# NOTE(cpp-cabrera): force the passing of time to age a message
timeutils.set_time_override(timeutils.utcnow())
timeutils.advance_time_seconds(10)
for msg_id in msg_ids:
message_uri = self.messages_path + '/' + msg_id
# Wrong project ID
self.simulate_get(message_uri, '777777')
self.assertEqual(self.srmock.status, falcon.HTTP_404)
# Correct project ID
result = self.simulate_get(message_uri, self.project_id)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
self.assertEqual(self.srmock.headers_dict['Content-Location'],
message_uri)
# Check message properties
message = json.loads(result[0])
self.assertEqual(message['href'], message_uri)
self.assertEqual(message['body'], lookup[message['ttl']])
# no negative age
# NOTE(cpp-cabrera): testtools lacks GreaterThanEqual on py26
self.assertThat(message['age'],
matchers.GreaterThan(-1))
timeutils.clear_time_override()
# Test bulk GET
query_string = 'ids=' + ','.join(msg_ids)
result = self.simulate_get(self.messages_path, self.project_id,
query_string=query_string)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
result_doc = json.loads(result[0])
expected_ttls = set(m['ttl'] for m in sample_messages)
actual_ttls = set(m['ttl'] for m in result_doc)
self.assertFalse(expected_ttls - actual_ttls)
def test_message_counter(self):
queue_name = 'marker_test'
iterations = 10
self.queue_controller.create(queue_name)
seed_marker1 = self.queue_controller._get_counter(queue_name)
self.assertEqual(seed_marker1, 1, 'First marker is 1')
for i in range(iterations):
self.controller.post(queue_name, [{'ttl': 60}], 'uuid')
marker1 = self.queue_controller._get_counter(queue_name)
marker2 = self.queue_controller._get_counter(queue_name)
marker3 = self.queue_controller._get_counter(queue_name)
self.assertEqual(marker1, marker2)
self.assertEqual(marker2, marker3)
self.assertEqual(marker1, i + 2)
new_value = self.queue_controller._inc_counter(queue_name)
self.assertIsNotNone(new_value)
value_before = self.queue_controller._get_counter(queue_name)
new_value = self.queue_controller._inc_counter(queue_name)
self.assertIsNotNone(new_value)
value_after = self.queue_controller._get_counter(queue_name)
self.assertEqual(value_after, value_before + 1)
value_before = value_after
new_value = self.queue_controller._inc_counter(queue_name, amount=7)
value_after = self.queue_controller._get_counter(queue_name)
self.assertEqual(value_after, value_before + 7)
self.assertEqual(value_after, new_value)
reference_value = value_after
unchanged = self.queue_controller._inc_counter(queue_name, window=10)
self.assertIsNone(unchanged)
# TODO(kgriffs): Pass utcnow to work around bug
# in set_time_override until we merge the fix in
# from upstream.
timeutils.set_time_override(timeutils.utcnow())
timeutils.advance_time_seconds(10)
changed = self.queue_controller._inc_counter(queue_name, window=5)
self.assertEqual(changed, reference_value + 1)
timeutils.clear_time_override()
def test_message_counter(self):
queue_name = 'marker_test'
iterations = 10
self.queue_controller.create(queue_name)
seed_marker1 = self.queue_controller._get_counter(queue_name)
self.assertEqual(seed_marker1, 1, 'First marker is 1')
for i in range(iterations):
self.controller.post(queue_name, [{'ttl': 60}], 'uuid')
marker1 = self.queue_controller._get_counter(queue_name)
marker2 = self.queue_controller._get_counter(queue_name)
marker3 = self.queue_controller._get_counter(queue_name)
self.assertEqual(marker1, marker2)
self.assertEqual(marker2, marker3)
self.assertEqual(marker1, i + 2)
new_value = self.queue_controller._inc_counter(queue_name)
self.assertIsNotNone(new_value)
value_before = self.queue_controller._get_counter(queue_name)
new_value = self.queue_controller._inc_counter(queue_name)
self.assertIsNotNone(new_value)
value_after = self.queue_controller._get_counter(queue_name)
self.assertEqual(value_after, value_before + 1)
value_before = value_after
new_value = self.queue_controller._inc_counter(queue_name, amount=7)
value_after = self.queue_controller._get_counter(queue_name)
self.assertEqual(value_after, value_before + 7)
self.assertEqual(value_after, new_value)
reference_value = value_after
unchanged = self.queue_controller._inc_counter(queue_name, window=10)
self.assertIsNone(unchanged)
# TODO(kgriffs): Pass utcnow to work around bug
# in set_time_override until we merge the fix in
# from upstream.
timeutils.set_time_override(timeutils.utcnow())
timeutils.advance_time_seconds(10)
changed = self.queue_controller._inc_counter(queue_name, window=5)
self.assertEqual(changed, reference_value + 1)
timeutils.clear_time_override()
def test_lifecycle(self):
doc = '{"ttl": 100, "grace": 60}'
# First, claim some messages
body = self.simulate_post(self.claims_path, self.project_id, body=doc)
self.assertEqual(self.srmock.status, falcon.HTTP_201)
claimed = json.loads(body[0])
claim_href = self.srmock.headers_dict['Location']
message_href, params = claimed[0]['href'].split('?')
# No more messages to claim
self.simulate_post(self.claims_path,
self.project_id,
body=doc,
query_string='limit=3')
self.assertEqual(self.srmock.status, falcon.HTTP_204)
headers = {
'Client-ID': str(uuid.uuid4()),
}
# Listing messages, by default, won't include claimed
body = self.simulate_get(self.messages_path,
self.project_id,
headers=headers)
self.assertEqual(self.srmock.status, falcon.HTTP_204)
# Include claimed messages this time
body = self.simulate_get(self.messages_path,
self.project_id,
query_string='include_claimed=true',
headers=headers)
listed = json.loads(body[0])
self.assertEqual(self.srmock.status, falcon.HTTP_200)
self.assertEqual(len(listed['messages']), len(claimed))
# Check the claim's metadata
## NOTE(cpp-cabrera): advance time to force claim aging
timeutils.set_time_override(timeutils.utcnow())
timeutils.advance_time_seconds(10)
body = self.simulate_get(claim_href, self.project_id)
timeutils.clear_time_override()
claim = json.loads(body[0])
self.assertEqual(self.srmock.status, falcon.HTTP_200)
self.assertEqual(self.srmock.headers_dict['Content-Location'],
claim_href)
self.assertEqual(claim['ttl'], 100)
## NOTE(cpp-cabrera): verify that claim age is non-negative
self.assertThat(claim['age'], matchers.GreaterThan(-1))
# Try to delete the message without submitting a claim_id
self.simulate_delete(message_href, self.project_id)
self.assertEqual(self.srmock.status, falcon.HTTP_403)
# Delete the message and its associated claim
self.simulate_delete(message_href,
self.project_id,
query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_204)
# Try to get it from the wrong project
self.simulate_get(message_href, 'bogus_project', query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_404)
# Get the message
self.simulate_get(message_href, self.project_id, query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_404)
# Update the claim
new_claim_ttl = '{"ttl": 60}'
creation = timeutils.utcnow()
self.simulate_patch(claim_href, self.project_id, body=new_claim_ttl)
self.assertEqual(self.srmock.status, falcon.HTTP_204)
# Get the claimed messages (again)
body = self.simulate_get(claim_href, self.project_id)
query = timeutils.utcnow()
claim = json.loads(body[0])
message_href, params = claim['messages'][0]['href'].split('?')
self.assertEqual(claim['ttl'], 60)
estimated_age = timeutils.delta_seconds(creation, query)
self.assertTrue(estimated_age > claim['age'])
# Delete the claim
self.simulate_delete(claim['href'], 'bad_id')
self.assertEqual(self.srmock.status, falcon.HTTP_204)
self.simulate_delete(claim['href'], self.project_id)
self.assertEqual(self.srmock.status, falcon.HTTP_204)
# Try to delete a message with an invalid claim ID
self.simulate_delete(message_href,
self.project_id,
query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_403)
# Make sure it wasn't deleted!
self.simulate_get(message_href, self.project_id, query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
# Try to get a claim that doesn't exist
self.simulate_get(claim['href'])
self.assertEqual(self.srmock.status, falcon.HTTP_404)
# Try to update a claim that doesn't exist
self.simulate_patch(claim['href'], body=doc)
self.assertEqual(self.srmock.status, falcon.HTTP_404)
def test_queue_lifecycle(self):
# Test Queue Creation
created = self.controller.create('test', project=self.project)
self.assertTrue(created)
# Test Queue Existence
self.assertTrue(self.controller.exists('test', project=self.project))
# Test Queue retrieval
metadata = self.controller.get_metadata('test', project=self.project)
self.assertEqual(metadata, {})
# Test Queue Update
created = self.controller.set_metadata('test', project=self.project,
metadata=dict(meta='test_meta'))
metadata = self.controller.get_metadata('test', project=self.project)
self.assertEqual(metadata['meta'], 'test_meta')
# Touching an existing queue does not affect metadata
created = self.controller.create('test', project=self.project)
self.assertFalse(created)
metadata = self.controller.get_metadata('test', project=self.project)
self.assertEqual(metadata['meta'], 'test_meta')
# Test Queue Statistic
_insert_fixtures(self.message_controller, 'test',
project=self.project, client_uuid='my_uuid',
num=6)
# NOTE(kgriffs): We can't get around doing this, because
# we don't know how the storage drive may be calculating
# message timestamps (and may not be monkey-patchable).
time.sleep(1)
_insert_fixtures(self.message_controller, 'test',
project=self.project, client_uuid='my_uuid',
num=6)
stats = self.controller.stats('test', project=self.project)
message_stats = stats['messages']
self.assertEqual(message_stats['free'], 12)
self.assertEqual(message_stats['claimed'], 0)
self.assertEqual(message_stats['total'], 12)
oldest = message_stats['oldest']
newest = message_stats['newest']
self.assertNotEqual(oldest, newest)
# NOTE(kgriffs): Ensure "now" is different enough
# for the next comparison to work.
timeutils.set_time_override()
timeutils.advance_time_seconds(10)
for message_stat in (oldest, newest):
created_iso = message_stat['created']
created = timeutils.parse_isotime(created_iso)
self.assertThat(timeutils.normalize_time(created),
matchers.LessThan(timeutils.utcnow()))
self.assertIn('id', message_stat)
self.assertThat(oldest['created'],
matchers.LessThan(newest['created']))
# Test Queue Deletion
self.controller.delete('test', project=self.project)
# Test Queue Existence
self.assertFalse(self.controller.exists('test', project=self.project))
# Test DoesNotExist Exception
with testing.expect(storage.exceptions.DoesNotExist):
self.controller.get_metadata('test', project=self.project)
with testing.expect(storage.exceptions.DoesNotExist):
self.controller.set_metadata('test', '{}', project=self.project)
def _test_post(self, sample_messages):
sample_doc = json.dumps(sample_messages)
result = self.simulate_post(self.messages_path,
self.project_id,
body=sample_doc,
headers=self.headers)
self.assertEqual(self.srmock.status, falcon.HTTP_201)
result_doc = json.loads(result[0])
msg_ids = self._get_msg_ids(self.srmock.headers_dict)
self.assertEqual(len(msg_ids), len(sample_messages))
expected_resources = [
unicode(self.messages_path + '/' + id) for id in msg_ids
]
self.assertEqual(expected_resources, result_doc['resources'])
self.assertFalse(result_doc['partial'])
self.assertEqual(len(msg_ids), len(sample_messages))
lookup = dict([(m['ttl'], m['body']) for m in sample_messages])
# Test GET on the message resource directly
# NOTE(cpp-cabrera): force the passing of time to age a message
timeutils.set_time_override(timeutils.utcnow())
timeutils.advance_time_seconds(10)
for msg_id in msg_ids:
message_uri = self.messages_path + '/' + msg_id
# Wrong project ID
self.simulate_get(message_uri, '777777')
self.assertEqual(self.srmock.status, falcon.HTTP_404)
# Correct project ID
result = self.simulate_get(message_uri, self.project_id)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
self.assertEqual(self.srmock.headers_dict['Content-Location'],
message_uri)
# Check message properties
message = json.loads(result[0])
self.assertEqual(message['href'], message_uri)
self.assertEqual(message['body'], lookup[message['ttl']])
# no negative age
# NOTE(cpp-cabrera): testtools lacks GreaterThanEqual on py26
self.assertThat(message['age'], matchers.GreaterThan(-1))
timeutils.clear_time_override()
# Test bulk GET
query_string = 'ids=' + ','.join(msg_ids)
result = self.simulate_get(self.messages_path,
self.project_id,
query_string=query_string)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
result_doc = json.loads(result[0])
expected_ttls = set(m['ttl'] for m in sample_messages)
actual_ttls = set(m['ttl'] for m in result_doc)
self.assertFalse(expected_ttls - actual_ttls)
def test_lifecycle(self):
doc = '{"ttl": 100, "grace": 60}'
# First, claim some messages
body = self.simulate_post(self.claims_path, self.project_id, body=doc)
self.assertEqual(self.srmock.status, falcon.HTTP_201)
claimed = json.loads(body[0])
claim_href = self.srmock.headers_dict['Location']
message_href, params = claimed[0]['href'].split('?')
# No more messages to claim
self.simulate_post(self.claims_path, self.project_id, body=doc,
query_string='limit=3')
self.assertEqual(self.srmock.status, falcon.HTTP_204)
headers = {
'Client-ID': str(uuid.uuid4()),
}
# Listing messages, by default, won't include claimed
body = self.simulate_get(self.messages_path, self.project_id,
headers=headers)
self.assertEqual(self.srmock.status, falcon.HTTP_204)
# Include claimed messages this time
body = self.simulate_get(self.messages_path, self.project_id,
query_string='include_claimed=true',
headers=headers)
listed = json.loads(body[0])
self.assertEqual(self.srmock.status, falcon.HTTP_200)
self.assertEqual(len(listed['messages']), len(claimed))
# Check the claim's metadata
## NOTE(cpp-cabrera): advance time to force claim aging
timeutils.set_time_override(timeutils.utcnow())
timeutils.advance_time_seconds(10)
body = self.simulate_get(claim_href, self.project_id)
timeutils.clear_time_override()
claim = json.loads(body[0])
self.assertEqual(self.srmock.status, falcon.HTTP_200)
self.assertEqual(self.srmock.headers_dict['Content-Location'],
claim_href)
self.assertEqual(claim['ttl'], 100)
## NOTE(cpp-cabrera): verify that claim age is non-negative
self.assertThat(claim['age'], matchers.GreaterThan(-1))
# Try to delete the message without submitting a claim_id
self.simulate_delete(message_href, self.project_id)
self.assertEqual(self.srmock.status, falcon.HTTP_403)
# Delete the message and its associated claim
self.simulate_delete(message_href, self.project_id,
query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_204)
# Try to get it from the wrong project
self.simulate_get(message_href, 'bogus_project', query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_404)
# Get the message
self.simulate_get(message_href, self.project_id, query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_404)
# Update the claim
new_claim_ttl = '{"ttl": 60}'
creation = timeutils.utcnow()
self.simulate_patch(claim_href, self.project_id, body=new_claim_ttl)
self.assertEqual(self.srmock.status, falcon.HTTP_204)
# Get the claimed messages (again)
body = self.simulate_get(claim_href, self.project_id)
query = timeutils.utcnow()
claim = json.loads(body[0])
message_href, params = claim['messages'][0]['href'].split('?')
self.assertEqual(claim['ttl'], 60)
estimated_age = timeutils.delta_seconds(creation, query)
self.assertTrue(estimated_age > claim['age'])
# Delete the claim
self.simulate_delete(claim['href'], 'bad_id')
self.assertEqual(self.srmock.status, falcon.HTTP_204)
self.simulate_delete(claim['href'], self.project_id)
self.assertEqual(self.srmock.status, falcon.HTTP_204)
# Try to delete a message with an invalid claim ID
self.simulate_delete(message_href, self.project_id,
query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_403)
# Make sure it wasn't deleted!
self.simulate_get(message_href, self.project_id, query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
# Try to get a claim that doesn't exist
self.simulate_get(claim['href'])
self.assertEqual(self.srmock.status, falcon.HTTP_404)
# Try to update a claim that doesn't exist
self.simulate_patch(claim['href'], body=doc)
self.assertEqual(self.srmock.status, falcon.HTTP_404)
def test_queue_lifecycle(self):
# Test Queue Creation
created = self.controller.create('test', project=self.project)
self.assertTrue(created)
# Test Queue Existence
self.assertTrue(self.controller.exists('test', project=self.project))
# Test Queue retrieval
metadata = self.controller.get_metadata('test', project=self.project)
self.assertEqual(metadata, {})
# Test Queue Update
created = self.controller.set_metadata('test',
project=self.project,
metadata=dict(meta='test_meta'))
metadata = self.controller.get_metadata('test', project=self.project)
self.assertEqual(metadata['meta'], 'test_meta')
# Touching an existing queue does not affect metadata
created = self.controller.create('test', project=self.project)
self.assertFalse(created)
metadata = self.controller.get_metadata('test', project=self.project)
self.assertEqual(metadata['meta'], 'test_meta')
# Test Queue Statistic
_insert_fixtures(self.message_controller,
'test',
project=self.project,
client_uuid='my_uuid',
num=6)
# NOTE(kgriffs): We can't get around doing this, because
# we don't know how the storage drive may be calculating
# message timestamps (and may not be monkey-patchable).
time.sleep(1)
_insert_fixtures(self.message_controller,
'test',
project=self.project,
client_uuid='my_uuid',
num=6)
stats = self.controller.stats('test', project=self.project)
message_stats = stats['messages']
self.assertEqual(message_stats['free'], 12)
self.assertEqual(message_stats['claimed'], 0)
self.assertEqual(message_stats['total'], 12)
oldest = message_stats['oldest']
newest = message_stats['newest']
self.assertNotEqual(oldest, newest)
# NOTE(kgriffs): Ensure "now" is different enough
# for the next comparison to work.
timeutils.set_time_override()
timeutils.advance_time_seconds(10)
for message_stat in (oldest, newest):
created_iso = message_stat['created']
created = timeutils.parse_isotime(created_iso)
self.assertThat(timeutils.normalize_time(created),
matchers.LessThan(timeutils.utcnow()))
self.assertIn('id', message_stat)
self.assertThat(oldest['created'],
matchers.LessThan(newest['created']))
# Test Queue Deletion
self.controller.delete('test', project=self.project)
# Test Queue Existence
self.assertFalse(self.controller.exists('test', project=self.project))
# Test DoesNotExist Exception
with testing.expect(storage.exceptions.DoesNotExist):
self.controller.get_metadata('test', project=self.project)
with testing.expect(storage.exceptions.DoesNotExist):
self.controller.set_metadata('test', '{}', project=self.project)