def oid_ts(oid):
"""Converts an ObjectId to a UNIX timestamp.
:raises: TypeError if oid isn't an ObjectId
"""
try:
return timeutils.delta_seconds(EPOCH, oid.generation_time)
except AttributeError:
raise TypeError(u"Expected ObjectId and got %s" % type(oid))
def oid_ts(oid):
"""Converts an ObjectId to a UNIX timestamp.
:raises: TypeError if oid isn't an ObjectId
"""
try:
return timeutils.delta_seconds(EPOCH, oid.generation_time)
except AttributeError:
raise TypeError(u'Expected ObjectId and got %s' % type(oid))
def _basic_message(msg, now):
oid = msg['_id']
age = timeutils.delta_seconds(utils.oid_utc(oid), now)
return {
'id': str(oid),
'age': int(age),
'ttl': msg['t'],
'body': msg['b'],
}
def _basic_message(msg, now):
oid = msg['_id']
age = timeutils.delta_seconds(utils.oid_utc(oid), now)
return {
'id': str(oid),
'age': int(age),
'ttl': msg['t'],
'body': msg['b'],
}
def stat_message(message, now):
"""Creates a stat document from the given message, relative to now."""
oid = message['_id']
created = oid_utc(oid)
age = timeutils.delta_seconds(created, now)
return {
'id': str(oid),
'age': int(age),
'created': timeutils.isotime(created),
}
def stat_message(message, now):
"""Creates a stat document from the given message, relative to now."""
oid = message['_id']
created = oid_utc(oid)
age = timeutils.delta_seconds(created, now)
return {
'id': str(oid),
'age': int(age),
'created': timeutils.isotime(created),
}
def get(self, queue, claim_id, project=None):
msg_ctrl = self.driver.message_controller
# Base query, always check expire time
now = timeutils.utcnow()
cid = utils.to_oid(claim_id)
if cid is None:
raise exceptions.ClaimDoesNotExist(queue, project, claim_id)
def messages(msg_iter):
msg = next(msg_iter)
yield msg.pop('claim')
yield msg
# Smoke it!
for msg in msg_iter:
del msg['claim']
yield msg
try:
# Lets get claim's data
# from the first message
# in the iterator
msgs = messages(msg_ctrl.claimed(queue, cid, now,
project=project))
claim = next(msgs)
update_time = claim['e'] - datetime.timedelta(seconds=claim['t'])
age = timeutils.delta_seconds(update_time, now)
claim = {
'age': int(age),
'ttl': claim.pop('t'),
'id': str(claim['id']),
}
except StopIteration:
raise exceptions.ClaimDoesNotExist(cid, queue, project)
return (claim, msgs)
def verify_message_stats(self, message):
"""Verifies the oldest & newest message stats
:param message: oldest (or) newest message returned by
queue_name/stats.
"""
expected_keys = ['age', 'created', 'href']
response_keys = message.keys()
response_keys.sort()
self.assertEqual(response_keys, expected_keys)
# Verify that age has valid values
age = message['age']
self.assertTrue(0 <= age <= self.limits.max_message_ttl,
msg='Invalid Age {0}'.format(age))
# Verify that GET on href returns 200
path = message['href']
result = self.client.get(path)
self.assertEqual(result.status_code, 200)
# Verify that created time falls within the last 10 minutes
# NOTE(malini): The messages are created during the test.
created_time = message['created']
created_time = timeutils.normalize_time(
timeutils.parse_isotime(created_time))
now = timeutils.utcnow()
delta = timeutils.delta_seconds(before=created_time, after=now)
# NOTE(malini): The 'int()' below is a work around for the small time
# difference between julianday & UTC.
# (needed to pass this test on sqlite driver)
delta = int(delta)
msg = 'Invalid Time Delta {0}, Created time {1}, Now {2}' \
.format(delta, created_time, now)
self.assertTrue(0 <= delta <= 6000, msg)
def verify_message_stats(self, message):
"""Verifies the oldest & newest message stats
:param message: oldest (or) newest message returned by
queue_name/stats.
"""
expected_keys = ['age', 'created', 'href']
response_keys = message.keys()
response_keys.sort()
self.assertEqual(response_keys, expected_keys)
# Verify that age has valid values
age = message['age']
self.assertTrue(0 <= age <= self.limits.max_message_ttl,
msg='Invalid Age {0}'.format(age))
# Verify that GET on href returns 200
path = message['href']
result = self.client.get(path)
self.assertEqual(result.status_code, 200)
# Verify that created time falls within the last 10 minutes
# NOTE(malini): The messages are created during the test.
created_time = message['created']
created_time = timeutils.normalize_time(
timeutils.parse_isotime(created_time))
now = timeutils.utcnow()
delta = timeutils.delta_seconds(before=created_time, after=now)
# NOTE(malini): The 'int()' below is a work around for the small time
# difference between julianday & UTC.
# (needed to pass this test on sqlite driver)
delta = int(delta)
msg = 'Invalid Time Delta {0}, Created time {1}, Now {2}' \
.format(delta, created_time, now)
self.assertTrue(0 <= delta <= 6000, msg)
def get(self, queue, claim_id, project=None):
msg_ctrl = self.driver.message_controller
# Base query, always check expire time
now = timeutils.utcnow()
cid = utils.to_oid(claim_id)
if cid is None:
raise exceptions.ClaimDoesNotExist(queue, project, claim_id)
def messages(msg_iter):
msg = next(msg_iter)
yield msg.pop('claim')
yield msg
# Smoke it!
for msg in msg_iter:
del msg['claim']
yield msg
try:
# Lets get claim's data
# from the first message
# in the iterator
msgs = messages(msg_ctrl.claimed(queue, cid, now, project=project))
claim = next(msgs)
update_time = claim['e'] - datetime.timedelta(seconds=claim['t'])
age = timeutils.delta_seconds(update_time, now)
claim = {
'age': int(age),
'ttl': claim.pop('t'),
'id': str(claim['id']),
}
except StopIteration:
raise exceptions.ClaimDoesNotExist(cid, queue, project)
return (claim, msgs)
def test_lifecycle(self):
doc = '{"ttl": 100, "grace": 60}'
# First, claim some messages
body = self.simulate_post(self.claims_path, self.project_id, body=doc)
self.assertEqual(self.srmock.status, falcon.HTTP_201)
claimed = json.loads(body[0])
claim_href = self.srmock.headers_dict['Location']
message_href, params = claimed[0]['href'].split('?')
# No more messages to claim
self.simulate_post(self.claims_path,
self.project_id,
body=doc,
query_string='limit=3')
self.assertEqual(self.srmock.status, falcon.HTTP_204)
headers = {
'Client-ID': str(uuid.uuid4()),
}
# Listing messages, by default, won't include claimed
body = self.simulate_get(self.messages_path,
self.project_id,
headers=headers)
self.assertEqual(self.srmock.status, falcon.HTTP_204)
# Include claimed messages this time
body = self.simulate_get(self.messages_path,
self.project_id,
query_string='include_claimed=true',
headers=headers)
listed = json.loads(body[0])
self.assertEqual(self.srmock.status, falcon.HTTP_200)
self.assertEqual(len(listed['messages']), len(claimed))
now = timeutils.utcnow() + datetime.timedelta(seconds=10)
timeutils_utcnow = 'marconi.openstack.common.timeutils.utcnow'
with mock.patch(timeutils_utcnow) as mock_utcnow:
mock_utcnow.return_value = now
body = self.simulate_get(claim_href, self.project_id)
claim = json.loads(body[0])
self.assertEqual(self.srmock.status, falcon.HTTP_200)
self.assertEqual(self.srmock.headers_dict['Content-Location'],
claim_href)
self.assertEqual(claim['ttl'], 100)
## NOTE(cpp-cabrera): verify that claim age is non-negative
self.assertThat(claim['age'], matchers.GreaterThan(-1))
# Try to delete the message without submitting a claim_id
self.simulate_delete(message_href, self.project_id)
self.assertEqual(self.srmock.status, falcon.HTTP_403)
# Delete the message and its associated claim
self.simulate_delete(message_href,
self.project_id,
query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_204)
# Try to get it from the wrong project
self.simulate_get(message_href, 'bogus_project', query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_404)
# Get the message
self.simulate_get(message_href, self.project_id, query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_404)
# Update the claim
new_claim_ttl = '{"ttl": 60}'
creation = timeutils.utcnow()
self.simulate_patch(claim_href, self.project_id, body=new_claim_ttl)
self.assertEqual(self.srmock.status, falcon.HTTP_204)
# Get the claimed messages (again)
body = self.simulate_get(claim_href, self.project_id)
query = timeutils.utcnow()
claim = json.loads(body[0])
message_href, params = claim['messages'][0]['href'].split('?')
self.assertEqual(claim['ttl'], 60)
estimated_age = timeutils.delta_seconds(creation, query)
self.assertTrue(estimated_age > claim['age'])
# Delete the claim
self.simulate_delete(claim['href'], 'bad_id')
self.assertEqual(self.srmock.status, falcon.HTTP_204)
self.simulate_delete(claim['href'], self.project_id)
self.assertEqual(self.srmock.status, falcon.HTTP_204)
# Try to delete a message with an invalid claim ID
self.simulate_delete(message_href,
self.project_id,
query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_403)
# Make sure it wasn't deleted!
self.simulate_get(message_href, self.project_id, query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
# Try to get a claim that doesn't exist
self.simulate_get(claim['href'])
self.assertEqual(self.srmock.status, falcon.HTTP_404)
# Try to update a claim that doesn't exist
self.simulate_patch(claim['href'], body=doc)
self.assertEqual(self.srmock.status, falcon.HTTP_404)
def test_lifecycle(self):
doc = '{"ttl": 100, "grace": 60}'
# First, claim some messages
body = self.simulate_post(self.claims_path, self.project_id, body=doc)
self.assertEquals(self.srmock.status, falcon.HTTP_201)
claimed = json.loads(body[0])
claim_href = self.srmock.headers_dict['Location']
message_href, params = claimed[0]['href'].split('?')
# No more messages to claim
self.simulate_post(self.claims_path, self.project_id, body=doc,
query_string='limit=3')
self.assertEquals(self.srmock.status, falcon.HTTP_204)
# Listing messages, by default, won't include claimed
body = self.simulate_get(self.messages_path, self.project_id,
headers={'Client-ID': 'foo'})
self.assertEquals(self.srmock.status, falcon.HTTP_204)
# Include claimed messages this time
body = self.simulate_get(self.messages_path, self.project_id,
query_string='include_claimed=true',
headers={'Client-ID': 'foo'})
listed = json.loads(body[0])
self.assertEquals(self.srmock.status, falcon.HTTP_200)
self.assertEquals(len(listed['messages']), len(claimed))
# Check the claim's metadata
body = self.simulate_get(claim_href, self.project_id)
claim = json.loads(body[0])
self.assertEquals(self.srmock.status, falcon.HTTP_200)
self.assertEquals(self.srmock.headers_dict['Content-Location'],
claim_href)
self.assertEquals(claim['ttl'], 100)
# Try to delete the message without submitting a claim_id
self.simulate_delete(message_href, self.project_id)
self.assertEquals(self.srmock.status, falcon.HTTP_403)
# Delete the message and its associated claim
self.simulate_delete(message_href, self.project_id,
query_string=params)
self.assertEquals(self.srmock.status, falcon.HTTP_204)
# Try to get it from the wrong project
self.simulate_get(message_href, 'bogus_project', query_string=params)
self.assertEquals(self.srmock.status, falcon.HTTP_404)
# Get the message
self.simulate_get(message_href, self.project_id, query_string=params)
self.assertEquals(self.srmock.status, falcon.HTTP_404)
# Update the claim
new_claim_ttl = '{"ttl": 60}'
creation = timeutils.utcnow()
self.simulate_patch(claim_href, self.project_id, body=new_claim_ttl)
self.assertEquals(self.srmock.status, falcon.HTTP_204)
# Get the claimed messages (again)
body = self.simulate_get(claim_href, self.project_id)
query = timeutils.utcnow()
claim = json.loads(body[0])
message_href, params = claim['messages'][0]['href'].split('?')
self.assertEquals(claim['ttl'], 60)
estimated_age = timeutils.delta_seconds(creation, query)
self.assertTrue(estimated_age > claim['age'])
# Delete the claim
self.simulate_delete(claim['href'], 'bad_id')
self.assertEquals(self.srmock.status, falcon.HTTP_204)
self.simulate_delete(claim['href'], self.project_id)
self.assertEquals(self.srmock.status, falcon.HTTP_204)
# Try to delete a message with an invalid claim ID
self.simulate_delete(message_href, self.project_id,
query_string=params)
self.assertEquals(self.srmock.status, falcon.HTTP_403)
# Make sure it wasn't deleted!
self.simulate_get(message_href, self.project_id, query_string=params)
self.assertEquals(self.srmock.status, falcon.HTTP_200)
# Try to get a claim that doesn't exist
self.simulate_get(claim['href'])
self.assertEquals(self.srmock.status, falcon.HTTP_404)
# Try to update a claim that doesn't exist
self.simulate_patch(claim['href'], body=doc)
self.assertEquals(self.srmock.status, falcon.HTTP_404)
def test_lifecycle(self):
doc = '{"ttl": 100, "grace": 60}'
# First, claim some messages
body = self.simulate_post(self.claims_path, self.project_id, body=doc)
self.assertEquals(self.srmock.status, falcon.HTTP_201)
claimed = json.loads(body[0])
claim_href = self.srmock.headers_dict['Location']
message_href, params = claimed[0]['href'].split('?')
# No more messages to claim
self.simulate_post(self.claims_path, self.project_id, body=doc,
query_string='limit=3')
self.assertEquals(self.srmock.status, falcon.HTTP_204)
# Listing messages, by default, won't include claimed
body = self.simulate_get(self.messages_path, self.project_id,
headers={'Client-ID': 'foo'})
self.assertEquals(self.srmock.status, falcon.HTTP_204)
# Include claimed messages this time
body = self.simulate_get(self.messages_path, self.project_id,
query_string='include_claimed=true',
headers={'Client-ID': 'foo'})
listed = json.loads(body[0])
self.assertEquals(self.srmock.status, falcon.HTTP_200)
self.assertEquals(len(listed['messages']), len(claimed))
# Check the claim's metadata
body = self.simulate_get(claim_href, self.project_id)
claim = json.loads(body[0])
self.assertEquals(self.srmock.status, falcon.HTTP_200)
self.assertEquals(self.srmock.headers_dict['Content-Location'],
claim_href)
self.assertEquals(claim['ttl'], 100)
# Try to delete the message without submitting a claim_id
self.simulate_delete(message_href, self.project_id)
self.assertEquals(self.srmock.status, falcon.HTTP_403)
# Delete the message and its associated claim
self.simulate_delete(message_href, self.project_id,
query_string=params)
self.assertEquals(self.srmock.status, falcon.HTTP_204)
# Try to get it from the wrong project
self.simulate_get(message_href, 'bogus_project', query_string=params)
self.assertEquals(self.srmock.status, falcon.HTTP_404)
# Get the message
self.simulate_get(message_href, self.project_id, query_string=params)
self.assertEquals(self.srmock.status, falcon.HTTP_404)
# Update the claim
new_claim_ttl = '{"ttl": 60}'
creation = timeutils.utcnow()
self.simulate_patch(claim_href, self.project_id, body=new_claim_ttl)
self.assertEquals(self.srmock.status, falcon.HTTP_204)
# Get the claimed messages (again)
body = self.simulate_get(claim_href, self.project_id)
query = timeutils.utcnow()
claim = json.loads(body[0])
message_href, params = claim['messages'][0]['href'].split('?')
self.assertEquals(claim['ttl'], 60)
estimated_age = timeutils.delta_seconds(creation, query)
self.assertTrue(estimated_age > claim['age'])
# Delete the claim
self.simulate_delete(claim['href'], 'bad_id')
self.assertEquals(self.srmock.status, falcon.HTTP_204)
self.simulate_delete(claim['href'], self.project_id)
self.assertEquals(self.srmock.status, falcon.HTTP_204)
# Try to delete a message with an invalid claim ID
self.simulate_delete(message_href, self.project_id,
query_string=params)
self.assertEquals(self.srmock.status, falcon.HTTP_403)
# Make sure it wasn't deleted!
self.simulate_get(message_href, self.project_id, query_string=params)
self.assertEquals(self.srmock.status, falcon.HTTP_200)
# Try to get a claim that doesn't exist
self.simulate_get(claim['href'])
self.assertEquals(self.srmock.status, falcon.HTTP_404)
# Try to update a claim that doesn't exist
self.simulate_patch(claim['href'], body=doc)
self.assertEquals(self.srmock.status, falcon.HTTP_404)
def test_lifecycle(self):
doc = '{"ttl": 100, "grace": 60}'
# First, claim some messages
body = self.simulate_post(self.claims_path, self.project_id, body=doc)
self.assertEqual(self.srmock.status, falcon.HTTP_201)
claimed = json.loads(body[0])
claim_href = self.srmock.headers_dict['Location']
message_href, params = claimed[0]['href'].split('?')
# No more messages to claim
self.simulate_post(self.claims_path, self.project_id, body=doc,
query_string='limit=3')
self.assertEqual(self.srmock.status, falcon.HTTP_204)
headers = {
'Client-ID': str(uuid.uuid4()),
}
# Listing messages, by default, won't include claimed
body = self.simulate_get(self.messages_path, self.project_id,
headers=headers)
self.assertEqual(self.srmock.status, falcon.HTTP_204)
# Include claimed messages this time
body = self.simulate_get(self.messages_path, self.project_id,
query_string='include_claimed=true',
headers=headers)
listed = json.loads(body[0])
self.assertEqual(self.srmock.status, falcon.HTTP_200)
self.assertEqual(len(listed['messages']), len(claimed))
now = timeutils.utcnow() + datetime.timedelta(seconds=10)
timeutils_utcnow = 'marconi.openstack.common.timeutils.utcnow'
with mock.patch(timeutils_utcnow) as mock_utcnow:
mock_utcnow.return_value = now
body = self.simulate_get(claim_href, self.project_id)
claim = json.loads(body[0])
self.assertEqual(self.srmock.status, falcon.HTTP_200)
self.assertEqual(self.srmock.headers_dict['Content-Location'],
claim_href)
self.assertEqual(claim['ttl'], 100)
## NOTE(cpp-cabrera): verify that claim age is non-negative
self.assertThat(claim['age'], matchers.GreaterThan(-1))
# Try to delete the message without submitting a claim_id
self.simulate_delete(message_href, self.project_id)
self.assertEqual(self.srmock.status, falcon.HTTP_403)
# Delete the message and its associated claim
self.simulate_delete(message_href, self.project_id,
query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_204)
# Try to get it from the wrong project
self.simulate_get(message_href, 'bogus_project', query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_404)
# Get the message
self.simulate_get(message_href, self.project_id, query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_404)
# Update the claim
new_claim_ttl = '{"ttl": 60}'
creation = timeutils.utcnow()
self.simulate_patch(claim_href, self.project_id, body=new_claim_ttl)
self.assertEqual(self.srmock.status, falcon.HTTP_204)
# Get the claimed messages (again)
body = self.simulate_get(claim_href, self.project_id)
query = timeutils.utcnow()
claim = json.loads(body[0])
message_href, params = claim['messages'][0]['href'].split('?')
self.assertEqual(claim['ttl'], 60)
estimated_age = timeutils.delta_seconds(creation, query)
self.assertTrue(estimated_age > claim['age'])
# Delete the claim
self.simulate_delete(claim['href'], 'bad_id')
self.assertEqual(self.srmock.status, falcon.HTTP_204)
self.simulate_delete(claim['href'], self.project_id)
self.assertEqual(self.srmock.status, falcon.HTTP_204)
# Try to delete a message with an invalid claim ID
self.simulate_delete(message_href, self.project_id,
query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_403)
# Make sure it wasn't deleted!
self.simulate_get(message_href, self.project_id, query_string=params)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
# Try to get a claim that doesn't exist
self.simulate_get(claim['href'])
self.assertEqual(self.srmock.status, falcon.HTTP_404)
# Try to update a claim that doesn't exist
self.simulate_patch(claim['href'], body=doc)
self.assertEqual(self.srmock.status, falcon.HTTP_404)
def _basic_message(msg, now):
oid = msg["_id"]
age = timeutils.delta_seconds(utils.oid_utc(oid), now)
return {"id": str(oid), "age": int(age), "ttl": msg["t"], "body": msg["b"]}