def list(self,
queue,
project=None,
marker=None,
limit=storage.DEFAULT_MESSAGES_PER_PAGE,
echo=True,
client_uuid=None,
include_claimed=False):
if not self._queue_controller.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
messages_set_id = utils.scope_messages_set(queue, project,
QUEUE_MESSAGES_LIST_SUFFIX)
client = self._client
start = client.zrank(messages_set_id, marker) or 0
# In a pooled environment, the default values are not being set.
limit = limit or storage.DEFAULT_MESSAGES_PER_CLAIM
message_ids = client.zrange(messages_set_id, start, start + limit)
filters = collections.defaultdict(dict)
# Build a list of filters for checking the following:
# 1. Message is claimed.
# 2. echo message to the client.
if not include_claimed:
filters['claimed_filter']['f'] = utils.msg_claimed_filter
filters['claimed_filter']['f.v'] = timeutils.utcnow_ts()
if not echo:
filters['echo_filter']['f'] = utils.msg_echo_filter
filters['echo_filter']['f.v'] = client_uuid
marker = {}
def _it(message_ids, filters={}):
"""Create a filtered iterator of messages.
The function accepts a list of filters to be filtered
before the the message can be included as a part of the reply.
"""
now = timeutils.utcnow_ts()
for message_id in message_ids:
message = client.hgetall(message_id)
if message:
for filter in six.itervalues(filters):
filter_func = filter['f']
filter_val = filter['f.v']
if filter_func(message, filter_val):
break
else:
marker['next'] = message_id
yield utils.basic_message(message, now)
yield _it(message_ids, filters)
yield marker['next']
def _get_first_message_id(self, queue, project, sort):
"""Fetch head/tail of the Queue.
Helper function to get the first message in the queue
sort > 0 get from the left else from the right.
"""
messages_set_id = utils.scope_messages_set(queue, project,
QUEUE_MESSAGES_LIST_SUFFIX)
sorter = self._client.zrange if sort > 0 else self._client.zrevrange
return sorter(messages_set_id, 0, 0)
def _exists(self, queue, project, key):
"""Check if message exists in the Queue.
Helper function which checks if a particular message_id
exists in the sorted set of the queues message ids.
Note(prashanthr_): Operation of the order of O(n)
"""
messages_set_id = utils.scope_messages_set(queue, project,
QUEUE_MESSAGES_LIST_SUFFIX)
return self._client.zrank(messages_set_id, key) is not None
def bulk_delete(self, queue, message_ids, project=None):
if not self._queue_controller.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
messages_set_id = utils.scope_messages_set(queue, project,
QUEUE_MESSAGES_LIST_SUFFIX)
pipe = self._pipeline
for message_id in message_ids:
pipe.delete(message_id).zrem(messages_set_id, message_id)
pipe.execute()
def test_scope_messages_set(self):
self.assertEqual(utils.scope_messages_set('my-q'), '.my-q.')
self.assertEqual(utils.scope_messages_set('my-q', 'p'), 'p.my-q.')
self.assertEqual(utils.scope_messages_set('my-q', 'p', 's'), 'p.my-q.s')
self.assertEqual(utils.scope_messages_set(None), '..')
self.assertEqual(utils.scope_messages_set(None, '123'), '123..')
self.assertEqual(utils.scope_messages_set(None, None, 's'), '..s')
def test_scope_messages_set(self):
self.assertEqual(utils.scope_messages_set('my-q'), '.my-q.')
self.assertEqual(utils.scope_messages_set('my-q', 'p'), 'p.my-q.')
self.assertEqual(utils.scope_messages_set('my-q', 'p', 's'),
'p.my-q.s')
self.assertEqual(utils.scope_messages_set(None), '..')
self.assertEqual(utils.scope_messages_set(None, '123'), '123..')
self.assertEqual(utils.scope_messages_set(None, None, 's'), '..s')
def test_get_count(self):
queue_name = 'get-count'
self.q_controller.create(queue_name)
msgs = [{'ttl': 300, 'body': 'di mo fy'} for i in range(0, 10)]
client_id = uuid.uuid4()
# Creating 10 messages
self.msg_controller.post(queue_name, msgs, client_id)
messages_set_id = utils.scope_messages_set(queue_name, None,
'messages')
num_msg = self.msg_controller._get_count(messages_set_id)
self.assertEqual(num_msg, 10)
def _delete_queue_messages(self, queue, project):
"""Method to remove all the messages belonging to an individual queue.
Will be referenced from the QueueController.
"""
client = self._client
pipe = self._pipeline
messages_set_id = utils.scope_messages_set(queue, project,
QUEUE_MESSAGES_LIST_SUFFIX)
message_ids = client.zrange(messages_set_id, 0, -1)
pipe.delete(messages_set_id)
for msg_id in message_ids:
pipe.delete(msg_id)
pipe.execute()
def test_get_count(self):
queue_name = 'get-count'
self.q_controller.create(queue_name)
msgs = [{
'ttl': 300,
'body': 'di mo fy'
} for i in range(0, 10)]
client_id = uuid.uuid4()
# Creating 10 messages
self.msg_controller.post(queue_name, msgs, client_id)
messages_set_id = utils.scope_messages_set(queue_name, None,
'messages')
num_msg = self.msg_controller._get_count(messages_set_id)
self.assertEqual(num_msg, 10)
def post(self, queue, messages, client_uuid, project=None):
if not self._queue_controller.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
messages_set_id = utils.scope_messages_set(queue, project,
QUEUE_MESSAGES_LIST_SUFFIX)
now = timeutils.utcnow_ts()
now_dt = datetime.datetime.utcfromtimestamp(now)
message_ids = []
num_messages = self._get_count(messages_set_id)
prepared_messages = [{
'id':
utils.generate_uuid(),
't':
message['ttl'],
'e':
now_dt + datetime.timedelta(seconds=message['ttl']),
'u':
client_uuid,
'c':
None,
'k':
num_messages + index,
'c.e':
now,
'b':
message['body'] if 'body' in message else {},
} for index, message in enumerate(messages)]
pipe = self._pipeline
self._queue_controller._inc_counter(queue, project, len(messages))
for message in prepared_messages:
m_id = message['id']
message_ids.append(m_id)
pipe.zadd(messages_set_id, message['c.e'], m_id)
pipe.hmset(m_id, message)
pipe.execute()
return message_ids
def delete(self, queue, message_id, project=None, claim=None):
messages_set_id = utils.scope_messages_set(queue, project,
QUEUE_MESSAGES_LIST_SUFFIX)
pipe = self._pipeline
if not self._queue_controller.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
if not self._exists(queue, project, message_id):
raise errors.MessageDoesNotExist(message_id, queue, project)
now = timeutils.utcnow_ts()
message = self._get(message_id)
msg_claim = message['c']
is_claimed = (msg_claim is not 'None' and int(message['c.e']) > now)
if claim is None:
if is_claimed:
raise errors.MessageIsClaimed(message_id)
else:
if msg_claim != claim:
raise errors.MessageIsClaimedBy(message_id, msg_claim)
pipe.delete(message_id).zrem(messages_set_id, message_id).execute()