def get(self, queue, message_id, project=None):
target = self._lookup(queue, project)
if target:
control = target.message_controller
return control.get(queue, message_id=message_id,
project=project)
raise errors.QueueDoesNotExist(queue, project)
def post(self, queue, messages, client_uuid, project=None):
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
msgset_key = utils.msgset_key(queue, project)
counter_key = utils.scope_queue_index(queue, project,
MESSAGE_RANK_COUNTER_SUFFIX)
message_ids = []
now = timeutils.utcnow_ts()
with self._client.pipeline() as pipe:
for msg in messages:
prepared_msg = Message(
ttl=msg['ttl'],
created=now,
client_uuid=client_uuid,
claim_id=None,
claim_expires=now,
body=msg.get('body', {}),
)
prepared_msg.to_redis(pipe)
message_ids.append(prepared_msg.id)
pipe.execute()
# NOTE(kgriffs): If this call fails, we will return
# an error to the client and the messages will be
# orphaned, but Redis will remove them when they
# expire, so we will just pretend they don't exist
# in that case.
self._index_messages(msgset_key, counter_key, message_ids)
return message_ids
def stats(self, name, project=None):
if not self.exists(name, project=project):
raise errors.QueueDoesNotExist(name, project)
total = self._message_ctrl._count(name, project)
if total:
claimed = self._claim_ctrl._count_messages(name, project)
else:
claimed = 0
message_stats = {
'claimed': claimed,
'free': total - claimed,
'total': total,
}
if total:
try:
newest = self._message_ctrl.first(name, project, -1)
oldest = self._message_ctrl.first(name, project, 1)
except errors.QueueIsEmpty:
pass
else:
message_stats['newest'] = newest
message_stats['oldest'] = oldest
return {'messages': message_stats}
def set_metadata(self, name, metadata, project=None):
target = self._lookup(name, project)
if target:
control = target.queue_controller
return control.set_metadata(name, metadata=metadata,
project=project)
raise errors.QueueDoesNotExist(name, project)
def get_qid(driver, queue, project):
sel = sa.sql.select([tables.Queues.c.id],
sa.and_(tables.Queues.c.project == project,
tables.Queues.c.name == queue))
try:
return driver.get(sel)[0]
except NoResult:
raise errors.QueueDoesNotExist(queue, project)
def post(self, queue, messages, client_uuid, project=None):
target = self._lookup(queue, project)
if target:
control = target.message_controller
return control.post(queue, project=project,
messages=messages,
client_uuid=client_uuid)
raise errors.QueueDoesNotExist(queue, project)
def set_metadata(self, name, metadata, project=None):
if not self.exists(name, project):
raise errors.QueueDoesNotExist(name, project)
key = utils.scope_queue_name(name, project)
fields = {'m': self._packer(metadata)}
self._client.hmset(key, fields)
def get_metadata(self, name, project=None):
if not self.exists(name, project):
raise errors.QueueDoesNotExist(name, project)
queue_key = utils.scope_queue_name(name, project)
metadata = self._get_queue_info(queue_key, b'm', None)[0]
return self._unpacker(metadata)
def get(self, queue, message_id, project=None):
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
message = Message.from_redis(message_id, self._client)
now = timeutils.utcnow_ts()
if message and not utils.msg_expired_filter(message, now):
return message.to_basic(now)
else:
raise errors.MessageDoesNotExist(message_id, queue, project)
def get_metadata(self, name, project):
if project is None:
project = ''
try:
sel = sa.sql.select([tables.Queues.c.metadata],
sa.and_(tables.Queues.c.project == project,
tables.Queues.c.name == name))
return utils.json_decode(self.driver.get(sel)[0])
except utils.NoResult:
raise errors.QueueDoesNotExist(name, project)
def _list(self,
queue,
project=None,
marker=None,
limit=storage.DEFAULT_MESSAGES_PER_PAGE,
echo=False,
client_uuid=None,
include_claimed=False,
to_basic=True):
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
msgset_key = utils.msgset_key(queue, project)
client = self._client
if not marker and not include_claimed:
# NOTE(kgriffs): Skip unclaimed messages at the head
# of the queue; otherwise we would just filter them all
# out and likely end up with an empty list to return.
marker = self._find_first_unclaimed(queue, project, limit)
start = client.zrank(msgset_key, marker) or 0
else:
rank = client.zrank(msgset_key, marker)
start = rank + 1 if rank else 0
message_ids = client.zrange(msgset_key, start, start + (limit - 1))
messages = Message.from_redis_bulk(message_ids, client)
# NOTE(prashanthr_): Build a list of filters for checking
# the following:
#
# 1. Message is expired
# 2. Message is claimed
# 3. Message should not be echoed
#
now = timeutils.utcnow_ts()
filters = [functools.partial(utils.msg_expired_filter, now=now)]
if not include_claimed:
filters.append(functools.partial(utils.msg_claimed_filter,
now=now))
if not echo:
filters.append(
functools.partial(utils.msg_echo_filter,
client_uuid=client_uuid))
marker = {}
yield _filter_messages(messages, filters, to_basic, marker)
yield marker['next']
def set_metadata(self, name, metadata, project):
if project is None:
project = ''
update = (tables.Queues.update().where(
sa.and_(tables.Queues.c.project == project,
tables.Queues.c.name == name)).values(
metadata=utils.json_encode(metadata)))
res = self.driver.run(update)
try:
if res.rowcount != 1:
raise errors.QueueDoesNotExist(name, project)
finally:
res.close()
def bulk_get(self, queue, message_ids, project=None):
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
# NOTE(prashanthr_): Pipelining is used here purely
# for performance.
with self._client.pipeline() as pipe:
for mid in message_ids:
pipe.hgetall(mid)
messages = pipe.execute()
# NOTE(kgriffs): Skip messages that may have been deleted
now = timeutils.utcnow_ts()
return (Message.from_hmap(msg).to_basic(now) for msg in messages
if msg)
def delete(self, queue, message_id, project=None, claim=None):
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
# NOTE(kgriffs): The message does not exist, so
# it is essentially "already" deleted.
if not self._exists(message_id):
return
# TODO(kgriffs): Create decorator for validating claim and message
# IDs, since those are not checked at the transport layer. This
# decorator should be applied to all relevant methods.
if claim is not None:
try:
uuid.UUID(claim)
except ValueError:
raise errors.ClaimDoesNotExist(queue, project, claim)
msg_claim = self._get_claim(message_id)
is_claimed = (msg_claim is not None)
# Authorize the request based on having the correct claim ID
if claim is None:
if is_claimed:
raise errors.MessageIsClaimed(message_id)
elif not is_claimed:
raise errors.MessageNotClaimed(message_id)
elif msg_claim['id'] != claim:
if not self._claim_ctrl._exists(queue, claim, project):
raise errors.ClaimDoesNotExist(queue, project, claim)
raise errors.MessageNotClaimedBy(message_id, claim)
msgset_key = utils.msgset_key(queue, project)
with self._client.pipeline() as pipe:
pipe.delete(message_id)
pipe.zrem(msgset_key, message_id)
if is_claimed:
self._claim_ctrl._del_message(queue, project, msg_claim['id'],
message_id, pipe)
pipe.execute()
def bulk_delete(self, queue, message_ids, project=None):
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
msgset_key = utils.msgset_key(queue, project)
with self._client.pipeline() as pipe:
for mid in message_ids:
if not self._exists(mid):
continue
pipe.delete(mid)
pipe.zrem(msgset_key, mid)
msg_claim = self._get_claim(mid)
if msg_claim is not None:
self._claim_ctrl._del_message(queue, project,
msg_claim['id'], mid, pipe)
pipe.execute()
def first(self, queue, project=None, sort=1):
target = self._lookup(queue, project)
if target:
control = target.message_controller
return control.first(queue, project=project, sort=sort)
raise errors.QueueDoesNotExist(queue, project)
def stats(self, name, project=None):
target = self._lookup(name, project)
if target:
control = target.queue_controller
return control.stats(name, project=project)
raise errors.QueueDoesNotExist(name, project)
