def _list(self, queue, project=None, marker=None,
limit=storage.DEFAULT_MESSAGES_PER_PAGE,
echo=False, client_uuid=None,
include_claimed=False, sort=1):
"""List messages in the queue, oldest first(ish)
Time ordering and message inclusion in lists are soft, there is no
global order and times are based on the UTC time of the zaqar-api
server that the message was created from.
Here be consistency dragons.
"""
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
client = self._client
container = utils._message_container(queue, project)
query_string = None
if sort == -1:
query_string = 'reverse=on'
try:
_, objects = client.get_container(
container,
marker=marker,
# list 2x the objects because some listing items may have
# expired
limit=limit * 2,
query_string=query_string)
except swiftclient.ClientException as exc:
if exc.http_status == 404:
raise errors.QueueDoesNotExist(queue, project)
raise
def is_claimed(msg, headers):
if include_claimed or msg['claim_id'] is None:
return False
claim_obj = self.driver.claim_controller._get(
queue, msg['claim_id'], project)
return claim_obj is not None and claim_obj['ttl'] > 0
def is_echo(msg, headers):
if echo:
return False
return headers['x-object-meta-clientid'] == str(client_uuid)
filters = [
is_echo,
is_claimed,
]
marker = {}
get_object = functools.partial(client.get_object, container)
list_objects = functools.partial(client.get_container, container,
limit=limit * 2,
query_string=query_string)
yield utils._filter_messages(objects, filters, marker, get_object,
list_objects, limit=limit)
yield marker and marker['next']
def set_metadata(self, name, metadata, project=None):
control = self._get_controller(name, project)
if control:
return control.set_metadata(name,
metadata=metadata,
project=project)
raise errors.QueueDoesNotExist(name, project)
def post(self, queue, messages, client_uuid, project=None):
control = self._get_controller(queue, project)
if control:
return control.post(queue, project=project,
messages=messages,
client_uuid=client_uuid)
raise errors.QueueDoesNotExist(queue, project)
def stats(self, name, project=None):
if not self._queue_ctrl.exists(name, project=project):
raise errors.QueueDoesNotExist(name, project)
total = self._message_ctrl._count(name, project)
if total:
claimed = self._claim_ctrl._count_messages(name, project)
else:
claimed = 0
message_stats = {
'claimed': claimed,
'free': total - claimed,
'total': total,
}
if total:
try:
newest = self._message_ctrl.first(name, project, -1)
oldest = self._message_ctrl.first(name, project, 1)
except errors.QueueIsEmpty:
pass
else:
message_stats['newest'] = newest
message_stats['oldest'] = oldest
return {'messages': message_stats}
def post(self, queue, messages, client_uuid, project=None):
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
msgset_key = utils.msgset_key(queue, project)
counter_key = utils.scope_queue_index(queue, project,
MESSAGE_RANK_COUNTER_SUFFIX)
message_ids = []
now = timeutils.utcnow_ts()
with self._client.pipeline() as pipe:
for msg in messages:
prepared_msg = Message(
ttl=msg['ttl'],
created=now,
client_uuid=client_uuid,
claim_id=None,
claim_expires=now,
body=msg.get('body', {}),
)
prepared_msg.to_redis(pipe)
message_ids.append(prepared_msg.id)
pipe.execute()
# NOTE(kgriffs): If this call fails, we will return
# an error to the client and the messages will be
# orphaned, but Redis will remove them when they
# expire, so we will just pretend they don't exist
# in that case.
self._index_messages(msgset_key, counter_key, message_ids)
return message_ids
def create(self, queue, subscriber, ttl, options, project=None):
subscription_id = str(uuid.uuid4())
subset_key = utils.scope_subscription_ids_set(queue, project,
SUBSCRIPTION_IDS_SUFFIX)
source = queue
now = timeutils.utcnow_ts()
ttl = int(ttl)
expires = now + ttl
subscription = {
'id': subscription_id,
's': source,
'u': subscriber,
't': ttl,
'e': expires,
'o': options,
'p': project
}
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
try:
# Pipeline ensures atomic inserts.
with self._client.pipeline() as pipe:
pipe.zadd(subset_key, 1,
subscription_id).hmset(subscription_id, subscription)
pipe.execute()
return subscription_id
except redis.exceptions.ResponseError:
return None
def get_qid(driver, queue, project):
sel = sa.sql.select([tables.Queues.c.id],
sa.and_(tables.Queues.c.project == project,
tables.Queues.c.name == queue))
try:
return driver.get(sel)[0]
except NoResult:
raise errors.QueueDoesNotExist(queue, project)
def _get(self, queue, message_id, project=None, check_queue=True):
if check_queue and not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
now = timeutils.utcnow_ts(True)
headers, msg = self._find_message(queue, message_id, project)
return utils._message_to_json(message_id, msg, headers, now)
def set_metadata(self, name, metadata, project=None):
if not self.exists(name, project):
raise errors.QueueDoesNotExist(name, project)
key = utils.scope_queue_name(name, project)
fields = {'m': self._packer(metadata)}
self._client.hmset(key, fields)
def get_metadata(self, name, project=None):
if not self.exists(name, project):
raise errors.QueueDoesNotExist(name, project)
queue_key = utils.scope_queue_name(name, project)
metadata = self._get_queue_info(queue_key, b'm', None)[0]
return self._unpacker(metadata)
def _list(self,
queue,
project=None,
marker=None,
limit=storage.DEFAULT_MESSAGES_PER_PAGE,
echo=False,
client_uuid=None,
include_claimed=False,
include_delayed=False,
to_basic=True):
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
msgset_key = utils.msgset_key(queue, project)
client = self._client
if not marker and not include_claimed:
# NOTE(kgriffs): Skip claimed messages at the head
# of the queue; otherwise we would just filter them all
# out and likely end up with an empty list to return.
marker = self._find_first_unclaimed(queue, project, limit)
start = client.zrank(msgset_key, marker) or 0
else:
rank = client.zrank(msgset_key, marker)
start = rank + 1 if rank else 0
message_ids = client.zrange(msgset_key, start, start + (limit - 1))
messages = Message.from_redis_bulk(message_ids, client)
# NOTE(prashanthr_): Build a list of filters for checking
# the following:
#
# 1. Message is expired
# 2. Message is claimed
# 3. Message should not be echoed
#
now = timeutils.utcnow_ts()
filters = [functools.partial(utils.msg_expired_filter, now=now)]
if not include_claimed:
filters.append(functools.partial(utils.msg_claimed_filter,
now=now))
if not include_delayed:
filters.append(functools.partial(utils.msg_delayed_filter,
now=now))
if not echo:
filters.append(
functools.partial(utils.msg_echo_filter,
client_uuid=client_uuid))
marker = {}
yield _filter_messages(messages, filters, to_basic, marker)
yield marker['next']
def get(self, queue, message_id, project=None):
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
message = Message.from_redis(message_id, self._client)
now = timeutils.utcnow_ts()
if message and not utils.msg_expired_filter(message, now):
return message.to_basic(now)
else:
raise errors.MessageDoesNotExist(message_id, queue, project)
def get_metadata(self, name, project):
if project is None:
project = ''
try:
sel = sa.sql.select([tables.Queues.c.metadata],
sa.and_(tables.Queues.c.project == project,
tables.Queues.c.name == name))
return utils.json_decode(self.driver.get(sel)[0])
except utils.NoResult:
raise errors.QueueDoesNotExist(name, project)
def get_metadata(self, name, project):
if project is None:
project = ''
sel = sa.sql.select([tables.Queues.c.metadata],
sa.and_(tables.Queues.c.project == project,
tables.Queues.c.name == name))
queue = self.driver.run(sel).fetchone()
if queue is None:
raise errors.QueueDoesNotExist(name, project)
return utils.json_decode(queue[0])
def get(self, queue, subscription_id, project=None):
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
subscription = SubscripitonEnvelope.from_redis(subscription_id,
self._client)
now = timeutils.utcnow_ts()
if subscription and not utils.subscription_expired_filter(
subscription, now):
return subscription.to_basic(now)
else:
raise errors.SubscriptionDoesNotExist(subscription_id)
def stats(self, name, project=None):
if not self._queue_ctrl.exists(name, project=project):
raise errors.QueueDoesNotExist(name, project)
total = 0
claimed = 0
container = utils._message_container(name, project)
try:
_, objects = self._client.get_container(container)
except swiftclient.ClientException as exc:
if exc.http_status == 404:
raise errors.QueueIsEmpty(name, project)
newest = None
oldest = None
now = timeutils.utcnow_ts(True)
for obj in objects:
try:
headers = self._client.head_object(container, obj['name'])
except swiftclient.ClientException as exc:
if exc.http_status != 404:
raise
else:
created = float(headers['x-timestamp'])
created_iso = datetime.datetime.utcfromtimestamp(
created).strftime('%Y-%m-%dT%H:%M:%SZ')
newest = {
'id': obj['name'],
'age': now - created,
'created': created_iso
}
if oldest is None:
oldest = copy.deepcopy(newest)
total += 1
if headers.get('x-object-meta-claimid'):
claimed += 1
msg_stats = {
'claimed': claimed,
'free': total - claimed,
'total': total,
}
if newest is not None:
msg_stats['newest'] = newest
msg_stats['oldest'] = oldest
return {'messages': msg_stats}
def set_metadata(self, name, metadata, project):
if project is None:
project = ''
update = (tables.Queues.update().where(
sa.and_(tables.Queues.c.project == project,
tables.Queues.c.name == name)).values(
metadata=utils.json_encode(metadata)))
res = self.driver.run(update)
try:
if res.rowcount != 1:
raise errors.QueueDoesNotExist(name, project)
finally:
res.close()
def bulk_get(self, queue, message_ids, project=None):
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
# NOTE(prashanthr_): Pipelining is used here purely
# for performance.
with self._client.pipeline() as pipe:
for mid in message_ids:
pipe.hgetall(mid)
messages = pipe.execute()
# NOTE(kgriffs): Skip messages that may have been deleted
now = timeutils.utcnow_ts()
return (Message.from_hmap(msg).to_basic(now)
for msg in messages if msg)
def delete(self, queue, message_id, project=None, claim=None):
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
# NOTE(kgriffs): The message does not exist, so
# it is essentially "already" deleted.
if not self._exists(message_id):
return
# TODO(kgriffs): Create decorator for validating claim and message
# IDs, since those are not checked at the transport layer. This
# decorator should be applied to all relevant methods.
if claim is not None:
try:
uuid.UUID(claim)
except ValueError:
raise errors.ClaimDoesNotExist(queue, project, claim)
msg_claim = self._get_claim(message_id)
is_claimed = (msg_claim is not None)
# Authorize the request based on having the correct claim ID
if claim is None:
if is_claimed:
raise errors.MessageIsClaimed(message_id)
elif not is_claimed:
raise errors.MessageNotClaimed(message_id)
elif msg_claim['id'] != claim:
if not self._claim_ctrl._exists(queue, claim, project):
raise errors.ClaimDoesNotExist(queue, project, claim)
raise errors.MessageNotClaimedBy(message_id, claim)
msgset_key = utils.msgset_key(queue, project)
with self._client.pipeline() as pipe:
pipe.delete(message_id)
pipe.zrem(msgset_key, message_id)
if is_claimed:
self._claim_ctrl._del_message(queue, project,
msg_claim['id'], message_id,
pipe)
pipe.execute()
def update(self, queue, claim_id, metadata, project=None):
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
container = utils._claim_container(queue, project)
try:
headers, obj = self._client.get_object(container, claim_id)
except swiftclient.ClientException as exc:
if exc.http_status == 404:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
raise
self._client.put_object(container,
claim_id,
obj,
content_type='application/json',
headers={'x-delete-after': metadata['ttl']})
def bulk_delete(self, queue, message_ids, project=None):
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
msgset_key = utils.msgset_key(queue, project)
with self._client.pipeline() as pipe:
for mid in message_ids:
if not self._exists(mid):
continue
pipe.delete(mid)
pipe.zrem(msgset_key, mid)
msg_claim = self._get_claim(mid)
if msg_claim is not None:
self._claim_ctrl._del_message(queue, project,
msg_claim['id'], mid, pipe)
pipe.execute()
def _stats(self, name, project=None):
control = self._get_controller(name, project)
if control:
return control.stats(name, project=project)
raise errors.QueueDoesNotExist(name, project)
def get(self, queue, message_id, project=None):
control = self._get_controller(queue, project)
if control:
return control.get(queue, message_id=message_id,
project=project)
raise errors.QueueDoesNotExist(queue, project)
def _stats(self, name, project=None):
mqHandler = self._get_controller(name, project)
if mqHandler:
return mqHandler.stats(name, project=project)
raise errors.QueueDoesNotExist(name, project)
def first(self, queue, project=None, sort=1):
control = self._get_controller(queue, project)
if control:
return control.first(queue, project=project, sort=sort)
raise errors.QueueDoesNotExist(queue, project)