def get(self, queue, claim_id, project=None):
msg_ctrl = self.driver.message_controller
# Base query, always check expire time
now = timeutils.utcnow_ts()
cid = utils.to_oid(claim_id)
if cid is None:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
try:
# Lets get claim's data
# from the first message
# in the iterator
msgs = _messages_iter(
msg_ctrl._claimed(queue, cid, now, project=project))
claim = next(msgs)
update_time = claim['e'] - claim['t']
age = now - update_time
claim_meta = {
'age': int(age),
'ttl': claim['t'],
'id': str(claim['id']),
}
except StopIteration:
raise errors.ClaimDoesNotExist(cid, queue, project)
return claim_meta, msgs
def get(self, queue, claim_id, project=None):
if project is None:
project = ''
cid = utils.cid_decode(claim_id)
if cid is None:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
with self.driver.trans() as trans:
sel = sa.sql.select([tables.Claims.c.id,
tables.Claims.c.ttl,
tables.Claims.c.created],
sa.and_(tables.Claims.c.ttl >
utils.get_age(tables.Claims.c.created),
tables.Claims.c.id == cid,
tables.Queues.c.project == project,
tables.Queues.c.name == queue),
from_obj=[tables.Queues.join(tables.Claims)])
res = trans.execute(sel).fetchone()
if res is None:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
cid, ttl, created = res
return (
{'id': claim_id,
'ttl': ttl,
'age': (timeutils.utcnow() - created).seconds},
list(self.__get(cid, trans))
)
def update(self, queue, claim_id, metadata, project=None):
cid = utils.to_oid(claim_id)
if cid is None:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
now = timeutils.utcnow_ts()
grace = metadata['grace']
ttl = metadata['ttl']
claim_expires = now + ttl
claim_expires_dt = datetime.datetime.utcfromtimestamp(claim_expires)
message_ttl = ttl + grace
message_expires = datetime.datetime.utcfromtimestamp(claim_expires +
grace)
msg_ctrl = self.driver.message_controller
claimed = msg_ctrl._claimed(queue,
cid,
expires=now,
limit=1,
project=project)
try:
next(claimed)
except StopIteration:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
meta = {
'id': cid,
't': ttl,
'e': claim_expires,
}
# TODO(kgriffs): Create methods for these so we don't interact
# with the messages collection directly (loose coupling)
scope = utils.scope_queue_name(queue, project)
collection = msg_ctrl._collection(queue, project)
collection.update_many({
'p_q': scope,
'c.id': cid
}, {'$set': {
'c': meta
}},
upsert=False)
# NOTE(flaper87): Dirty hack!
# This sets the expiration time to
# `expires` on messages that would
# expire before claim.
collection.update_many(
{
'p_q': scope,
'e': {
'$lt': claim_expires_dt
},
'c.id': cid
}, {'$set': {
'e': message_expires,
't': message_ttl
}},
upsert=False)
def update(self, queue, claim_id, metadata, project=None):
if project is None:
project = ''
cid = utils.cid_decode(claim_id)
if cid is None:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
age = utils.get_age(tables.Claims.c.created)
with self.driver.trans() as trans:
qid = utils.get_qid(self.driver, queue, project)
update = tables.Claims.update().where(sa.and_(
tables.Claims.c.ttl > age,
tables.Claims.c.id == cid,
tables.Claims.c.id == qid))
update = update.values(ttl=metadata['ttl'])
res = trans.execute(update)
if res.rowcount != 1:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
update = (tables.Messages.update().
values(ttl=metadata['ttl'] + metadata['grace']).
where(sa.and_(
tables.Messages.c.ttl < metadata['ttl'],
tables.Messages.c.cid == cid)))
trans.execute(update)
def get(self, queue, claim_id, project=None):
if not self._exists(queue, claim_id, project):
raise errors.ClaimDoesNotExist(queue, project, claim_id)
claim_msgs_key = utils.scope_claim_messages(claim_id,
CLAIM_MESSAGES_SUFFIX)
# basic_messages
msg_keys = self._get_claimed_message_keys(claim_msgs_key)
claimed_msgs = messages.Message.from_redis_bulk(msg_keys,
self._client)
now = timeutils.utcnow_ts()
basic_messages = [msg.to_basic(now)
for msg in claimed_msgs if msg]
# claim_meta
now = timeutils.utcnow_ts()
expires, ttl = self._get_claim_info(claim_id, [b'e', b't'])
update_time = expires - ttl
age = now - update_time
claim_meta = {
'age': age,
'ttl': ttl,
'id': claim_id,
}
return claim_meta, basic_messages
def delete(self, queue, message_id, project=None, claim=None):
claim_ctrl = self.driver.claim_controller
if not self._queue_ctrl.exists(queue, project):
return
# NOTE(kgriffs): The message does not exist, so
# it is essentially "already" deleted.
if not self._exists(message_id):
return
# TODO(kgriffs): Create decorator for validating claim and message
# IDs, since those are not checked at the transport layer. This
# decorator should be applied to all relevant methods.
if claim is not None:
try:
uuid.UUID(claim)
except ValueError:
raise errors.ClaimDoesNotExist(claim, queue, project)
msg_claim = self._get_claim(message_id)
is_claimed = (msg_claim is not None)
# Authorize the request based on having the correct claim ID
if claim is None:
if is_claimed:
raise errors.MessageIsClaimed(message_id)
elif not is_claimed:
raise errors.MessageNotClaimed(message_id)
elif msg_claim['id'] != claim:
if not claim_ctrl._exists(queue, claim, project):
raise errors.ClaimDoesNotExist(claim, queue, project)
raise errors.MessageNotClaimedBy(message_id, claim)
msgset_key = utils.msgset_key(queue, project)
with self._client.pipeline() as pipe:
pipe.delete(message_id)
pipe.zrem(msgset_key, message_id)
if is_claimed:
claim_ctrl._del_message(queue, project, msg_claim['id'],
message_id, pipe)
pipe.execute()
def update(self, queue, claim_id, metadata, project=None):
control = self._get_controller(queue, project)
if control:
return control.update(queue,
claim_id=claim_id,
project=project,
metadata=metadata)
raise errors.ClaimDoesNotExist(claim_id, queue, project)
def _exists(self, queue, claim_id, project=None):
try:
return self._client.head_object(
utils._claim_container(queue, project), claim_id)
except swiftclient.ClientException as exc:
if exc.http_status == 404:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
raise
def update(self, queue, claim_id, metadata, project=None):
if not self._exists(queue, claim_id, project):
raise errors.ClaimDoesNotExist(claim_id, queue, project)
now = timeutils.utcnow_ts()
claim_ttl = metadata['ttl']
claim_expires = now + claim_ttl
grace = metadata['grace']
msg_ttl = claim_ttl + grace
msg_expires = claim_expires + grace
claim_msgs_key = utils.scope_claim_messages(claim_id,
CLAIM_MESSAGES_SUFFIX)
msg_keys = self._get_claimed_message_keys(claim_msgs_key)
claimed_msgs = messages.MessageEnvelope.from_redis_bulk(msg_keys,
self._client)
claim_info = {
't': claim_ttl,
'e': claim_expires,
}
with self._client.pipeline() as pipe:
for msg in claimed_msgs:
if msg:
msg.claim_id = claim_id
msg.claim_expires = claim_expires
if _msg_would_expire(msg, claim_expires):
msg.ttl = msg_ttl
msg.expires = msg_expires
# TODO(kgriffs): Rather than writing back the
# entire message, only set the fields that
# have changed.
#
# When this change is made, don't forget to
# also call pipe.expire with the new TTL value.
msg.to_redis(pipe)
# Update the claim id and claim expiration info
# for all the messages.
pipe.hmset(claim_id, claim_info)
pipe.expire(claim_id, claim_ttl)
pipe.expire(claim_msgs_key, claim_ttl)
claims_set_key = utils.scope_claims_set(queue, project,
QUEUE_CLAIMS_SUFFIX)
pipe.zadd(claims_set_key, claim_expires, claim_id)
pipe.execute()
def update(self, queue, claim_id, metadata, project=None):
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
container = utils._claim_container(queue, project)
try:
headers, obj = self._client.get_object(container, claim_id)
except swiftclient.ClientException as exc:
if exc.http_status == 404:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
raise
self._client.put_object(container,
claim_id,
obj,
content_type='application/json',
headers={'x-delete-after': metadata['ttl']})
def delete(self, queue, message_id, project, claim=None):
if project is None:
project = ''
mid = utils.msgid_decode(message_id)
if mid is None:
return
with self.driver.trans() as trans:
if not self._exists(queue, message_id, project):
return
statement = tables.Messages.delete()
and_stmt = [tables.Messages.c.id == mid]
exists = sa.sql.select([tables.Messages.c.id], sa.and_(*and_stmt))
if not trans.execute(exists).first():
return
cid = claim and utils.cid_decode(claim) or None
if claim and cid is None:
raise errors.ClaimDoesNotExist(queue, project, claim)
and_stmt.append(tables.Messages.c.cid == cid)
statement = statement.where(sa.and_(*and_stmt))
res = trans.execute(statement)
if res.rowcount == 0:
# NOTE(kgriffs): Either the message is not claimed,
# or if it is, the specified claim does not exist.
cid = self._get_cid(mid)
if cid is None:
raise errors.MessageNotClaimed(mid)
# NOTE(kgriffs): The message exists, but the claim
# must have expired or something, since it
# was not associated with the message.
raise errors.MessageNotClaimedBy(mid, claim)
def get(self, queue, claim_id, project=None):
control = self._get_controller(queue, project)
if control:
return control.get(queue, claim_id=claim_id,
project=project)
raise errors.ClaimDoesNotExist(claim_id, queue, project)