def get(self, queue, claim_id, project=None):
if project is None:
project = ''
cid = utils.cid_decode(claim_id)
if cid is None:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
with self.driver.trans() as trans:
sel = sa.sql.select([
tables.Claims.c.id, tables.Claims.c.ttl,
tables.Claims.c.created
],
sa.and_(
tables.Claims.c.ttl > utils.get_age(
tables.Claims.c.created),
tables.Claims.c.id == cid,
tables.Queues.c.project == project,
tables.Queues.c.name == queue),
from_obj=[tables.Queues.join(tables.Claims)])
res = trans.execute(sel).fetchone()
if res is None:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
cid, ttl, created = res
return ({
'id': claim_id,
'ttl': ttl,
'age': (timeutils.utcnow() - created).seconds
}, list(self.__get(cid, trans)))
def update(self, queue, claim_id, metadata, project=None):
if project is None:
project = ''
cid = utils.cid_decode(claim_id)
if cid is None:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
age = utils.get_age(tables.Claims.c.created)
with self.driver.trans() as trans:
qid = utils.get_qid(self.driver, queue, project)
update = tables.Claims.update().where(
sa.and_(tables.Claims.c.ttl > age, tables.Claims.c.id == cid,
tables.Claims.c.id == qid))
update = update.values(ttl=metadata['ttl'])
res = trans.execute(update)
if res.rowcount != 1:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
update = (tables.Messages.update().values(
ttl=metadata['ttl']).where(
sa.and_(tables.Messages.c.ttl < metadata['ttl'],
tables.Messages.c.cid == cid)))
trans.execute(update)
def update(self, queue, claim_id, metadata, project=None):
cid = utils.to_oid(claim_id)
if cid is None:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
now = timeutils.utcnow_ts()
ttl = int(metadata.get('ttl', 60))
expires = now + ttl
msg_ctrl = self.driver.message_controller
claimed = msg_ctrl._claimed(queue,
cid,
expires=now,
limit=1,
project=project)
try:
next(claimed)
except StopIteration:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
meta = {
'id': cid,
't': ttl,
'e': expires,
}
# TODO(kgriffs): Create methods for these so we don't interact
# with the messages collection directly (loose coupling)
scope = utils.scope_queue_name(queue, project)
collection = msg_ctrl._collection(queue, project)
collection.update({
'p_q': scope,
'c.id': cid
}, {'$set': {
'c': meta
}},
upsert=False,
multi=True)
# NOTE(flaper87): Dirty hack!
# This sets the expiration time to
# `expires` on messages that would
# expire before claim.
collection.update({
'p_q': scope,
'e': {
'$lt': expires
},
'c.id': cid
}, {'$set': {
'e': expires,
't': ttl
}},
upsert=False,
multi=True)
def update(self, queue, claim_id, metadata, project=None):
target = self._lookup(queue, project)
if target:
control = target.claim_controller
return control.update(queue, claim_id=claim_id,
project=project, metadata=metadata)
raise errors.ClaimDoesNotExist(claim_id, queue, project)
def get(self, queue, claim_id, project=None):
target = self._lookup(queue, project)
if target:
control = target.claim_controller
return control.get(queue, claim_id=claim_id,
project=project)
raise errors.ClaimDoesNotExist(claim_id, queue, project)
def get(self, queue, claim_id, project=None):
if not self._exists(queue, claim_id, project):
raise errors.ClaimDoesNotExist(queue, project, claim_id)
claim_msgs_key = utils.scope_claim_messages(claim_id,
CLAIM_MESSAGES_SUFFIX)
# basic_messages
msg_keys = self._get_claimed_message_keys(claim_msgs_key)
claimed_msgs = messages.Message.from_redis_bulk(msg_keys, self._client)
now = timeutils.utcnow_ts()
basic_messages = [msg.to_basic(now) for msg in claimed_msgs if msg]
# claim_meta
now = timeutils.utcnow_ts()
expires, ttl = self._get_claim_info(claim_id, [b'e', b't'])
update_time = expires - ttl
age = now - update_time
claim_meta = {
'age': age,
'ttl': ttl,
'id': claim_id,
}
return claim_meta, basic_messages
def delete(self, queue, message_id, project=None, claim=None):
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
# NOTE(kgriffs): The message does not exist, so
# it is essentially "already" deleted.
if not self._exists(message_id):
return
# TODO(kgriffs): Create decorator for validating claim and message
# IDs, since those are not checked at the transport layer. This
# decorator should be applied to all relevant methods.
if claim is not None:
try:
uuid.UUID(claim)
except ValueError:
raise errors.ClaimDoesNotExist(queue, project, claim)
msg_claim = self._get_claim(message_id)
is_claimed = (msg_claim is not None)
# Authorize the request based on having the correct claim ID
if claim is None:
if is_claimed:
raise errors.MessageIsClaimed(message_id)
elif not is_claimed:
raise errors.MessageNotClaimed(message_id)
elif msg_claim['id'] != claim:
if not self._claim_ctrl._exists(queue, claim, project):
raise errors.ClaimDoesNotExist(queue, project, claim)
raise errors.MessageNotClaimedBy(message_id, claim)
msgset_key = utils.msgset_key(queue, project)
with self._client.pipeline() as pipe:
pipe.delete(message_id)
pipe.zrem(msgset_key, message_id)
if is_claimed:
self._claim_ctrl._del_message(queue, project, msg_claim['id'],
message_id, pipe)
pipe.execute()
def update(self, queue, claim_id, metadata, project=None):
if not self._exists(queue, claim_id, project):
raise errors.ClaimDoesNotExist(claim_id, queue, project)
now = timeutils.utcnow_ts()
claim_ttl = int(metadata.get('ttl', 60))
claim_expires = now + claim_ttl
grace = int(metadata.get('grace', 60))
msg_ttl = claim_ttl + grace
msg_expires = claim_expires + grace
claim_msgs_key = utils.scope_claim_messages(claim_id,
CLAIM_MESSAGES_SUFFIX)
msg_keys = self._get_claimed_message_keys(claim_msgs_key)
claimed_msgs = messages.MessageEnvelope.from_redis_bulk(
msg_keys, self._client)
claim_info = {
't': claim_ttl,
'e': claim_expires,
}
with self._client.pipeline() as pipe:
for msg in claimed_msgs:
if msg:
msg.claim_id = claim_id
msg.claim_expires = claim_expires
if _msg_would_expire(msg, msg_expires):
msg.ttl = msg_ttl
msg.expires = msg_expires
# TODO(kgriffs): Rather than writing back the
# entire message, only set the fields that
# have changed.
#
# When this change is made, don't forget to
# also call pipe.expire with the new TTL value.
msg.to_redis(pipe)
# Update the claim id and claim expiration info
# for all the messages.
pipe.hmset(claim_id, claim_info)
pipe.expire(claim_id, claim_ttl)
pipe.expire(claim_msgs_key, claim_ttl)
claims_set_key = utils.scope_claims_set(queue, project,
QUEUE_CLAIMS_SUFFIX)
pipe.zadd(claims_set_key, claim_expires, claim_id)
pipe.execute()
def get(self, queue, claim_id, project=None):
msg_ctrl = self.driver.message_controller
# Base query, always check expire time
now = timeutils.utcnow_ts()
cid = utils.to_oid(claim_id)
if cid is None:
raise errors.ClaimDoesNotExist(queue, project, claim_id)
def messages(msg_iter):
msg = next(msg_iter)
yield msg.pop('claim')
yield msg
# Smoke it!
for msg in msg_iter:
del msg['claim']
yield msg
try:
# Lets get claim's data
# from the first message
# in the iterator
msgs = messages(msg_ctrl._claimed(queue, cid, now,
project=project))
claim = next(msgs)
update_time = claim['e'] - claim['t']
age = now - update_time
claim_meta = {
'age': int(age),
'ttl': claim['t'],
'id': str(claim['id']),
}
except StopIteration:
raise errors.ClaimDoesNotExist(cid, queue, project)
return (claim_meta, msgs)
def delete(self, queue, message_id, project, claim=None):
if project is None:
project = ''
mid = utils.msgid_decode(message_id)
if mid is None:
return
with self.driver.trans() as trans:
if not self._exists(queue, message_id, project):
return
statement = tables.Messages.delete()
and_stmt = [tables.Messages.c.id == mid]
exists = sa.sql.select([tables.Messages.c.id], sa.and_(*and_stmt))
if not trans.execute(exists).first():
return
cid = claim and utils.cid_decode(claim) or None
if claim and cid is None:
raise errors.ClaimDoesNotExist(queue, project, claim)
and_stmt.append(tables.Messages.c.cid == cid)
statement = statement.where(sa.and_(*and_stmt))
res = trans.execute(statement)
if res.rowcount == 0:
# NOTE(kgriffs): Either the message is not claimed,
# or if it is, the specified claim does not exist.
cid = self._get_cid(mid)
if cid is None:
raise errors.MessageNotClaimed(mid)
# NOTE(kgriffs): The message exists, but the claim
# must have expired or something, since it
# was not associated with the message.
raise errors.MessageNotClaimedBy(mid, claim)