def delete(self, queue_name, message_id, project=None, claim=None):
# NOTE(cpp-cabrera): return early - this is an invalid message
# id so we won't be able to find it any way
mid = utils.to_oid(message_id)
if mid is None:
return
query = {"q": queue_name, "p": project, "_id": mid}
# NOTE(cpp-cabrera): return early - the user gaves us an
# invalid claim id and that renders the rest of this
# request moot
cid = utils.to_oid(claim)
if cid is None:
return
now = timeutils.utcnow()
query["e"] = {"$gt": now}
message = self._col.find_one(query)
if message is None:
return
is_claimed = message["c"]["id"] is not None and message["c"]["e"] > now
if claim is None:
if is_claimed:
raise exceptions.MessageIsClaimed(message_id)
else:
if message["c"]["id"] != cid:
raise exceptions.MessageIsClaimedBy(message_id, claim)
self._col.remove(query["_id"], w=0)
def delete(self, queue_name, message_id, project=None, claim=None):
# NOTE(cpp-cabrera): return early - this is an invalid message
# id so we won't be able to find it any way
mid = utils.to_oid(message_id)
if mid is None:
return
query = {'q': queue_name, 'p': project, '_id': mid}
# NOTE(cpp-cabrera): return early - the user gaves us an
# invalid claim id and that renders the rest of this
# request moot
cid = utils.to_oid(claim)
if cid is None:
return
now = timeutils.utcnow()
query['e'] = {'$gt': now}
message = self._col.find_one(query)
if message is None:
return
is_claimed = (message['c']['id'] is not None
and message['c']['e'] > now)
if claim is None:
if is_claimed:
raise exceptions.MessageIsClaimed(message_id)
else:
if message['c']['id'] != cid:
raise exceptions.MessageIsClaimedBy(message_id, claim)
self._col.remove(query['_id'], w=0)
def delete(self, queue, message_id, project=None, claim=None):
try:
mid = utils.to_oid(message_id)
query = {
'q': self._get_queue_id(queue, project),
'_id': mid
}
if claim:
now = timeutils.utcnow()
query['e'] = {'$gt': now}
message = self._col.find_one(query)
if message is None:
return
cid = utils.to_oid(claim)
if not ('c' in message and
message['c']['id'] == cid and
message['c']['e'] > now):
raise exceptions.ClaimNotPermitted(message_id, claim)
self._col.remove(query['_id'], w=0)
else:
self._col.remove(query, w=0)
except exceptions.QueueDoesNotExist:
pass
def get(self, queue_name, message_id, project=None):
"""Gets a single message by ID.
:raises: exceptions.MessageDoesNotExist
"""
mid = utils.to_oid(message_id)
if mid is None:
raise exceptions.MessageDoesNotExist(message_id, queue_name,
project)
now = timeutils.utcnow()
query = {
'_id': mid,
'q': queue_name,
'p': project,
'e': {'$gt': now}
}
message = list(self._col.find(query).limit(1).hint([('_id', 1)]))
if not message:
raise exceptions.MessageDoesNotExist(message_id, queue_name,
project)
return _basic_message(message[0], now)
def claimed(self, queue, claim_id=None, expires=None, limit=None):
query = {
"c.id": claim_id,
"q": utils.to_oid(queue),
}
if not claim_id:
# lookup over c.id to use the index
query["c.id"] = {"$ne": None}
if expires:
query["c.e"] = {"$gt": expires}
msgs = self._col.find(query, sort=[("_id", 1)])
if limit:
msgs = msgs.limit(limit)
now = timeutils.utcnow_ts()
def denormalizer(msg):
oid = msg.get("_id")
age = now - utils.oid_ts(oid)
return {
"id": str(oid),
"age": age,
"ttl": msg["t"],
"body": msg["b"],
"claim": msg["c"]
}
return utils.HookedCursor(msgs, denormalizer)
def get(self, queue_name, message_id, project=None):
"""Gets a single message by ID.
:raises: exceptions.MessageDoesNotExist
"""
mid = utils.to_oid(message_id)
if mid is None:
raise exceptions.MessageDoesNotExist(message_id, queue_name,
project)
now = timeutils.utcnow()
query = {
'_id': mid,
'q': queue_name,
'p': project,
'e': {'$gt': now}
}
message = list(self._col.find(query).limit(1).hint([('_id', 1)]))
if not message:
raise exceptions.MessageDoesNotExist(message_id, queue_name,
project)
return _basic_message(message[0], now)
def claimed(self, queue_id, claim_id=None, expires=None, limit=None):
query = {
'c.id': claim_id,
'c.e': {'$gt': expires or timeutils.utcnow()},
'q': utils.to_oid(queue_id),
}
if not claim_id:
# lookup over c.id to use the index
query['c.id'] = {'$ne': None}
msgs = self._col.find(query, sort=[('_id', 1)])
if limit:
msgs = msgs.limit(limit)
now = timeutils.utcnow()
def denormalizer(msg):
oid = msg['_id']
age = now - utils.oid_utc(oid)
return {
'id': str(oid),
'age': age.seconds,
'ttl': msg['t'],
'body': msg['b'],
'claim': msg['c']
}
return utils.HookedCursor(msgs, denormalizer)
def get(self, queue, message_ids, project=None):
if not isinstance(message_ids, list):
message_ids = [message_ids]
message_ids = [utils.to_oid(id) for id in message_ids]
now = timeutils.utcnow()
# Base query, always check expire time
query = {
'q': self._get_queue_id(queue, project),
'e': {'$gt': now},
'_id': {'$in': message_ids},
}
messages = self._col.find(query)
def denormalizer(msg):
oid = msg['_id']
age = now - utils.oid_utc(oid)
return {
'id': str(oid),
'age': age.seconds,
'ttl': msg['t'],
'body': msg['b'],
}
return utils.HookedCursor(messages, denormalizer)
def unclaim(self, queue_name, claim_id, project=None):
try:
cid = utils.to_oid(claim_id)
except ValueError:
return
self._col.update({'q': queue_name, 'p': project, 'c.id': cid},
{'$set': {'c': {'id': None, 'e': 0}}},
upsert=False, multi=True)
def update(self, queue, claim_id, metadata, project=None):
cid = utils.to_oid(claim_id)
if cid is None:
raise exceptions.ClaimDoesNotExist(claim_id, queue, project)
now = timeutils.utcnow()
ttl = int(metadata.get('ttl', 60))
ttl_delta = datetime.timedelta(seconds=ttl)
expires = now + ttl_delta
msg_ctrl = self.driver.message_controller
claimed = msg_ctrl.claimed(queue,
cid,
expires=now,
limit=1,
project=project)
try:
next(claimed)
except StopIteration:
raise exceptions.ClaimDoesNotExist(claim_id, queue, project)
meta = {
'id': cid,
't': ttl,
'e': expires,
}
msg_ctrl._col.update({
'q': queue,
'p': project,
'c.id': cid
}, {'$set': {
'c': meta
}},
upsert=False,
multi=True)
# NOTE(flaper87): Dirty hack!
# This sets the expiration time to
# `expires` on messages that would
# expire before claim.
msg_ctrl._col.update(
{
'q': queue,
'p': project,
'e': {
'$lt': expires
},
'c.id': cid
}, {'$set': {
'e': expires,
't': ttl
}},
upsert=False,
multi=True)
def unclaim(self, claim_id):
try:
cid = utils.to_oid(claim_id)
except ValueError:
return
self._col.update({'c.id': cid},
{'$set': {'c': {'id': None, 'e': 0}}},
upsert=False, multi=True)
def active(self, queue_id, marker=None, echo=False,
client_uuid=None, fields=None):
# NOTE(kgriffs): Since this is a public method, queue_id
# might not be an ObjectID. Usually it will be, since active()
# is a utility method, so short-circuit for performance.
if not isinstance(queue_id, objectid.ObjectId):
queue_id = utils.to_oid(queue_id)
return self._list(queue_id, marker, echo, client_uuid, fields,
include_claimed=False)
def bulk_delete(self, queue_name, message_ids, project=None):
try:
message_ids = [utils.to_oid(id) for id in message_ids]
query = {
'q': queue_name,
'p': project,
'_id': {'$in': message_ids},
}
self._col.remove(query, w=0)
except exceptions.QueueDoesNotExist:
pass
def active(self, queue, marker=None, echo=False,
client_uuid=None, fields=None):
now = timeutils.utcnow_ts()
query = {
# Messages must belong to this queue
"q": utils.to_oid(queue),
"c.e": {"$lte": now},
"e": {"$gt": now}
}
if fields and not isinstance(fields, (dict, list)):
raise TypeError(_("Fields must be an instance of list / dict"))
if not echo and client_uuid:
query["u"] = {"$ne": client_uuid}
if marker:
query["_id"] = {"$gt": utils.to_oid(marker)}
return self._col.find(query, fields=fields)
def unclaim(self, queue_name, claim_id, project=None):
cid = utils.to_oid(claim_id)
# NOTE(cpp-cabrera): early abort - avoid a DB query if we're handling
# an invalid ID
if cid is None:
return
# NOTE(cpp-cabrera): unclaim by setting the claim ID to None
# and the claim expiration time to now
now = timeutils.utcnow()
self._col.update({'q': queue_name, 'p': project, 'c.id': cid},
{'$set': {'c': {'id': None, 'e': now}}},
upsert=False, multi=True)
def unclaim(self, queue_name, claim_id, project=None):
cid = utils.to_oid(claim_id)
# NOTE(cpp-cabrera): early abort - avoid a DB query if we're handling
# an invalid ID
if cid is None:
return
# NOTE(cpp-cabrera): unclaim by setting the claim ID to None
# and the claim expiration time to now
now = timeutils.utcnow()
self._col.update({'q': queue_name, 'p': project, 'c.id': cid},
{'$set': {'c': {'id': None, 'e': now}}},
upsert=False, multi=True)
def unclaim(self, queue_name, claim_id, project=None):
cid = utils.to_oid(claim_id)
# NOTE(cpp-cabrera): early abort - avoid a DB query if we're handling
# an invalid ID
if cid is None:
return
# NOTE(cpp-cabrera): unclaim by setting the claim ID to None
# and the claim expiration time to now
now = timeutils.utcnow()
self._col.update(
{"q": queue_name, "p": project, "c.id": cid},
{"$set": {"c": {"id": None, "e": now}}},
upsert=False,
multi=True,
)
def update(self, queue, claim_id, metadata, project=None):
try:
cid = utils.to_oid(claim_id)
except ValueError:
raise exceptions.ClaimDoesNotExist(claim_id, queue, project)
now = timeutils.utcnow()
ttl = int(metadata.get('ttl', 60))
ttl_delta = datetime.timedelta(seconds=ttl)
expires = now + ttl_delta
if now > expires:
raise ValueError('New ttl will make the claim expires')
msg_ctrl = self.driver.message_controller
claimed = msg_ctrl.claimed(queue, cid, expires=now,
limit=1, project=project)
try:
next(claimed)
except StopIteration:
raise exceptions.ClaimDoesNotExist(claim_id, queue, project)
meta = {
'id': cid,
't': ttl,
'e': expires,
}
msg_ctrl._col.update({'q': queue, 'p': project, 'c.id': cid},
{'$set': {'c': meta}},
upsert=False, multi=True)
# NOTE(flaper87): Dirty hack!
# This sets the expiration time to
# `expires` on messages that would
# expire before claim.
msg_ctrl._col.update({'q': queue,
'p': project,
'e': {'$lt': expires},
'c.id': cid},
{'$set': {'e': expires, 't': ttl}},
upsert=False, multi=True)
def get(self, queue_name, message_id, project=None):
"""Gets a single message by ID.
:raises: exceptions.MessageDoesNotExist
"""
mid = utils.to_oid(message_id)
if mid is None:
raise exceptions.MessageDoesNotExist(message_id, queue_name, project)
now = timeutils.utcnow()
query = {"_id": mid, "q": queue_name, "p": project, "e": {"$gt": now}}
message = list(self._col.find(query).limit(1).hint([("_id", 1)]))
if not message:
raise exceptions.MessageDoesNotExist(message_id, queue_name, project)
return _basic_message(message[0], now)
def bulk_get(self, queue_name, message_ids, project=None):
message_ids = [utils.to_oid(id) for id in message_ids]
now = timeutils.utcnow()
# Base query, always check expire time
query = {
'q': queue_name,
'p': project,
'_id': {'$in': message_ids},
'e': {'$gt': now},
}
# NOTE(flaper87): Should this query
# be sorted?
messages = self._col.find(query).hint([('_id', 1)])
def denormalizer(msg):
return _basic_message(msg, now)
return utils.HookedCursor(messages, denormalizer)
def get(self, queue, claim_id, project=None):
msg_ctrl = self.driver.message_controller
# Check whether the queue exists or not
qid = self._get_queue_id(queue, project)
# Base query, always check expire time
now = timeutils.utcnow()
try:
cid = utils.to_oid(claim_id)
except ValueError:
raise exceptions.ClaimDoesNotExist()
age = now - utils.oid_utc(cid)
def messages(msg_iter):
msg = next(msg_iter)
yield msg.pop('claim')
yield msg
# Smoke it!
for msg in msg_iter:
del msg['claim']
yield msg
try:
# Lets get claim's data
# from the first message
# in the iterator
messages = messages(msg_ctrl.claimed(qid, cid, now))
claim = next(messages)
claim = {
'age': age.seconds,
'ttl': claim.pop('t'),
'id': str(claim['id']),
}
except StopIteration:
raise exceptions.ClaimDoesNotExist(cid, queue, project)
return (claim, messages)
def update(self, queue, claim_id, metadata, tenant=None):
cid = utils.to_oid(claim_id)
now = timeutils.utcnow_ts()
ttl = int(metadata.get("ttl", 60))
# Lets remove the timezone,
# we want it to be plain utc
expires = utils.oid_ts(cid) + ttl
if now > expires:
msg = _("New ttl will make the claim expires")
raise ValueError(msg)
qid = self._get_queue_id(queue, tenant)
msg_ctrl = self.driver.message_controller
claimed = msg_ctrl.claimed(qid, cid, expires=now, limit=1)
try:
claimed.next()
except StopIteration:
raise exceptions.ClaimDoesNotExist(cid, queue, tenant)
meta = {
"id": cid,
"t": ttl,
"e": expires,
}
msg_ctrl._col.update({"q": qid, "c.id": cid},
{"$set": {"c": meta}},
upsert=False, multi=True)
# NOTE(flaper87): Dirty hack!
# This sets the expiration time to
# `expires` on messages that would
# expire before claim.
msg_ctrl._col.update({"q": qid,
"e": {"$lte": expires},
"c.id": cid},
{"$set": {"e": expires}},
upsert=False, multi=True)
def get(self, queue, claim_id, project=None):
msg_ctrl = self.driver.message_controller
# Base query, always check expire time
now = timeutils.utcnow()
cid = utils.to_oid(claim_id)
if cid is None:
raise exceptions.ClaimDoesNotExist(queue, project, claim_id)
def messages(msg_iter):
msg = next(msg_iter)
yield msg.pop('claim')
yield msg
# Smoke it!
for msg in msg_iter:
del msg['claim']
yield msg
try:
# Lets get claim's data
# from the first message
# in the iterator
msgs = messages(msg_ctrl.claimed(queue, cid, now,
project=project))
claim = next(msgs)
update_time = claim['e'] - datetime.timedelta(seconds=claim['t'])
age = timeutils.delta_seconds(update_time, now)
claim = {
'age': int(age),
'ttl': claim.pop('t'),
'id': str(claim['id']),
}
except StopIteration:
raise exceptions.ClaimDoesNotExist(cid, queue, project)
return (claim, msgs)
def get(self, queue, claim_id, project=None):
msg_ctrl = self.driver.message_controller
# Base query, always check expire time
now = timeutils.utcnow()
cid = utils.to_oid(claim_id)
if cid is None:
raise exceptions.ClaimDoesNotExist(queue, project, claim_id)
def messages(msg_iter):
msg = next(msg_iter)
yield msg.pop('claim')
yield msg
# Smoke it!
for msg in msg_iter:
del msg['claim']
yield msg
try:
# Lets get claim's data
# from the first message
# in the iterator
msgs = messages(msg_ctrl.claimed(queue, cid, now, project=project))
claim = next(msgs)
update_time = claim['e'] - datetime.timedelta(seconds=claim['t'])
age = timeutils.delta_seconds(update_time, now)
claim = {
'age': int(age),
'ttl': claim.pop('t'),
'id': str(claim['id']),
}
except StopIteration:
raise exceptions.ClaimDoesNotExist(cid, queue, project)
return (claim, msgs)
def get(self, queue, message_id, tenant=None):
# Base query, always check expire time
mid = utils.to_oid(message_id)
query = {
"q": self._get_queue_id(queue, tenant),
"e": {"$gt": timeutils.utcnow_ts()},
"_id": mid
}
message = self._col.find_one(query)
if message is None:
raise exceptions.MessageDoesNotExist(mid, queue, tenant)
oid = message.get("_id")
age = timeutils.utcnow_ts() - utils.oid_ts(oid)
return {
"id": oid,
"age": age,
"ttl": message["t"],
"body": message["b"],
}
def active(self, queue_id, marker=None, echo=False,
client_uuid=None, fields=None):
now = timeutils.utcnow()
query = {
# Messages must belong to this queue
'q': utils.to_oid(queue_id),
# The messages can not be expired
'e': {'$gt': now},
# Include messages that are part of expired claims
'c.e': {'$lte': now},
}
if fields and not isinstance(fields, (dict, list)):
raise TypeError(_('Fields must be an instance of list / dict'))
if not echo and client_uuid:
query['u'] = {'$ne': client_uuid}
if marker:
query['k'] = {'$gt': marker}
return self._col.find(query, fields=fields)
def get(self, queue, claim_id, tenant=None):
msg_ctrl = self.driver.message_controller
# Check whether the queue exists or not
qid = self._get_queue_id(queue, tenant)
# Base query, always check expire time
now = timeutils.utcnow_ts()
cid = utils.to_oid(claim_id)
age = now - utils.oid_ts(cid)
def messages(msg_iter):
msg = msg_iter.next()
yield msg.pop("claim")
yield msg
# Smoke it!
for msg in msg_iter:
del msg["claim"]
yield msg
try:
# Lets get claim's data
# from the first message
# in the iterator
messages = messages(msg_ctrl.claimed(qid, cid, now))
claim = messages.next()
claim = {
"age": age,
"ttl": claim.pop("t"),
"id": str(claim.pop("id")),
}
except StopIteration:
raise exceptions.ClaimDoesNotExist(cid, queue, tenant)
return (claim, messages)
def delete(self, queue, message_id, tenant=None, claim=None):
self._get_queue_id(queue, tenant)
mid = utils.to_oid(message_id)
self._col.remove(mid, w=0)
def unclaim(self, claim_id):
cid = utils.to_oid(claim_id)
self._col.update({"c.id": cid},
{"$unset": {"c": True}},
upsert=False, multi=True)