def delete(self, queue_name, message_id, project=None, claim=None): # NOTE(cpp-cabrera): return early - this is an invalid message # id so we won't be able to find it any way mid = utils.to_oid(message_id) if mid is None: return query = {"q": queue_name, "p": project, "_id": mid} # NOTE(cpp-cabrera): return early - the user gaves us an # invalid claim id and that renders the rest of this # request moot cid = utils.to_oid(claim) if cid is None: return now = timeutils.utcnow() query["e"] = {"$gt": now} message = self._col.find_one(query) if message is None: return is_claimed = message["c"]["id"] is not None and message["c"]["e"] > now if claim is None: if is_claimed: raise exceptions.MessageIsClaimed(message_id) else: if message["c"]["id"] != cid: raise exceptions.MessageIsClaimedBy(message_id, claim) self._col.remove(query["_id"], w=0)
def delete(self, queue_name, message_id, project=None, claim=None): # NOTE(cpp-cabrera): return early - this is an invalid message # id so we won't be able to find it any way mid = utils.to_oid(message_id) if mid is None: return query = {'q': queue_name, 'p': project, '_id': mid} # NOTE(cpp-cabrera): return early - the user gaves us an # invalid claim id and that renders the rest of this # request moot cid = utils.to_oid(claim) if cid is None: return now = timeutils.utcnow() query['e'] = {'$gt': now} message = self._col.find_one(query) if message is None: return is_claimed = (message['c']['id'] is not None and message['c']['e'] > now) if claim is None: if is_claimed: raise exceptions.MessageIsClaimed(message_id) else: if message['c']['id'] != cid: raise exceptions.MessageIsClaimedBy(message_id, claim) self._col.remove(query['_id'], w=0)
def delete(self, queue, message_id, project=None, claim=None): try: mid = utils.to_oid(message_id) query = { 'q': self._get_queue_id(queue, project), '_id': mid } if claim: now = timeutils.utcnow() query['e'] = {'$gt': now} message = self._col.find_one(query) if message is None: return cid = utils.to_oid(claim) if not ('c' in message and message['c']['id'] == cid and message['c']['e'] > now): raise exceptions.ClaimNotPermitted(message_id, claim) self._col.remove(query['_id'], w=0) else: self._col.remove(query, w=0) except exceptions.QueueDoesNotExist: pass
def get(self, queue_name, message_id, project=None): """Gets a single message by ID. :raises: exceptions.MessageDoesNotExist """ mid = utils.to_oid(message_id) if mid is None: raise exceptions.MessageDoesNotExist(message_id, queue_name, project) now = timeutils.utcnow() query = { '_id': mid, 'q': queue_name, 'p': project, 'e': {'$gt': now} } message = list(self._col.find(query).limit(1).hint([('_id', 1)])) if not message: raise exceptions.MessageDoesNotExist(message_id, queue_name, project) return _basic_message(message[0], now)
def claimed(self, queue, claim_id=None, expires=None, limit=None): query = { "c.id": claim_id, "q": utils.to_oid(queue), } if not claim_id: # lookup over c.id to use the index query["c.id"] = {"$ne": None} if expires: query["c.e"] = {"$gt": expires} msgs = self._col.find(query, sort=[("_id", 1)]) if limit: msgs = msgs.limit(limit) now = timeutils.utcnow_ts() def denormalizer(msg): oid = msg.get("_id") age = now - utils.oid_ts(oid) return { "id": str(oid), "age": age, "ttl": msg["t"], "body": msg["b"], "claim": msg["c"] } return utils.HookedCursor(msgs, denormalizer)
def claimed(self, queue_id, claim_id=None, expires=None, limit=None): query = { 'c.id': claim_id, 'c.e': {'$gt': expires or timeutils.utcnow()}, 'q': utils.to_oid(queue_id), } if not claim_id: # lookup over c.id to use the index query['c.id'] = {'$ne': None} msgs = self._col.find(query, sort=[('_id', 1)]) if limit: msgs = msgs.limit(limit) now = timeutils.utcnow() def denormalizer(msg): oid = msg['_id'] age = now - utils.oid_utc(oid) return { 'id': str(oid), 'age': age.seconds, 'ttl': msg['t'], 'body': msg['b'], 'claim': msg['c'] } return utils.HookedCursor(msgs, denormalizer)
def get(self, queue, message_ids, project=None): if not isinstance(message_ids, list): message_ids = [message_ids] message_ids = [utils.to_oid(id) for id in message_ids] now = timeutils.utcnow() # Base query, always check expire time query = { 'q': self._get_queue_id(queue, project), 'e': {'$gt': now}, '_id': {'$in': message_ids}, } messages = self._col.find(query) def denormalizer(msg): oid = msg['_id'] age = now - utils.oid_utc(oid) return { 'id': str(oid), 'age': age.seconds, 'ttl': msg['t'], 'body': msg['b'], } return utils.HookedCursor(messages, denormalizer)
def unclaim(self, queue_name, claim_id, project=None): try: cid = utils.to_oid(claim_id) except ValueError: return self._col.update({'q': queue_name, 'p': project, 'c.id': cid}, {'$set': {'c': {'id': None, 'e': 0}}}, upsert=False, multi=True)
def update(self, queue, claim_id, metadata, project=None): cid = utils.to_oid(claim_id) if cid is None: raise exceptions.ClaimDoesNotExist(claim_id, queue, project) now = timeutils.utcnow() ttl = int(metadata.get('ttl', 60)) ttl_delta = datetime.timedelta(seconds=ttl) expires = now + ttl_delta msg_ctrl = self.driver.message_controller claimed = msg_ctrl.claimed(queue, cid, expires=now, limit=1, project=project) try: next(claimed) except StopIteration: raise exceptions.ClaimDoesNotExist(claim_id, queue, project) meta = { 'id': cid, 't': ttl, 'e': expires, } msg_ctrl._col.update({ 'q': queue, 'p': project, 'c.id': cid }, {'$set': { 'c': meta }}, upsert=False, multi=True) # NOTE(flaper87): Dirty hack! # This sets the expiration time to # `expires` on messages that would # expire before claim. msg_ctrl._col.update( { 'q': queue, 'p': project, 'e': { '$lt': expires }, 'c.id': cid }, {'$set': { 'e': expires, 't': ttl }}, upsert=False, multi=True)
def unclaim(self, claim_id): try: cid = utils.to_oid(claim_id) except ValueError: return self._col.update({'c.id': cid}, {'$set': {'c': {'id': None, 'e': 0}}}, upsert=False, multi=True)
def active(self, queue_id, marker=None, echo=False, client_uuid=None, fields=None): # NOTE(kgriffs): Since this is a public method, queue_id # might not be an ObjectID. Usually it will be, since active() # is a utility method, so short-circuit for performance. if not isinstance(queue_id, objectid.ObjectId): queue_id = utils.to_oid(queue_id) return self._list(queue_id, marker, echo, client_uuid, fields, include_claimed=False)
def bulk_delete(self, queue_name, message_ids, project=None): try: message_ids = [utils.to_oid(id) for id in message_ids] query = { 'q': queue_name, 'p': project, '_id': {'$in': message_ids}, } self._col.remove(query, w=0) except exceptions.QueueDoesNotExist: pass
def active(self, queue, marker=None, echo=False, client_uuid=None, fields=None): now = timeutils.utcnow_ts() query = { # Messages must belong to this queue "q": utils.to_oid(queue), "c.e": {"$lte": now}, "e": {"$gt": now} } if fields and not isinstance(fields, (dict, list)): raise TypeError(_("Fields must be an instance of list / dict")) if not echo and client_uuid: query["u"] = {"$ne": client_uuid} if marker: query["_id"] = {"$gt": utils.to_oid(marker)} return self._col.find(query, fields=fields)
def unclaim(self, queue_name, claim_id, project=None): cid = utils.to_oid(claim_id) # NOTE(cpp-cabrera): early abort - avoid a DB query if we're handling # an invalid ID if cid is None: return # NOTE(cpp-cabrera): unclaim by setting the claim ID to None # and the claim expiration time to now now = timeutils.utcnow() self._col.update({'q': queue_name, 'p': project, 'c.id': cid}, {'$set': {'c': {'id': None, 'e': now}}}, upsert=False, multi=True)
def unclaim(self, queue_name, claim_id, project=None): cid = utils.to_oid(claim_id) # NOTE(cpp-cabrera): early abort - avoid a DB query if we're handling # an invalid ID if cid is None: return # NOTE(cpp-cabrera): unclaim by setting the claim ID to None # and the claim expiration time to now now = timeutils.utcnow() self._col.update( {"q": queue_name, "p": project, "c.id": cid}, {"$set": {"c": {"id": None, "e": now}}}, upsert=False, multi=True, )
def update(self, queue, claim_id, metadata, project=None): try: cid = utils.to_oid(claim_id) except ValueError: raise exceptions.ClaimDoesNotExist(claim_id, queue, project) now = timeutils.utcnow() ttl = int(metadata.get('ttl', 60)) ttl_delta = datetime.timedelta(seconds=ttl) expires = now + ttl_delta if now > expires: raise ValueError('New ttl will make the claim expires') msg_ctrl = self.driver.message_controller claimed = msg_ctrl.claimed(queue, cid, expires=now, limit=1, project=project) try: next(claimed) except StopIteration: raise exceptions.ClaimDoesNotExist(claim_id, queue, project) meta = { 'id': cid, 't': ttl, 'e': expires, } msg_ctrl._col.update({'q': queue, 'p': project, 'c.id': cid}, {'$set': {'c': meta}}, upsert=False, multi=True) # NOTE(flaper87): Dirty hack! # This sets the expiration time to # `expires` on messages that would # expire before claim. msg_ctrl._col.update({'q': queue, 'p': project, 'e': {'$lt': expires}, 'c.id': cid}, {'$set': {'e': expires, 't': ttl}}, upsert=False, multi=True)
def get(self, queue_name, message_id, project=None): """Gets a single message by ID. :raises: exceptions.MessageDoesNotExist """ mid = utils.to_oid(message_id) if mid is None: raise exceptions.MessageDoesNotExist(message_id, queue_name, project) now = timeutils.utcnow() query = {"_id": mid, "q": queue_name, "p": project, "e": {"$gt": now}} message = list(self._col.find(query).limit(1).hint([("_id", 1)])) if not message: raise exceptions.MessageDoesNotExist(message_id, queue_name, project) return _basic_message(message[0], now)
def bulk_get(self, queue_name, message_ids, project=None): message_ids = [utils.to_oid(id) for id in message_ids] now = timeutils.utcnow() # Base query, always check expire time query = { 'q': queue_name, 'p': project, '_id': {'$in': message_ids}, 'e': {'$gt': now}, } # NOTE(flaper87): Should this query # be sorted? messages = self._col.find(query).hint([('_id', 1)]) def denormalizer(msg): return _basic_message(msg, now) return utils.HookedCursor(messages, denormalizer)
def get(self, queue, claim_id, project=None): msg_ctrl = self.driver.message_controller # Check whether the queue exists or not qid = self._get_queue_id(queue, project) # Base query, always check expire time now = timeutils.utcnow() try: cid = utils.to_oid(claim_id) except ValueError: raise exceptions.ClaimDoesNotExist() age = now - utils.oid_utc(cid) def messages(msg_iter): msg = next(msg_iter) yield msg.pop('claim') yield msg # Smoke it! for msg in msg_iter: del msg['claim'] yield msg try: # Lets get claim's data # from the first message # in the iterator messages = messages(msg_ctrl.claimed(qid, cid, now)) claim = next(messages) claim = { 'age': age.seconds, 'ttl': claim.pop('t'), 'id': str(claim['id']), } except StopIteration: raise exceptions.ClaimDoesNotExist(cid, queue, project) return (claim, messages)
def update(self, queue, claim_id, metadata, tenant=None): cid = utils.to_oid(claim_id) now = timeutils.utcnow_ts() ttl = int(metadata.get("ttl", 60)) # Lets remove the timezone, # we want it to be plain utc expires = utils.oid_ts(cid) + ttl if now > expires: msg = _("New ttl will make the claim expires") raise ValueError(msg) qid = self._get_queue_id(queue, tenant) msg_ctrl = self.driver.message_controller claimed = msg_ctrl.claimed(qid, cid, expires=now, limit=1) try: claimed.next() except StopIteration: raise exceptions.ClaimDoesNotExist(cid, queue, tenant) meta = { "id": cid, "t": ttl, "e": expires, } msg_ctrl._col.update({"q": qid, "c.id": cid}, {"$set": {"c": meta}}, upsert=False, multi=True) # NOTE(flaper87): Dirty hack! # This sets the expiration time to # `expires` on messages that would # expire before claim. msg_ctrl._col.update({"q": qid, "e": {"$lte": expires}, "c.id": cid}, {"$set": {"e": expires}}, upsert=False, multi=True)
def get(self, queue, claim_id, project=None): msg_ctrl = self.driver.message_controller # Base query, always check expire time now = timeutils.utcnow() cid = utils.to_oid(claim_id) if cid is None: raise exceptions.ClaimDoesNotExist(queue, project, claim_id) def messages(msg_iter): msg = next(msg_iter) yield msg.pop('claim') yield msg # Smoke it! for msg in msg_iter: del msg['claim'] yield msg try: # Lets get claim's data # from the first message # in the iterator msgs = messages(msg_ctrl.claimed(queue, cid, now, project=project)) claim = next(msgs) update_time = claim['e'] - datetime.timedelta(seconds=claim['t']) age = timeutils.delta_seconds(update_time, now) claim = { 'age': int(age), 'ttl': claim.pop('t'), 'id': str(claim['id']), } except StopIteration: raise exceptions.ClaimDoesNotExist(cid, queue, project) return (claim, msgs)
def get(self, queue, message_id, tenant=None): # Base query, always check expire time mid = utils.to_oid(message_id) query = { "q": self._get_queue_id(queue, tenant), "e": {"$gt": timeutils.utcnow_ts()}, "_id": mid } message = self._col.find_one(query) if message is None: raise exceptions.MessageDoesNotExist(mid, queue, tenant) oid = message.get("_id") age = timeutils.utcnow_ts() - utils.oid_ts(oid) return { "id": oid, "age": age, "ttl": message["t"], "body": message["b"], }
def active(self, queue_id, marker=None, echo=False, client_uuid=None, fields=None): now = timeutils.utcnow() query = { # Messages must belong to this queue 'q': utils.to_oid(queue_id), # The messages can not be expired 'e': {'$gt': now}, # Include messages that are part of expired claims 'c.e': {'$lte': now}, } if fields and not isinstance(fields, (dict, list)): raise TypeError(_('Fields must be an instance of list / dict')) if not echo and client_uuid: query['u'] = {'$ne': client_uuid} if marker: query['k'] = {'$gt': marker} return self._col.find(query, fields=fields)
def get(self, queue, claim_id, tenant=None): msg_ctrl = self.driver.message_controller # Check whether the queue exists or not qid = self._get_queue_id(queue, tenant) # Base query, always check expire time now = timeutils.utcnow_ts() cid = utils.to_oid(claim_id) age = now - utils.oid_ts(cid) def messages(msg_iter): msg = msg_iter.next() yield msg.pop("claim") yield msg # Smoke it! for msg in msg_iter: del msg["claim"] yield msg try: # Lets get claim's data # from the first message # in the iterator messages = messages(msg_ctrl.claimed(qid, cid, now)) claim = messages.next() claim = { "age": age, "ttl": claim.pop("t"), "id": str(claim.pop("id")), } except StopIteration: raise exceptions.ClaimDoesNotExist(cid, queue, tenant) return (claim, messages)
def delete(self, queue, message_id, tenant=None, claim=None): self._get_queue_id(queue, tenant) mid = utils.to_oid(message_id) self._col.remove(mid, w=0)
def unclaim(self, claim_id): cid = utils.to_oid(claim_id) self._col.update({"c.id": cid}, {"$unset": {"c": True}}, upsert=False, multi=True)