def delete(self, queue, claim_id, project=None): # NOTE(prashanthr_): Return silently when the claim # does not exist if not self._exists(queue, claim_id, project): return now = timeutils.utcnow_ts() claim_msgs_key = utils.scope_claim_messages(claim_id, CLAIM_MESSAGES_SUFFIX) msg_keys = self._get_claimed_message_keys(claim_msgs_key) claimed_msgs = messages.MessageEnvelope.from_redis_bulk(msg_keys, self._client) # Update the claim id and claim expiration info # for all the messages. claims_set_key = utils.scope_claims_set(queue, project, QUEUE_CLAIMS_SUFFIX) with self._client.pipeline() as pipe: pipe.zrem(claims_set_key, claim_id) pipe.delete(claim_id) pipe.delete(claim_msgs_key) for msg in claimed_msgs: if msg: msg.claim_id = None msg.claim_expires = now # TODO(kgriffs): Rather than writing back the # entire message, only set the fields that # have changed. msg.to_redis(pipe) pipe.execute()
def _exists(self, queue, claim_id, project): client = self._client claims_set_key = utils.scope_claims_set(queue, project, QUEUE_CLAIMS_SUFFIX) # In some cases, the queue maybe doesn't exist. So we should check # whether the queue exists. Return False if no such queue exists. # Todo(flwang): We should delete all related data after the queue is # deleted. See the blueprint for more detail: # https://blueprints.launchpad.net/zaqar/+spec/clear-resources-after-delete-queue if not self._queue_ctrl._exists(queue, project): return False # Return False if no such claim exists # TODO(prashanthr_): Discuss the feasibility of a bloom filter. if client.zscore(claims_set_key, claim_id) is None: return False expires = self._get_claim_info(claim_id, b'e')[0] now = timeutils.utcnow_ts() if expires <= now: # NOTE(kgriffs): Redis should automatically remove the # other records in the very near future. This one # has to be manually deleted, however. client.zrem(claims_set_key, claim_id) return False return True
def create(self, queue, metadata, project=None, limit=storage.DEFAULT_MESSAGES_PER_CLAIM): claim_ttl = metadata['ttl'] grace = metadata['grace'] now = timeutils.utcnow_ts() msg_ttl = claim_ttl + grace claim_expires = now + claim_ttl msg_expires = claim_expires + grace claim_id = uuidutils.generate_uuid() claimed_msgs = [] # NOTE(kgriffs): Claim some messages msgset_key = utils.msgset_key(queue, project) claimed_ids = self._claim_messages(msgset_key, now, limit, claim_id, claim_expires, msg_ttl, msg_expires) if claimed_ids: claimed_msgs = messages.Message.from_redis_bulk(claimed_ids, self._client) claimed_msgs = [msg.to_basic(now) for msg in claimed_msgs] # NOTE(kgriffs): Perist claim records with self._client.pipeline() as pipe: claim_msgs_key = utils.scope_claim_messages( claim_id, CLAIM_MESSAGES_SUFFIX) for mid in claimed_ids: pipe.rpush(claim_msgs_key, mid) pipe.expire(claim_msgs_key, claim_ttl) claim_info = { 'id': claim_id, 't': claim_ttl, 'e': claim_expires, 'n': len(claimed_ids), } pipe.hmset(claim_id, claim_info) pipe.expire(claim_id, claim_ttl) # NOTE(kgriffs): Add the claim ID to a set so that # existence checks can be performed quickly. This # is also used as a watch key in order to gaurd # against race conditions. # # A sorted set is used to facilitate cleaning # up the IDs of expired claims. claims_set_key = utils.scope_claims_set(queue, project, QUEUE_CLAIMS_SUFFIX) pipe.zadd(claims_set_key, claim_expires, claim_id) pipe.execute() return claim_id, claimed_msgs
def update(self, queue, claim_id, metadata, project=None): if not self._exists(queue, claim_id, project): raise errors.ClaimDoesNotExist(claim_id, queue, project) now = timeutils.utcnow_ts() claim_ttl = metadata['ttl'] claim_expires = now + claim_ttl grace = metadata['grace'] msg_ttl = claim_ttl + grace msg_expires = claim_expires + grace claim_msgs_key = utils.scope_claim_messages(claim_id, CLAIM_MESSAGES_SUFFIX) msg_keys = self._get_claimed_message_keys(claim_msgs_key) claimed_msgs = messages.MessageEnvelope.from_redis_bulk(msg_keys, self._client) claim_info = { 't': claim_ttl, 'e': claim_expires, } with self._client.pipeline() as pipe: for msg in claimed_msgs: if msg: msg.claim_id = claim_id msg.claim_expires = claim_expires if _msg_would_expire(msg, claim_expires): msg.ttl = msg_ttl msg.expires = msg_expires # TODO(kgriffs): Rather than writing back the # entire message, only set the fields that # have changed. # # When this change is made, don't forget to # also call pipe.expire with the new TTL value. msg.to_redis(pipe) # Update the claim id and claim expiration info # for all the messages. pipe.hmset(claim_id, claim_info) pipe.expire(claim_id, claim_ttl) pipe.expire(claim_msgs_key, claim_ttl) claims_set_key = utils.scope_claims_set(queue, project, QUEUE_CLAIMS_SUFFIX) pipe.zadd(claims_set_key, claim_expires, claim_id) pipe.execute()
def _gc(self, queue, project): """Garbage-collect expired claim data. Not all claim data can be automatically expired. This method cleans up the remainder. :returns: Number of claims removed """ claims_set_key = utils.scope_claims_set(queue, project, QUEUE_CLAIMS_SUFFIX) now = timeutils.utcnow_ts() num_removed = self._client.zremrangebyscore(claims_set_key, 0, now) return num_removed
def _exists(self, queue, claim_id, project): client = self._client claims_set_key = utils.scope_claims_set(queue, project, QUEUE_CLAIMS_SUFFIX) # Return False if no such claim exists # TODO(prashanthr_): Discuss the feasibility of a bloom filter. if client.zscore(claims_set_key, claim_id) is None: return False expires = self._get_claim_info(claim_id, b'e')[0] now = timeutils.utcnow_ts() if expires <= now: # NOTE(kgriffs): Redis should automatically remove the # other records in the very near future. This one # has to be manually deleted, however. client.zrem(claims_set_key, claim_id) return False return True
def _count_messages(self, queue, project): """Count and return the total number of claimed messages.""" # NOTE(kgriffs): Iterate through all claims, adding up the # number of messages per claim. This is obviously slower # than keeping a side counter, but is also less error-prone. # Plus, it avoids having to do a lot of extra work during # garbage collection passes. Also, considering that most # workloads won't require a large number of claims, most of # the time we can do this in a single pass, so it is still # pretty fast. claims_set_key = utils.scope_claims_set(queue, project, QUEUE_CLAIMS_SUFFIX) num_claimed = 0 offset = 0 while True: claim_ids = self._client.zrange(claims_set_key, offset, offset + COUNTING_BATCH_SIZE - 1) if not claim_ids: break offset += len(claim_ids) with self._client.pipeline() as pipe: for cid in claim_ids: pipe.hmget(cid, 'n') claim_infos = pipe.execute() for info in claim_infos: # NOTE(kgriffs): In case the claim was deleted out # from under us, sanity-check that we got a non-None # info list. if info: num_claimed += int(info[0]) return num_claimed
def create(self, queue, metadata, project=None, limit=storage.DEFAULT_MESSAGES_PER_CLAIM): queue_ctrl = self.driver.queue_controller msg_ctrl = self.driver.message_controller claim_ttl = metadata['ttl'] grace = metadata['grace'] now = timeutils.utcnow_ts() msg_ttl = claim_ttl + grace claim_expires = now + claim_ttl msg_expires = claim_expires + grace # Get the maxClaimCount and deadLetterQueue from current queue's meta queue_meta = queue_ctrl.get(queue, project=project) claim_id = uuidutils.generate_uuid() claimed_msgs = [] # NOTE(kgriffs): Claim some messages msgset_key = utils.msgset_key(queue, project) claimed_ids = self._claim_messages(msgset_key, now, limit, claim_id, claim_expires, msg_ttl, msg_expires) if claimed_ids: claimed_msgs = messages.Message.from_redis_bulk(claimed_ids, self._client) claimed_msgs = [msg.to_basic(now) for msg in claimed_msgs] # NOTE(kgriffs): Perist claim records with self._client.pipeline() as pipe: claim_msgs_key = utils.scope_claim_messages( claim_id, CLAIM_MESSAGES_SUFFIX) for mid in claimed_ids: pipe.rpush(claim_msgs_key, mid) pipe.expire(claim_msgs_key, claim_ttl) claim_info = { 'id': claim_id, 't': claim_ttl, 'e': claim_expires, 'n': len(claimed_ids), } pipe.hmset(claim_id, claim_info) pipe.expire(claim_id, claim_ttl) # NOTE(kgriffs): Add the claim ID to a set so that # existence checks can be performed quickly. This # is also used as a watch key in order to guard # against race conditions. # # A sorted set is used to facilitate cleaning # up the IDs of expired claims. claims_set_key = utils.scope_claims_set(queue, project, QUEUE_CLAIMS_SUFFIX) pipe.zadd(claims_set_key, {claim_id: claim_expires}) pipe.execute() if ('_max_claim_count' in queue_meta and '_dead_letter_queue' in queue_meta): claimed_msgs_removed = [] for msg in claimed_msgs: if msg: claimed_count = msg['claim_count'] if claimed_count < queue_meta['_max_claim_count']: # 1. Save the new max claim count for message claim_count = claimed_count + 1 dic = {"c.c": claim_count} pipe.hmset(msg['id'], dic) pipe.execute() else: # 2. Check if the message's claim count has # exceeded the max claim count defined in the # queue, if so, move the message to the dead # letter queue and modify it's ttl. # NOTE(gengchc): We're moving message by # moving the message id from queue to dead # letter queue directly.That means, the queue # and dead letter queue must be created on # the same pool. ddl = utils.scope_queue_name( queue_meta['_dead_letter_queue'], project) ddl_ttl = queue_meta.get( "_dead_letter_queue_messages_ttl") dic = {"t": msg['ttl']} if ddl_ttl: dic = {"t": ddl_ttl} pipe.hmset(msg['id'], dic) queueproject = [s for s in ddl.split('.')] msgs_key_ddl = utils.msgset_key( queueproject[1], queueproject[0]) counter_key_ddl = utils.scope_queue_index( queueproject[1], queueproject[0], MESSAGE_RANK_COUNTER_SUFFIX) msgs_key = utils.msgset_key( queue, project=project) pipe.zrem(msgs_key, msg['id']) message_ids = [] message_ids.append(msg['id']) msg_ctrl._index_messages(msgs_key_ddl, counter_key_ddl, message_ids) pipe.execute() # Add dead letter message to # claimed_msgs_removed, finally remove # them from claimed_msgs. claimed_msgs_removed.append(msg) # Remove dead letter messages from claimed_msgs. for msg_remove in claimed_msgs_removed: claimed_msgs.remove(msg_remove) if len(claimed_msgs) == 0: return None, iter([]) return claim_id, claimed_msgs
def create(self, queue, metadata, project=None, limit=storage.DEFAULT_MESSAGES_PER_CLAIM): queue_ctrl = self.driver.queue_controller msg_ctrl = self.driver.message_controller claim_ttl = metadata['ttl'] grace = metadata['grace'] now = timeutils.utcnow_ts() msg_ttl = claim_ttl + grace claim_expires = now + claim_ttl msg_expires = claim_expires + grace # Get the maxClaimCount and deadLetterQueue from current queue's meta queue_meta = queue_ctrl.get(queue, project=project) claim_id = uuidutils.generate_uuid() claimed_msgs = [] # NOTE(kgriffs): Claim some messages msgset_key = utils.msgset_key(queue, project) claimed_ids = self._claim_messages(msgset_key, now, limit, claim_id, claim_expires, msg_ttl, msg_expires) if claimed_ids: claimed_msgs = messages.Message.from_redis_bulk( claimed_ids, self._client) claimed_msgs = [msg.to_basic(now) for msg in claimed_msgs] # NOTE(kgriffs): Perist claim records with self._client.pipeline() as pipe: claim_msgs_key = utils.scope_claim_messages( claim_id, CLAIM_MESSAGES_SUFFIX) for mid in claimed_ids: pipe.rpush(claim_msgs_key, mid) pipe.expire(claim_msgs_key, claim_ttl) claim_info = { 'id': claim_id, 't': claim_ttl, 'e': claim_expires, 'n': len(claimed_ids), } pipe.hmset(claim_id, claim_info) pipe.expire(claim_id, claim_ttl) # NOTE(kgriffs): Add the claim ID to a set so that # existence checks can be performed quickly. This # is also used as a watch key in order to guard # against race conditions. # # A sorted set is used to facilitate cleaning # up the IDs of expired claims. claims_set_key = utils.scope_claims_set( queue, project, QUEUE_CLAIMS_SUFFIX) pipe.zadd(claims_set_key, claim_expires, claim_id) pipe.execute() if ('_max_claim_count' in queue_meta and '_dead_letter_queue' in queue_meta): claimed_msgs_removed = [] for msg in claimed_msgs: if msg: claimed_count = msg['claim_count'] if claimed_count < queue_meta['_max_claim_count']: # 1. Save the new max claim count for message claim_count = claimed_count + 1 dic = {"c.c": claim_count} pipe.hmset(msg['id'], dic) pipe.execute() else: # 2. Check if the message's claim count has # exceeded the max claim count defined in the # queue, if so, move the message to the dead # letter queue and modify it's ttl. # NOTE(gengchc): We're moving message by # moving the message id from queue to dead # letter queue directly.That means, the queue # and dead letter queue must be created on # the same pool. ddl = utils.scope_queue_name( queue_meta['_dead_letter_queue'], project) ddl_ttl = queue_meta.get( "_dead_letter_queue_messages_ttl") dic = {"t": msg['ttl']} if ddl_ttl: dic = {"t": ddl_ttl} pipe.hmset(msg['id'], dic) queueproject = [s for s in ddl.split('.')] msgs_key_ddl = utils.msgset_key( queueproject[1], queueproject[0]) counter_key_ddl = utils.scope_queue_index( queueproject[1], queueproject[0], MESSAGE_RANK_COUNTER_SUFFIX) msgs_key = utils.msgset_key(queue, project=project) pipe.zrem(msgs_key, msg['id']) message_ids = [] message_ids.append(msg['id']) msg_ctrl._index_messages( msgs_key_ddl, counter_key_ddl, message_ids) pipe.execute() # Add dead letter message to # claimed_msgs_removed, finally remove # them from claimed_msgs. claimed_msgs_removed.append(msg) # Remove dead letter messages from claimed_msgs. for msg_remove in claimed_msgs_removed: claimed_msgs.remove(msg_remove) if len(claimed_msgs) == 0: return None, iter([]) return claim_id, claimed_msgs