def get(self, queue, claim_id, project=None): if not self._exists(queue, claim_id, project): raise errors.ClaimDoesNotExist(queue, project, claim_id) claim_msgs_key = utils.scope_claim_messages(claim_id, CLAIM_MESSAGES_SUFFIX) # basic_messages msg_keys = self._get_claimed_message_keys(claim_msgs_key) claimed_msgs = messages.Message.from_redis_bulk(msg_keys, self._client) now = timeutils.utcnow_ts() basic_messages = [msg.to_basic(now) for msg in claimed_msgs if msg] # claim_meta now = timeutils.utcnow_ts() expires, ttl = self._get_claim_info(claim_id, [b'e', b't']) update_time = expires - ttl age = now - update_time claim_meta = { 'age': age, 'ttl': ttl, 'id': claim_id, } return claim_meta, basic_messages
def delete(self, queue, claim_id, project=None): # NOTE(prashanthr_): Return silently when the claim # does not exist if not self._exists(queue, claim_id, project): return now = timeutils.utcnow_ts() claim_msgs_key = utils.scope_claim_messages(claim_id, CLAIM_MESSAGES_SUFFIX) msg_keys = self._get_claimed_message_keys(claim_msgs_key) claimed_msgs = messages.MessageEnvelope.from_redis_bulk(msg_keys, self._client) # Update the claim id and claim expiration info # for all the messages. claims_set_key = utils.scope_claims_set(queue, project, QUEUE_CLAIMS_SUFFIX) with self._client.pipeline() as pipe: pipe.zrem(claims_set_key, claim_id) pipe.delete(claim_id) pipe.delete(claim_msgs_key) for msg in claimed_msgs: if msg: msg.claim_id = None msg.claim_expires = now # TODO(kgriffs): Rather than writing back the # entire message, only set the fields that # have changed. msg.to_redis(pipe) pipe.execute()
def get(self, queue, claim_id, project=None): if not self._exists(queue, claim_id, project): raise errors.ClaimDoesNotExist(claim_id, queue, project) claim_msgs_key = utils.scope_claim_messages(claim_id, CLAIM_MESSAGES_SUFFIX) # basic_messages msg_keys = self._get_claimed_message_keys(claim_msgs_key) claimed_msgs = messages.Message.from_redis_bulk(msg_keys, self._client) now = timeutils.utcnow_ts() basic_messages = [msg.to_basic(now) for msg in claimed_msgs if msg] # claim_meta now = timeutils.utcnow_ts() expires, ttl = self._get_claim_info(claim_id, [b'e', b't']) update_time = expires - ttl age = now - update_time claim_meta = { 'age': age, 'ttl': ttl, 'id': claim_id, } return claim_meta, basic_messages
def delete(self, queue, claim_id, project=None): # NOTE(prashanthr_): Return silently when the claim # does not exist if not self._exists(queue, claim_id, project): return now = timeutils.utcnow_ts() claim_msgs_key = utils.scope_claim_messages(claim_id, CLAIM_MESSAGES_SUFFIX) msg_keys = self._get_claimed_message_keys(claim_msgs_key) claimed_msgs = messages.MessageEnvelope.from_redis_bulk( msg_keys, self._client) # Update the claim id and claim expiration info # for all the messages. claims_set_key = utils.scope_claims_set(queue, project, QUEUE_CLAIMS_SUFFIX) with self._client.pipeline() as pipe: pipe.zrem(claims_set_key, claim_id) pipe.delete(claim_id) pipe.delete(claim_msgs_key) for msg in claimed_msgs: if msg: msg.claim_id = None msg.claim_expires = now # TODO(kgriffs): Rather than writing back the # entire message, only set the fields that # have changed. msg.to_redis(pipe) pipe.execute()
def create(self, queue, metadata, project=None, limit=storage.DEFAULT_MESSAGES_PER_CLAIM): claim_ttl = metadata['ttl'] grace = metadata['grace'] now = timeutils.utcnow_ts() msg_ttl = claim_ttl + grace claim_expires = now + claim_ttl msg_expires = claim_expires + grace claim_id = uuidutils.generate_uuid() claimed_msgs = [] # NOTE(kgriffs): Claim some messages msgset_key = utils.msgset_key(queue, project) claimed_ids = self._claim_messages(msgset_key, now, limit, claim_id, claim_expires, msg_ttl, msg_expires) if claimed_ids: claimed_msgs = messages.Message.from_redis_bulk(claimed_ids, self._client) claimed_msgs = [msg.to_basic(now) for msg in claimed_msgs] # NOTE(kgriffs): Perist claim records with self._client.pipeline() as pipe: claim_msgs_key = utils.scope_claim_messages( claim_id, CLAIM_MESSAGES_SUFFIX) for mid in claimed_ids: pipe.rpush(claim_msgs_key, mid) pipe.expire(claim_msgs_key, claim_ttl) claim_info = { 'id': claim_id, 't': claim_ttl, 'e': claim_expires, 'n': len(claimed_ids), } pipe.hmset(claim_id, claim_info) pipe.expire(claim_id, claim_ttl) # NOTE(kgriffs): Add the claim ID to a set so that # existence checks can be performed quickly. This # is also used as a watch key in order to gaurd # against race conditions. # # A sorted set is used to facilitate cleaning # up the IDs of expired claims. claims_set_key = utils.scope_claims_set(queue, project, QUEUE_CLAIMS_SUFFIX) pipe.zadd(claims_set_key, claim_expires, claim_id) pipe.execute() return claim_id, claimed_msgs
def update(self, queue, claim_id, metadata, project=None): if not self._exists(queue, claim_id, project): raise errors.ClaimDoesNotExist(claim_id, queue, project) now = timeutils.utcnow_ts() claim_ttl = metadata['ttl'] claim_expires = now + claim_ttl grace = metadata['grace'] msg_ttl = claim_ttl + grace msg_expires = claim_expires + grace claim_msgs_key = utils.scope_claim_messages(claim_id, CLAIM_MESSAGES_SUFFIX) msg_keys = self._get_claimed_message_keys(claim_msgs_key) claimed_msgs = messages.MessageEnvelope.from_redis_bulk(msg_keys, self._client) claim_info = { 't': claim_ttl, 'e': claim_expires, } with self._client.pipeline() as pipe: for msg in claimed_msgs: if msg: msg.claim_id = claim_id msg.claim_expires = claim_expires if _msg_would_expire(msg, claim_expires): msg.ttl = msg_ttl msg.expires = msg_expires # TODO(kgriffs): Rather than writing back the # entire message, only set the fields that # have changed. # # When this change is made, don't forget to # also call pipe.expire with the new TTL value. msg.to_redis(pipe) # Update the claim id and claim expiration info # for all the messages. pipe.hmset(claim_id, claim_info) pipe.expire(claim_id, claim_ttl) pipe.expire(claim_msgs_key, claim_ttl) claims_set_key = utils.scope_claims_set(queue, project, QUEUE_CLAIMS_SUFFIX) pipe.zadd(claims_set_key, claim_expires, claim_id) pipe.execute()
def _del_message(self, queue, project, claim_id, message_id, pipe): """Called by MessageController when messages are being deleted. This method removes the message from claim data structures. """ claim_msgs_key = utils.scope_claim_messages(claim_id, CLAIM_MESSAGES_SUFFIX) # NOTE(kgriffs): In practice, scanning will be quite fast, # since the usual pattern is to delete messages from oldest # to newest, and the list is sorted in that order. Also, # the length of the list will usually be ~10 messages. pipe.lrem(claim_msgs_key, 1, message_id) # NOTE(kgriffs): Decrement the message counter used for stats pipe.hincrby(claim_id, 'n', -1)
def create(self, queue, metadata, project=None, limit=storage.DEFAULT_MESSAGES_PER_CLAIM): queue_ctrl = self.driver.queue_controller msg_ctrl = self.driver.message_controller claim_ttl = metadata['ttl'] grace = metadata['grace'] now = timeutils.utcnow_ts() msg_ttl = claim_ttl + grace claim_expires = now + claim_ttl msg_expires = claim_expires + grace # Get the maxClaimCount and deadLetterQueue from current queue's meta queue_meta = queue_ctrl.get(queue, project=project) claim_id = uuidutils.generate_uuid() claimed_msgs = [] # NOTE(kgriffs): Claim some messages msgset_key = utils.msgset_key(queue, project) claimed_ids = self._claim_messages(msgset_key, now, limit, claim_id, claim_expires, msg_ttl, msg_expires) if claimed_ids: claimed_msgs = messages.Message.from_redis_bulk(claimed_ids, self._client) claimed_msgs = [msg.to_basic(now) for msg in claimed_msgs] # NOTE(kgriffs): Perist claim records with self._client.pipeline() as pipe: claim_msgs_key = utils.scope_claim_messages( claim_id, CLAIM_MESSAGES_SUFFIX) for mid in claimed_ids: pipe.rpush(claim_msgs_key, mid) pipe.expire(claim_msgs_key, claim_ttl) claim_info = { 'id': claim_id, 't': claim_ttl, 'e': claim_expires, 'n': len(claimed_ids), } pipe.hmset(claim_id, claim_info) pipe.expire(claim_id, claim_ttl) # NOTE(kgriffs): Add the claim ID to a set so that # existence checks can be performed quickly. This # is also used as a watch key in order to guard # against race conditions. # # A sorted set is used to facilitate cleaning # up the IDs of expired claims. claims_set_key = utils.scope_claims_set(queue, project, QUEUE_CLAIMS_SUFFIX) pipe.zadd(claims_set_key, {claim_id: claim_expires}) pipe.execute() if ('_max_claim_count' in queue_meta and '_dead_letter_queue' in queue_meta): claimed_msgs_removed = [] for msg in claimed_msgs: if msg: claimed_count = msg['claim_count'] if claimed_count < queue_meta['_max_claim_count']: # 1. Save the new max claim count for message claim_count = claimed_count + 1 dic = {"c.c": claim_count} pipe.hmset(msg['id'], dic) pipe.execute() else: # 2. Check if the message's claim count has # exceeded the max claim count defined in the # queue, if so, move the message to the dead # letter queue and modify it's ttl. # NOTE(gengchc): We're moving message by # moving the message id from queue to dead # letter queue directly.That means, the queue # and dead letter queue must be created on # the same pool. ddl = utils.scope_queue_name( queue_meta['_dead_letter_queue'], project) ddl_ttl = queue_meta.get( "_dead_letter_queue_messages_ttl") dic = {"t": msg['ttl']} if ddl_ttl: dic = {"t": ddl_ttl} pipe.hmset(msg['id'], dic) queueproject = [s for s in ddl.split('.')] msgs_key_ddl = utils.msgset_key( queueproject[1], queueproject[0]) counter_key_ddl = utils.scope_queue_index( queueproject[1], queueproject[0], MESSAGE_RANK_COUNTER_SUFFIX) msgs_key = utils.msgset_key( queue, project=project) pipe.zrem(msgs_key, msg['id']) message_ids = [] message_ids.append(msg['id']) msg_ctrl._index_messages(msgs_key_ddl, counter_key_ddl, message_ids) pipe.execute() # Add dead letter message to # claimed_msgs_removed, finally remove # them from claimed_msgs. claimed_msgs_removed.append(msg) # Remove dead letter messages from claimed_msgs. for msg_remove in claimed_msgs_removed: claimed_msgs.remove(msg_remove) if len(claimed_msgs) == 0: return None, iter([]) return claim_id, claimed_msgs
def create(self, queue, metadata, project=None, limit=storage.DEFAULT_MESSAGES_PER_CLAIM): queue_ctrl = self.driver.queue_controller msg_ctrl = self.driver.message_controller claim_ttl = metadata['ttl'] grace = metadata['grace'] now = timeutils.utcnow_ts() msg_ttl = claim_ttl + grace claim_expires = now + claim_ttl msg_expires = claim_expires + grace # Get the maxClaimCount and deadLetterQueue from current queue's meta queue_meta = queue_ctrl.get(queue, project=project) claim_id = uuidutils.generate_uuid() claimed_msgs = [] # NOTE(kgriffs): Claim some messages msgset_key = utils.msgset_key(queue, project) claimed_ids = self._claim_messages(msgset_key, now, limit, claim_id, claim_expires, msg_ttl, msg_expires) if claimed_ids: claimed_msgs = messages.Message.from_redis_bulk( claimed_ids, self._client) claimed_msgs = [msg.to_basic(now) for msg in claimed_msgs] # NOTE(kgriffs): Perist claim records with self._client.pipeline() as pipe: claim_msgs_key = utils.scope_claim_messages( claim_id, CLAIM_MESSAGES_SUFFIX) for mid in claimed_ids: pipe.rpush(claim_msgs_key, mid) pipe.expire(claim_msgs_key, claim_ttl) claim_info = { 'id': claim_id, 't': claim_ttl, 'e': claim_expires, 'n': len(claimed_ids), } pipe.hmset(claim_id, claim_info) pipe.expire(claim_id, claim_ttl) # NOTE(kgriffs): Add the claim ID to a set so that # existence checks can be performed quickly. This # is also used as a watch key in order to guard # against race conditions. # # A sorted set is used to facilitate cleaning # up the IDs of expired claims. claims_set_key = utils.scope_claims_set( queue, project, QUEUE_CLAIMS_SUFFIX) pipe.zadd(claims_set_key, claim_expires, claim_id) pipe.execute() if ('_max_claim_count' in queue_meta and '_dead_letter_queue' in queue_meta): claimed_msgs_removed = [] for msg in claimed_msgs: if msg: claimed_count = msg['claim_count'] if claimed_count < queue_meta['_max_claim_count']: # 1. Save the new max claim count for message claim_count = claimed_count + 1 dic = {"c.c": claim_count} pipe.hmset(msg['id'], dic) pipe.execute() else: # 2. Check if the message's claim count has # exceeded the max claim count defined in the # queue, if so, move the message to the dead # letter queue and modify it's ttl. # NOTE(gengchc): We're moving message by # moving the message id from queue to dead # letter queue directly.That means, the queue # and dead letter queue must be created on # the same pool. ddl = utils.scope_queue_name( queue_meta['_dead_letter_queue'], project) ddl_ttl = queue_meta.get( "_dead_letter_queue_messages_ttl") dic = {"t": msg['ttl']} if ddl_ttl: dic = {"t": ddl_ttl} pipe.hmset(msg['id'], dic) queueproject = [s for s in ddl.split('.')] msgs_key_ddl = utils.msgset_key( queueproject[1], queueproject[0]) counter_key_ddl = utils.scope_queue_index( queueproject[1], queueproject[0], MESSAGE_RANK_COUNTER_SUFFIX) msgs_key = utils.msgset_key(queue, project=project) pipe.zrem(msgs_key, msg['id']) message_ids = [] message_ids.append(msg['id']) msg_ctrl._index_messages( msgs_key_ddl, counter_key_ddl, message_ids) pipe.execute() # Add dead letter message to # claimed_msgs_removed, finally remove # them from claimed_msgs. claimed_msgs_removed.append(msg) # Remove dead letter messages from claimed_msgs. for msg_remove in claimed_msgs_removed: claimed_msgs.remove(msg_remove) if len(claimed_msgs) == 0: return None, iter([]) return claim_id, claimed_msgs