Example #1
0
    def delete(self, queue, claim_id, project=None):
        # NOTE(prashanthr_): Return silently when the claim
        # does not exist
        if not self._exists(queue, claim_id, project):
            return

        now = timeutils.utcnow_ts()
        claim_msgs_key = utils.scope_claim_messages(claim_id,
                                                    CLAIM_MESSAGES_SUFFIX)

        msg_keys = self._get_claimed_message_keys(claim_msgs_key)
        claimed_msgs = messages.MessageEnvelope.from_redis_bulk(msg_keys,
                                                                self._client)
        # Update the claim id and claim expiration info
        # for all the messages.
        claims_set_key = utils.scope_claims_set(queue, project,
                                                QUEUE_CLAIMS_SUFFIX)

        with self._client.pipeline() as pipe:
            pipe.zrem(claims_set_key, claim_id)
            pipe.delete(claim_id)
            pipe.delete(claim_msgs_key)

            for msg in claimed_msgs:
                if msg:
                    msg.claim_id = None
                    msg.claim_expires = now

                    # TODO(kgriffs): Rather than writing back the
                    # entire message, only set the fields that
                    # have changed.
                    msg.to_redis(pipe)

            pipe.execute()
Example #2
0
    def _exists(self, queue, claim_id, project):
        client = self._client
        claims_set_key = utils.scope_claims_set(queue, project,
                                                QUEUE_CLAIMS_SUFFIX)
        # In some cases, the queue maybe doesn't exist. So we should check
        # whether the queue exists. Return False if no such queue exists.

        # Todo(flwang): We should delete all related data after the queue is
        # deleted. See the blueprint for more detail:
        # https://blueprints.launchpad.net/zaqar/+spec/clear-resources-after-delete-queue
        if not self._queue_ctrl._exists(queue, project):
            return False

        # Return False if no such claim exists
        # TODO(prashanthr_): Discuss the feasibility of a bloom filter.
        if client.zscore(claims_set_key, claim_id) is None:
            return False

        expires = self._get_claim_info(claim_id, b'e')[0]
        now = timeutils.utcnow_ts()

        if expires <= now:
            # NOTE(kgriffs): Redis should automatically remove the
            # other records in the very near future. This one
            # has to be manually deleted, however.
            client.zrem(claims_set_key, claim_id)
            return False

        return True
Example #3
0
    def _exists(self, queue, claim_id, project):
        client = self._client
        claims_set_key = utils.scope_claims_set(queue, project,
                                                QUEUE_CLAIMS_SUFFIX)
        # In some cases, the queue maybe doesn't exist. So we should check
        # whether the queue exists. Return False if no such queue exists.

        # Todo(flwang): We should delete all related data after the queue is
        # deleted. See the blueprint for more detail:
        # https://blueprints.launchpad.net/zaqar/+spec/clear-resources-after-delete-queue
        if not self._queue_ctrl._exists(queue, project):
            return False

        # Return False if no such claim exists
        # TODO(prashanthr_): Discuss the feasibility of a bloom filter.
        if client.zscore(claims_set_key, claim_id) is None:
            return False

        expires = self._get_claim_info(claim_id, b'e')[0]
        now = timeutils.utcnow_ts()

        if expires <= now:
            # NOTE(kgriffs): Redis should automatically remove the
            # other records in the very near future. This one
            # has to be manually deleted, however.
            client.zrem(claims_set_key, claim_id)
            return False

        return True
Example #4
0
    def create(self, queue, metadata, project=None,
               limit=storage.DEFAULT_MESSAGES_PER_CLAIM):

        claim_ttl = metadata['ttl']
        grace = metadata['grace']

        now = timeutils.utcnow_ts()
        msg_ttl = claim_ttl + grace
        claim_expires = now + claim_ttl
        msg_expires = claim_expires + grace

        claim_id = uuidutils.generate_uuid()
        claimed_msgs = []

        # NOTE(kgriffs): Claim some messages
        msgset_key = utils.msgset_key(queue, project)
        claimed_ids = self._claim_messages(msgset_key, now, limit,
                                           claim_id, claim_expires,
                                           msg_ttl, msg_expires)

        if claimed_ids:
            claimed_msgs = messages.Message.from_redis_bulk(claimed_ids,
                                                            self._client)
            claimed_msgs = [msg.to_basic(now) for msg in claimed_msgs]

            # NOTE(kgriffs): Perist claim records
            with self._client.pipeline() as pipe:
                claim_msgs_key = utils.scope_claim_messages(
                    claim_id, CLAIM_MESSAGES_SUFFIX)

                for mid in claimed_ids:
                    pipe.rpush(claim_msgs_key, mid)

                pipe.expire(claim_msgs_key, claim_ttl)

                claim_info = {
                    'id': claim_id,
                    't': claim_ttl,
                    'e': claim_expires,
                    'n': len(claimed_ids),
                }

                pipe.hmset(claim_id, claim_info)
                pipe.expire(claim_id, claim_ttl)

                # NOTE(kgriffs): Add the claim ID to a set so that
                # existence checks can be performed quickly. This
                # is also used as a watch key in order to gaurd
                # against race conditions.
                #
                # A sorted set is used to facilitate cleaning
                # up the IDs of expired claims.
                claims_set_key = utils.scope_claims_set(queue, project,
                                                        QUEUE_CLAIMS_SUFFIX)

                pipe.zadd(claims_set_key, claim_expires, claim_id)
                pipe.execute()

        return claim_id, claimed_msgs
Example #5
0
    def update(self, queue, claim_id, metadata, project=None):
        if not self._exists(queue, claim_id, project):
            raise errors.ClaimDoesNotExist(claim_id, queue, project)

        now = timeutils.utcnow_ts()

        claim_ttl = metadata['ttl']
        claim_expires = now + claim_ttl

        grace = metadata['grace']
        msg_ttl = claim_ttl + grace
        msg_expires = claim_expires + grace

        claim_msgs_key = utils.scope_claim_messages(claim_id,
                                                    CLAIM_MESSAGES_SUFFIX)

        msg_keys = self._get_claimed_message_keys(claim_msgs_key)
        claimed_msgs = messages.MessageEnvelope.from_redis_bulk(msg_keys,
                                                                self._client)
        claim_info = {
            't': claim_ttl,
            'e': claim_expires,
        }

        with self._client.pipeline() as pipe:
            for msg in claimed_msgs:
                if msg:
                    msg.claim_id = claim_id
                    msg.claim_expires = claim_expires

                    if _msg_would_expire(msg, claim_expires):
                        msg.ttl = msg_ttl
                        msg.expires = msg_expires

                    # TODO(kgriffs): Rather than writing back the
                    # entire message, only set the fields that
                    # have changed.
                    #
                    # When this change is made, don't forget to
                    # also call pipe.expire with the new TTL value.
                    msg.to_redis(pipe)

            # Update the claim id and claim expiration info
            # for all the messages.
            pipe.hmset(claim_id, claim_info)
            pipe.expire(claim_id, claim_ttl)

            pipe.expire(claim_msgs_key, claim_ttl)

            claims_set_key = utils.scope_claims_set(queue, project,
                                                    QUEUE_CLAIMS_SUFFIX)

            pipe.zadd(claims_set_key, claim_expires, claim_id)

            pipe.execute()
Example #6
0
    def _gc(self, queue, project):
        """Garbage-collect expired claim data.

        Not all claim data can be automatically expired. This method
        cleans up the remainder.

        :returns: Number of claims removed
        """

        claims_set_key = utils.scope_claims_set(queue, project,
                                                QUEUE_CLAIMS_SUFFIX)
        now = timeutils.utcnow_ts()
        num_removed = self._client.zremrangebyscore(claims_set_key, 0, now)
        return num_removed
Example #7
0
    def _exists(self, queue, claim_id, project):
        client = self._client
        claims_set_key = utils.scope_claims_set(queue, project,
                                                QUEUE_CLAIMS_SUFFIX)

        # Return False if no such claim exists
        # TODO(prashanthr_): Discuss the feasibility of a bloom filter.
        if client.zscore(claims_set_key, claim_id) is None:
            return False

        expires = self._get_claim_info(claim_id, b'e')[0]
        now = timeutils.utcnow_ts()

        if expires <= now:
            # NOTE(kgriffs): Redis should automatically remove the
            # other records in the very near future. This one
            # has to be manually deleted, however.
            client.zrem(claims_set_key, claim_id)
            return False

        return True
Example #8
0
    def _count_messages(self, queue, project):
        """Count and return the total number of claimed messages."""

        # NOTE(kgriffs): Iterate through all claims, adding up the
        # number of messages per claim. This is obviously slower
        # than keeping a side counter, but is also less error-prone.
        # Plus, it avoids having to do a lot of extra work during
        # garbage collection passes. Also, considering that most
        # workloads won't require a large number of claims, most of
        # the time we can do this in a single pass, so it is still
        # pretty fast.

        claims_set_key = utils.scope_claims_set(queue, project,
                                                QUEUE_CLAIMS_SUFFIX)
        num_claimed = 0
        offset = 0

        while True:
            claim_ids = self._client.zrange(claims_set_key, offset,
                                            offset + COUNTING_BATCH_SIZE - 1)
            if not claim_ids:
                break

            offset += len(claim_ids)

            with self._client.pipeline() as pipe:
                for cid in claim_ids:
                    pipe.hmget(cid, 'n')

                claim_infos = pipe.execute()

            for info in claim_infos:
                # NOTE(kgriffs): In case the claim was deleted out
                # from under us, sanity-check that we got a non-None
                # info list.
                if info:
                    num_claimed += int(info[0])

        return num_claimed
Example #9
0
    def create(self, queue, metadata, project=None,
               limit=storage.DEFAULT_MESSAGES_PER_CLAIM):

        queue_ctrl = self.driver.queue_controller
        msg_ctrl = self.driver.message_controller

        claim_ttl = metadata['ttl']
        grace = metadata['grace']

        now = timeutils.utcnow_ts()
        msg_ttl = claim_ttl + grace
        claim_expires = now + claim_ttl
        msg_expires = claim_expires + grace

        # Get the maxClaimCount and deadLetterQueue from current queue's meta
        queue_meta = queue_ctrl.get(queue, project=project)

        claim_id = uuidutils.generate_uuid()
        claimed_msgs = []

        # NOTE(kgriffs): Claim some messages
        msgset_key = utils.msgset_key(queue, project)
        claimed_ids = self._claim_messages(msgset_key, now, limit,
                                           claim_id, claim_expires,
                                           msg_ttl, msg_expires)

        if claimed_ids:
            claimed_msgs = messages.Message.from_redis_bulk(claimed_ids,
                                                            self._client)
            claimed_msgs = [msg.to_basic(now) for msg in claimed_msgs]

            # NOTE(kgriffs): Perist claim records
            with self._client.pipeline() as pipe:
                claim_msgs_key = utils.scope_claim_messages(
                    claim_id, CLAIM_MESSAGES_SUFFIX)

                for mid in claimed_ids:
                    pipe.rpush(claim_msgs_key, mid)

                pipe.expire(claim_msgs_key, claim_ttl)

                claim_info = {
                    'id': claim_id,
                    't': claim_ttl,
                    'e': claim_expires,
                    'n': len(claimed_ids),
                }

                pipe.hmset(claim_id, claim_info)
                pipe.expire(claim_id, claim_ttl)

                # NOTE(kgriffs): Add the claim ID to a set so that
                # existence checks can be performed quickly. This
                # is also used as a watch key in order to guard
                # against race conditions.
                #
                # A sorted set is used to facilitate cleaning
                # up the IDs of expired claims.
                claims_set_key = utils.scope_claims_set(queue, project,
                                                        QUEUE_CLAIMS_SUFFIX)

                pipe.zadd(claims_set_key, {claim_id: claim_expires})
                pipe.execute()

                if ('_max_claim_count' in queue_meta and
                        '_dead_letter_queue' in queue_meta):
                    claimed_msgs_removed = []
                    for msg in claimed_msgs:
                        if msg:
                            claimed_count = msg['claim_count']
                            if claimed_count < queue_meta['_max_claim_count']:
                                # 1. Save the new max claim count for message
                                claim_count = claimed_count + 1
                                dic = {"c.c": claim_count}
                                pipe.hmset(msg['id'], dic)
                                pipe.execute()
                            else:
                                # 2. Check if the message's claim count has
                                # exceeded the max claim count defined in the
                                # queue, if so, move the message to the dead
                                # letter queue and modify it's ttl.
                                # NOTE(gengchc):  We're moving message by
                                # moving the message id from queue to dead
                                # letter queue directly.That means, the queue
                                # and dead letter queue must be created on
                                # the same pool.
                                ddl = utils.scope_queue_name(
                                    queue_meta['_dead_letter_queue'], project)
                                ddl_ttl = queue_meta.get(
                                    "_dead_letter_queue_messages_ttl")
                                dic = {"t": msg['ttl']}
                                if ddl_ttl:
                                    dic = {"t": ddl_ttl}
                                pipe.hmset(msg['id'], dic)
                                queueproject = [s for s in ddl.split('.')]
                                msgs_key_ddl = utils.msgset_key(
                                    queueproject[1], queueproject[0])
                                counter_key_ddl = utils.scope_queue_index(
                                    queueproject[1], queueproject[0],
                                    MESSAGE_RANK_COUNTER_SUFFIX)
                                msgs_key = utils.msgset_key(
                                    queue, project=project)
                                pipe.zrem(msgs_key, msg['id'])
                                message_ids = []
                                message_ids.append(msg['id'])
                                msg_ctrl._index_messages(msgs_key_ddl,
                                                         counter_key_ddl,
                                                         message_ids)
                                pipe.execute()
                                # Add dead letter message to
                                # claimed_msgs_removed, finally remove
                                # them from claimed_msgs.
                                claimed_msgs_removed.append(msg)

                    # Remove dead letter messages from claimed_msgs.
                    for msg_remove in claimed_msgs_removed:
                        claimed_msgs.remove(msg_remove)
                    if len(claimed_msgs) == 0:
                        return None, iter([])

        return claim_id, claimed_msgs
Example #10
0
    def create(self,
               queue,
               metadata,
               project=None,
               limit=storage.DEFAULT_MESSAGES_PER_CLAIM):

        queue_ctrl = self.driver.queue_controller
        msg_ctrl = self.driver.message_controller

        claim_ttl = metadata['ttl']
        grace = metadata['grace']

        now = timeutils.utcnow_ts()
        msg_ttl = claim_ttl + grace
        claim_expires = now + claim_ttl
        msg_expires = claim_expires + grace

        # Get the maxClaimCount and deadLetterQueue from current queue's meta
        queue_meta = queue_ctrl.get(queue, project=project)

        claim_id = uuidutils.generate_uuid()
        claimed_msgs = []

        # NOTE(kgriffs): Claim some messages
        msgset_key = utils.msgset_key(queue, project)
        claimed_ids = self._claim_messages(msgset_key, now, limit, claim_id,
                                           claim_expires, msg_ttl, msg_expires)

        if claimed_ids:
            claimed_msgs = messages.Message.from_redis_bulk(
                claimed_ids, self._client)
            claimed_msgs = [msg.to_basic(now) for msg in claimed_msgs]

            # NOTE(kgriffs): Perist claim records
            with self._client.pipeline() as pipe:
                claim_msgs_key = utils.scope_claim_messages(
                    claim_id, CLAIM_MESSAGES_SUFFIX)

                for mid in claimed_ids:
                    pipe.rpush(claim_msgs_key, mid)

                pipe.expire(claim_msgs_key, claim_ttl)

                claim_info = {
                    'id': claim_id,
                    't': claim_ttl,
                    'e': claim_expires,
                    'n': len(claimed_ids),
                }

                pipe.hmset(claim_id, claim_info)
                pipe.expire(claim_id, claim_ttl)

                # NOTE(kgriffs): Add the claim ID to a set so that
                # existence checks can be performed quickly. This
                # is also used as a watch key in order to guard
                # against race conditions.
                #
                # A sorted set is used to facilitate cleaning
                # up the IDs of expired claims.
                claims_set_key = utils.scope_claims_set(
                    queue, project, QUEUE_CLAIMS_SUFFIX)

                pipe.zadd(claims_set_key, claim_expires, claim_id)
                pipe.execute()

                if ('_max_claim_count' in queue_meta
                        and '_dead_letter_queue' in queue_meta):
                    claimed_msgs_removed = []
                    for msg in claimed_msgs:
                        if msg:
                            claimed_count = msg['claim_count']
                            if claimed_count < queue_meta['_max_claim_count']:
                                # 1. Save the new max claim count for message
                                claim_count = claimed_count + 1
                                dic = {"c.c": claim_count}
                                pipe.hmset(msg['id'], dic)
                                pipe.execute()
                            else:
                                # 2. Check if the message's claim count has
                                # exceeded the max claim count defined in the
                                # queue, if so, move the message to the dead
                                # letter queue and modify it's ttl.
                                # NOTE(gengchc):  We're moving message by
                                # moving the message id from queue to dead
                                # letter queue directly.That means, the queue
                                # and dead letter queue must be created on
                                # the same pool.
                                ddl = utils.scope_queue_name(
                                    queue_meta['_dead_letter_queue'], project)
                                ddl_ttl = queue_meta.get(
                                    "_dead_letter_queue_messages_ttl")
                                dic = {"t": msg['ttl']}
                                if ddl_ttl:
                                    dic = {"t": ddl_ttl}
                                pipe.hmset(msg['id'], dic)
                                queueproject = [s for s in ddl.split('.')]
                                msgs_key_ddl = utils.msgset_key(
                                    queueproject[1], queueproject[0])
                                counter_key_ddl = utils.scope_queue_index(
                                    queueproject[1], queueproject[0],
                                    MESSAGE_RANK_COUNTER_SUFFIX)
                                msgs_key = utils.msgset_key(queue,
                                                            project=project)
                                pipe.zrem(msgs_key, msg['id'])
                                message_ids = []
                                message_ids.append(msg['id'])
                                msg_ctrl._index_messages(
                                    msgs_key_ddl, counter_key_ddl, message_ids)
                                pipe.execute()
                                # Add dead letter message to
                                # claimed_msgs_removed, finally remove
                                # them from claimed_msgs.
                                claimed_msgs_removed.append(msg)

                    # Remove dead letter messages from claimed_msgs.
                    for msg_remove in claimed_msgs_removed:
                        claimed_msgs.remove(msg_remove)
                    if len(claimed_msgs) == 0:
                        return None, iter([])

        return claim_id, claimed_msgs