Exemplo n.º 1
0
    def get(self, queue, claim_id, project=None):
        if not self._exists(queue, claim_id, project):
            raise errors.ClaimDoesNotExist(queue, project, claim_id)

        claim_msgs_key = utils.scope_claim_messages(claim_id,
                                                    CLAIM_MESSAGES_SUFFIX)

        # basic_messages
        msg_keys = self._get_claimed_message_keys(claim_msgs_key)
        claimed_msgs = messages.Message.from_redis_bulk(msg_keys,
                                                        self._client)
        now = timeutils.utcnow_ts()
        basic_messages = [msg.to_basic(now)
                          for msg in claimed_msgs if msg]

        # claim_meta
        now = timeutils.utcnow_ts()
        expires, ttl = self._get_claim_info(claim_id, [b'e', b't'])
        update_time = expires - ttl
        age = now - update_time

        claim_meta = {
            'age': age,
            'ttl': ttl,
            'id': claim_id,
        }

        return claim_meta, basic_messages
Exemplo n.º 2
0
    def _claimed(self, queue_name, claim_id,
                 expires=None, limit=None, project=None):

        if claim_id is None:
            claim_id = {'$ne': None}

        query = {
            PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
            'c.id': claim_id,
            'c.e': {'$gt': expires or timeutils.utcnow_ts()},
        }

        # NOTE(kgriffs): Claimed messages bust be queried from
        # the primary to avoid a race condition caused by the
        # multi-phased "create claim" algorithm.
        preference = pymongo.read_preferences.ReadPreference.PRIMARY
        collection = self._collection(queue_name, project)
        msgs = collection.find(query, sort=[('k', 1)],
                               read_preference=preference).hint(
                                   CLAIMED_INDEX_FIELDS)

        if limit is not None:
            msgs = msgs.limit(limit)

        now = timeutils.utcnow_ts()

        def denormalizer(msg):
            doc = _basic_message(msg, now)
            doc['claim'] = msg['c']

            return doc

        return utils.HookedCursor(msgs, denormalizer)
Exemplo n.º 3
0
    def test_set_time_override_using_default(self):
        now = timeutils.utcnow_ts()

        # NOTE(kgriffs): Normally it's bad form to sleep in a unit test,
        # but this is the only way to test that set_time_override defaults
        # to setting the override to the current time.
        time.sleep(1)

        timeutils.set_time_override()
        overriden_now = timeutils.utcnow_ts()
        self.assertThat(now, matchers.LessThan(overriden_now))
Exemplo n.º 4
0
    def test_utcnow_ts_microsecond(self):
        skynet_self_aware_ts = 872835240.000123
        skynet_dt = datetime.datetime.utcfromtimestamp(skynet_self_aware_ts)
        self.assertEqual(self.skynet_self_aware_ms_time, skynet_dt)

        # NOTE(kgriffs): timeutils.utcnow_ts() uses time.time()
        # IFF time override is not set.
        with mock.patch('time.time') as time_mock:
            time_mock.return_value = skynet_self_aware_ts
            ts = timeutils.utcnow_ts(microsecond=True)
            self.assertEqual(ts, skynet_self_aware_ts)

        timeutils.set_time_override(skynet_dt)
        ts = timeutils.utcnow_ts(microsecond=True)
        self.assertEqual(ts, skynet_self_aware_ts)
Exemplo n.º 5
0
Arquivo: queues.py Projeto: rose/zaqar
    def _create(self, name, metadata=None, project=None):
        # TODO(prashanthr_): Implement as a lua script.
        queue_key = utils.scope_queue_name(name, project)
        qset_key = utils.scope_queue_name(QUEUES_SET_STORE_NAME, project)

        # Check if the queue already exists.
        if self.exists(name, project):
            return False

        queue = {
            'c': 0,
            'cl': 0,
            'm': self._packer(metadata or {}),
            't': timeutils.utcnow_ts()
        }

        # Pipeline ensures atomic inserts.
        with self._client.pipeline() as pipe:
            pipe.zadd(qset_key, 1, queue_key).hmset(queue_key, queue)
            self._message_ctrl._create_msgset(name, project, pipe)

            try:
                pipe.execute()
            except redis.exceptions.ResponseError:
                return False

        return True
Exemplo n.º 6
0
 def set(self, key, value, time=0, min_compress_len=0):
     """Sets the value for a key."""
     timeout = 0
     if time != 0:
         timeout = timeutils.utcnow_ts() + time
     self.cache[key] = (timeout, value)
     return True
Exemplo n.º 7
0
    def test_gc(self):
        self.queue_ctrl.create(self.queue_name)

        for _ in range(100):
            self.message_ctrl.post(self.queue_name,
                                   [{'ttl': 300, 'body': 'yo gabba'}],
                                   client_uuid=str(uuid.uuid4()))

        now = timeutils.utcnow_ts()
        timeutils_utcnow = 'oslo.utils.timeutils.utcnow_ts'

        # Test a single claim
        with mock.patch(timeutils_utcnow) as mock_utcnow:
            mock_utcnow.return_value = now - 1
            self.controller.create(self.queue_name, {'ttl': 1, 'grace': 60})

        num_removed = self.controller._gc(self.queue_name, None)
        self.assertEqual(num_removed, 1)

        # Test multiple claims
        with mock.patch(timeutils_utcnow) as mock_utcnow:
            mock_utcnow.return_value = now - 1

            for _ in range(5):
                self.controller.create(self.queue_name,
                                       {'ttl': 1, 'grace': 60})

        # NOTE(kgriffs): These ones should not be cleaned up
        self.controller.create(self.queue_name, {'ttl': 60, 'grace': 60})
        self.controller.create(self.queue_name, {'ttl': 60, 'grace': 60})

        num_removed = self.controller._gc(self.queue_name, None)
        self.assertEqual(num_removed, 5)
Exemplo n.º 8
0
    def delete(self, queue, claim_id, project=None):
        # NOTE(prashanthr_): Return silently when the claim
        # does not exist
        if not self._exists(queue, claim_id, project):
            return

        now = timeutils.utcnow_ts()
        claim_msgs_key = utils.scope_claim_messages(claim_id,
                                                    CLAIM_MESSAGES_SUFFIX)

        msg_keys = self._get_claimed_message_keys(claim_msgs_key)
        claimed_msgs = messages.MessageEnvelope.from_redis_bulk(msg_keys,
                                                                self._client)
        # Update the claim id and claim expiration info
        # for all the messages.
        claims_set_key = utils.scope_claims_set(queue, project,
                                                QUEUE_CLAIMS_SUFFIX)

        with self._client.pipeline() as pipe:
            pipe.zrem(claims_set_key, claim_id)
            pipe.delete(claim_id)
            pipe.delete(claim_msgs_key)

            for msg in claimed_msgs:
                if msg:
                    msg.claim_id = None
                    msg.claim_expires = now

                    # TODO(kgriffs): Rather than writing back the
                    # entire message, only set the fields that
                    # have changed.
                    msg.to_redis(pipe)

            pipe.execute()
Exemplo n.º 9
0
    def _count(self, queue_name, project=None, include_claimed=False):
        """Return total number of messages in a queue.

        This method is designed to very quickly count the number
        of messages in a given queue. Expired messages are not
        counted, of course. If the queue does not exist, the
        count will always be 0.

        Note: Some expired messages may be included in the count if
            they haven't been GC'd yet. This is done for performance.
        """
        query = {
            # Messages must belong to this queue and project.
            PROJ_QUEUE: utils.scope_queue_name(queue_name, project),

            # NOTE(kgriffs): Messages must be finalized (i.e., must not
            # be part of an unfinalized transaction).
            #
            # See also the note wrt 'tx' within the definition
            # of ACTIVE_INDEX_FIELDS.
            'tx': None,
        }

        if not include_claimed:
            # Exclude messages that are claimed
            query['c.e'] = {'$lte': timeutils.utcnow_ts()}

        collection = self._collection(queue_name, project)
        return collection.find(query).hint(COUNTING_INDEX_FIELDS).count()
Exemplo n.º 10
0
def _filter_messages(messages, filters, to_basic, marker):
    """Create a filtering iterator over a list of messages.

    The function accepts a list of filters to be filtered
    before the the message can be included as a part of the reply.
    """
    now = timeutils.utcnow_ts()

    for msg in messages:
        # NOTE(kgriffs): Message may have been deleted, so
        # check each value to ensure we got a message back
        if msg is None:
            continue

        # NOTE(kgriffs): Check to see if any of the filters
        # indiciate that this message should be skipped.
        for should_skip in filters:
            if should_skip(msg):
                break
        else:
            marker['next'] = msg.id

            if to_basic:
                yield msg.to_basic(now)
            else:
                yield msg
Exemplo n.º 11
0
    def list(self, queue_name, project=None, marker=None,
             limit=storage.DEFAULT_MESSAGES_PER_PAGE,
             echo=False, client_uuid=None, include_claimed=False):

        if marker is not None:
            try:
                marker = int(marker)
            except ValueError:
                yield iter([])

        messages = self._list(queue_name, project=project, marker=marker,
                              client_uuid=client_uuid, echo=echo,
                              include_claimed=include_claimed, limit=limit)

        marker_id = {}

        now = timeutils.utcnow_ts()

        # NOTE (kgriffs) @utils.raises_conn_error not needed on this
        # function, since utils.HookedCursor already has it.
        def denormalizer(msg):
            marker_id['next'] = msg['k']

            return _basic_message(msg, now)

        yield utils.HookedCursor(messages, denormalizer)
        yield str(marker_id['next'])
Exemplo n.º 12
0
 def _exists_unlocked(self, key):
     now = timeutils.utcnow_ts()
     try:
         timeout = self._cache[key][0]
         return not timeout or now <= timeout
     except KeyError:
         return False
Exemplo n.º 13
0
    def track_instance(instance_id, environment_id, instance_type,
                       type_name, type_title=None, unit_count=None):

        unit = db_session.get_session()
        try:
            with unit.begin():
                env = unit.query(models.Environment).get(environment_id)
                instance = models.Instance()
                instance.instance_id = instance_id
                instance.environment_id = environment_id
                instance.tenant_id = env.tenant_id
                instance.instance_type = instance_type
                instance.created = timeutils.utcnow_ts()
                instance.destroyed = None
                instance.type_name = type_name
                instance.type_title = type_title
                instance.unit_count = unit_count

                unit.add(instance)
        except exception.DBDuplicateEntry:
            unit.execute(
                sqlalchemy.update(models.Instance).where(
                    models.Instance.instance_id == instance_id and
                    models.Instance.environment_id == environment_id).values(
                        unit_count=unit_count))
Exemplo n.º 14
0
 def destroy_instance(instance_id, environment_id):
     unit = db_session.get_session()
     instance = unit.query(models.Instance).get(
         (environment_id, instance_id))
     if instance and not instance.destroyed:
         instance.destroyed = timeutils.utcnow_ts()
         instance.save(unit)
Exemplo n.º 15
0
 def put(self, key, value, time=CACHE_TIMEOUT):
     """Sets the value for a key."""
     timeout = 0
     if time != 0:
         timeout = timeutils.utcnow_ts() + time
     self._cache[key] = (timeout, value)
     return True
Exemplo n.º 16
0
    def _get_claim(self, message_id):
        """Gets minimal claim doc for a message.

        :returns: {'id': cid, 'expires': ts} IFF the message is claimed,
            and that claim has not expired.
        """

        claim = self._client.hmget(message_id, 'c', 'c.e')

        if claim == [None, None]:
            # NOTE(kgriffs): message_id was not found
            return None

        info = {
            # NOTE(kgriffs): A "None" claim is serialized as an empty str
            'id': encodeutils.safe_decode(claim[0]) or None,
            'expires': int(claim[1]),
        }

        # Is the message claimed?
        now = timeutils.utcnow_ts()
        if info['id'] and (now < info['expires']):
            return info

        # Not claimed
        return None
Exemplo n.º 17
0
    def _get_unlocked(self, key, default=None):
        now = timeutils.utcnow_ts()

        try:
            timeout, value = self._cache[key]
        except KeyError:
            return (0, default)

        if timeout and now >= timeout:

            # NOTE(flaper87): Record expired,
            # remove it from the cache but catch
            # KeyError and ValueError in case
            # _purge_expired removed this key already.
            try:
                del self._cache[key]
            except KeyError:
                pass

            try:
                # NOTE(flaper87): Keys with ttl == 0
                # don't exist in the _keys_expires dict
                self._keys_expires[timeout].remove(key)
            except (KeyError, ValueError):
                pass

            return (0, default)

        return (timeout, value)
Exemplo n.º 18
0
    def post(self, queue, messages, client_uuid, project=None):
        if not self._queue_ctrl.exists(queue, project):
            raise errors.QueueDoesNotExist(queue, project)

        msgset_key = utils.msgset_key(queue, project)
        counter_key = utils.scope_queue_index(queue, project,
                                              MESSAGE_RANK_COUNTER_SUFFIX)

        message_ids = []
        now = timeutils.utcnow_ts()

        with self._client.pipeline() as pipe:
            for msg in messages:
                prepared_msg = Message(
                    ttl=msg['ttl'],
                    created=now,
                    client_uuid=client_uuid,
                    claim_id=None,
                    claim_expires=now,
                    body=msg.get('body', {}),
                )

                prepared_msg.to_redis(pipe)
                message_ids.append(prepared_msg.id)

            pipe.execute()

        # NOTE(kgriffs): If this call fails, we will return
        # an error to the client and the messages will be
        # orphaned, but Redis will remove them when they
        # expire, so we will just pretend they don't exist
        # in that case.
        self._index_messages(msgset_key, counter_key, message_ids)

        return message_ids
Exemplo n.º 19
0
    def bulk_get(self, queue, message_ids, project):
        if project is None:
            project = ''

        message_ids = [id for id in
                       map(utils.msgid_decode, message_ids)
                       if id is not None]

        statement = sa.sql.select([tables.Messages.c.id,
                                   tables.Messages.c.body,
                                   tables.Messages.c.ttl,
                                   tables.Messages.c.created,
                                   tables.Messages.c.cid])

        and_stmt = [tables.Messages.c.id.in_(message_ids)]
        and_stmt.extend(self._and_stmt_with_ttl(queue, project))

        j = sa.join(tables.Messages, tables.Queues,
                    tables.Messages.c.qid == tables.Queues.c.id)

        statement = statement.select_from(j).where(sa.and_(*and_stmt))

        now = timeutils.utcnow_ts()
        records = self.driver.run(statement)
        for id, body, ttl, created, cid in records:
            yield {
                'id': utils.msgid_encode(int(id)),
                'ttl': ttl,
                'age': now - calendar.timegm(created.timetuple()),
                'body': utils.json_decode(body),
                'claim_id': utils.cid_encode(cid) if cid else None,
            }
Exemplo n.º 20
0
    def stats(self, name, project=None):
        if not self.exists(name, project=project):
            raise errors.QueueDoesNotExist(name, project)

        controller = self.driver.message_controller

        active = controller._count(name, project=project,
                                   include_claimed=False)
        total = controller._count(name, project=project,
                                  include_claimed=True)

        message_stats = {
            'claimed': total - active,
            'free': active,
            'total': total,
        }

        try:
            oldest = controller.first(name, project=project, sort=1)
            newest = controller.first(name, project=project, sort=-1)
        except errors.QueueIsEmpty:
            pass
        else:
            now = timeutils.utcnow_ts()
            message_stats['oldest'] = utils.stat_message(oldest, now)
            message_stats['newest'] = utils.stat_message(newest, now)

        return {'messages': message_stats}
Exemplo n.º 21
0
    def create(self, queue, metadata, project=None,
               limit=storage.DEFAULT_MESSAGES_PER_CLAIM):

        claim_ttl = int(metadata.get('ttl', 60))
        grace = int(metadata.get('grace', 60))

        now = timeutils.utcnow_ts()
        msg_ttl = claim_ttl + grace
        claim_expires = now + claim_ttl
        msg_expires = claim_expires + grace

        claim_id = utils.generate_uuid()
        claimed_msgs = []

        # NOTE(kgriffs): Claim some messages
        msgset_key = utils.msgset_key(queue, project)
        claimed_ids = self._claim_messages(msgset_key, now, limit,
                                           claim_id, claim_expires,
                                           msg_ttl, msg_expires)

        if claimed_ids:
            claimed_msgs = messages.Message.from_redis_bulk(claimed_ids,
                                                            self._client)
            claimed_msgs = [msg.to_basic(now) for msg in claimed_msgs]

            # NOTE(kgriffs): Perist claim records
            with self._client.pipeline() as pipe:
                claim_msgs_key = utils.scope_claim_messages(
                    claim_id, CLAIM_MESSAGES_SUFFIX)

                for mid in claimed_ids:
                    pipe.rpush(claim_msgs_key, mid)

                pipe.expire(claim_msgs_key, claim_ttl)

                claim_info = {
                    'id': claim_id,
                    't': claim_ttl,
                    'e': claim_expires,
                    'n': len(claimed_ids),
                }

                pipe.hmset(claim_id, claim_info)
                pipe.expire(claim_id, claim_ttl)

                # NOTE(kgriffs): Add the claim ID to a set so that
                # existence checks can be performed quickly. This
                # is also used as a watch key in order to gaurd
                # against race conditions.
                #
                # A sorted set is used to facilitate cleaning
                # up the IDs of expired claims.
                claims_set_key = utils.scope_claims_set(queue, project,
                                                        QUEUE_CLAIMS_SUFFIX)

                pipe.zadd(claims_set_key, claim_expires, claim_id)
                pipe.execute()

        return claim_id, claimed_msgs
Exemplo n.º 22
0
def increment_serial(serial=0):
    # This provides for *roughly* unix timestamp based serial numbers
    new_serial = timeutils.utcnow_ts()

    if new_serial <= serial:
        new_serial = serial + 1

    return new_serial
Exemplo n.º 23
0
    def _set_unlocked(self, key, value, ttl=0):
        expires_at = 0
        if ttl != 0:
            expires_at = timeutils.utcnow_ts() + ttl

        self._cache[key] = (expires_at, value)

        if expires_at:
            self._keys_expires[expires_at].add(key)
Exemplo n.º 24
0
    def get(self, key):
        """Retrieves the value for a key or None."""
        now = timeutils.utcnow_ts()
        for k in list(self._cache):
            (timeout, _value) = self._cache[k]
            if timeout and now >= timeout:
                del self._cache[k]

        return self._cache.get(key, (0, None))[1]
Exemplo n.º 25
0
 def get(self, queue, message_id, project):
     body, ttl, created, cid = self._get(queue, message_id, project)
     now = timeutils.utcnow_ts()
     return {
         'id': message_id,
         'ttl': ttl,
         'age': now - calendar.timegm(created.timetuple()),
         'body': utils.json_decode(body),
         'claim_id': utils.cid_encode(cid) if cid else None,
     }
Exemplo n.º 26
0
    def update(self, queue, claim_id, metadata, project=None):
        if not self._exists(queue, claim_id, project):
            raise errors.ClaimDoesNotExist(claim_id, queue, project)

        now = timeutils.utcnow_ts()

        claim_ttl = int(metadata.get('ttl', 60))
        claim_expires = now + claim_ttl

        grace = int(metadata.get('grace', 60))
        msg_ttl = claim_ttl + grace
        msg_expires = claim_expires + grace

        claim_msgs_key = utils.scope_claim_messages(claim_id,
                                                    CLAIM_MESSAGES_SUFFIX)

        msg_keys = self._get_claimed_message_keys(claim_msgs_key)
        claimed_msgs = messages.MessageEnvelope.from_redis_bulk(msg_keys,
                                                                self._client)
        claim_info = {
            't': claim_ttl,
            'e': claim_expires,
        }

        with self._client.pipeline() as pipe:
            for msg in claimed_msgs:
                if msg:
                    msg.claim_id = claim_id
                    msg.claim_expires = claim_expires

                    if _msg_would_expire(msg, msg_expires):
                        msg.ttl = msg_ttl
                        msg.expires = msg_expires

                    # TODO(kgriffs): Rather than writing back the
                    # entire message, only set the fields that
                    # have changed.
                    #
                    # When this change is made, don't forget to
                    # also call pipe.expire with the new TTL value.
                    msg.to_redis(pipe)

            # Update the claim id and claim expiration info
            # for all the messages.
            pipe.hmset(claim_id, claim_info)
            pipe.expire(claim_id, claim_ttl)

            pipe.expire(claim_msgs_key, claim_ttl)

            claims_set_key = utils.scope_claims_set(queue, project,
                                                    QUEUE_CLAIMS_SUFFIX)

            pipe.zadd(claims_set_key, claim_expires, claim_id)

            pipe.execute()
Exemplo n.º 27
0
 def it():
     now = timeutils.utcnow_ts()
     for id, body, ttl, created, cid in records:
         marker_id['next'] = id
         yield {
             'id': utils.msgid_encode(id),
             'ttl': ttl,
             'age': now - calendar.timegm(created.timetuple()),
             'body': utils.json_decode(body),
             'claim_id': utils.cid_encode(cid) if cid else None,
         }
Exemplo n.º 28
0
    def first(self, queue_name, project=None, sort=1):
        cursor = self._list(queue_name, project=project,
                            include_claimed=True, sort=sort,
                            limit=1)
        try:
            message = next(cursor)
        except StopIteration:
            raise errors.QueueIsEmpty(queue_name, project)

        now = timeutils.utcnow_ts()
        return _basic_message(message, now)
Exemplo n.º 29
0
    def first(self, queue_name, project=None, sort=1):
        cursor = self._list(queue_name, project=project,
                            include_claimed=True, sort=sort,
                            limit=1)
        try:
            message = next(cursor)
        except StopIteration:
            raise errors.QueueIsEmpty(queue_name, project)

        now = timeutils.utcnow_ts()
        return _basic_message(message, now)
Exemplo n.º 30
0
    def get(self, queue, message_id, project=None):
        if not self._queue_ctrl.exists(queue, project):
            raise errors.QueueDoesNotExist(queue, project)

        message = Message.from_redis(message_id, self._client)
        now = timeutils.utcnow_ts()

        if message and not utils.msg_expired_filter(message, now):
            return message.to_basic(now)
        else:
            raise errors.MessageDoesNotExist(message_id, queue, project)
Exemplo n.º 31
0
    def _incr_append(self, key, other):
        with lockutils.lock(key):
            timeout, value = self._get_unlocked(key)

            if value is None:
                return None

            ttl = timeutils.utcnow_ts() - timeout
            new_value = value + other
            self._set_unlocked(key, new_value, ttl)
            return new_value
Exemplo n.º 32
0
    def _incr_append(self, key, other):
        with lockutils.lock(key):
            timeout, value = self._get_unlocked(key)

            if value is None:
                return None

            ttl = timeutils.utcnow_ts() - timeout
            new_value = value + other
            self._set_unlocked(key, new_value, ttl)
            return new_value
Exemplo n.º 33
0
 def it():
     now = timeutils.utcnow_ts()
     for id, body, ttl, created, cid in records:
         marker_id['next'] = id
         yield {
             'id': utils.msgid_encode(id),
             'ttl': ttl,
             'age': now - calendar.timegm(created.timetuple()),
             'body': utils.json_decode(body),
             'claim_id': utils.cid_encode(cid) if cid else None,
         }
Exemplo n.º 34
0
    def test_msg_claimed_filter(self):
        now = timeutils.utcnow_ts()

        unclaimed_msg = _create_sample_message()
        self.assertFalse(utils.msg_claimed_filter(unclaimed_msg, now))

        claimed_msg = _create_sample_message(claimed=True)
        self.assertTrue(utils.msg_claimed_filter(claimed_msg, now))

        # NOTE(kgriffs): Has a claim ID, but the claim is expired
        claimed_msg.claim_expires = now - 60
        self.assertFalse(utils.msg_claimed_filter(claimed_msg, now))
Exemplo n.º 35
0
    def _inc_counter(self, name, project=None, amount=1, window=None):
        """Increments the message counter and returns the new value.

        :param name: Name of the queue to which the counter is scoped
        :param project: Queue's project name
        :param amount: (Default 1) Amount by which to increment the counter
        :param window: (Default None) A time window, in seconds, that
            must have elapsed since the counter was last updated, in
            order to increment the counter.

        :returns: Updated message counter value, or None if window
            was specified, and the counter has already been updated
            within the specified time period.

        :raises: storage.errors.QueueDoesNotExist
        """
        now = timeutils.utcnow_ts()

        update = {'$inc': {'c.v': amount}, '$set': {'c.t': now}}
        query = _get_scoped_query(name, project)
        if window is not None:
            threshold = now - window
            query['c.t'] = {'$lt': threshold}

        while True:
            try:
                doc = self._collection.find_and_modify(
                    query, update, new=True, fields={'c.v': 1, '_id': 0})

                break
            except pymongo.errors.AutoReconnect as ex:
                LOG.exception(ex)

        if doc is None:
            if window is None:
                # NOTE(kgriffs): Since we did not filter by a time window,
                # the queue should have been found and updated. Perhaps
                # the queue has been deleted?
                message = _(u'Failed to increment the message '
                            u'counter for queue %(name)s and '
                            u'project %(project)s')
                message %= dict(name=name, project=project)

                LOG.warning(message)

                raise errors.QueueDoesNotExist(name, project)

            # NOTE(kgriffs): Assume the queue existed, but the counter
            # was recently updated, causing the range query on 'c.t' to
            # exclude the record.
            return None

        return doc['c']['v']
Exemplo n.º 36
0
    def pop(self, queue_name, limit, project=None):
        if project is None:
            project = ''

        with self.driver.trans() as trans:
            sel = sa.sql.select([
                tables.Messages.c.id, tables.Messages.c.body,
                tables.Messages.c.ttl, tables.Messages.c.created,
                tables.Messages.c.cid
            ])

            j = sa.join(tables.Messages, tables.Queues,
                        tables.Messages.c.qid == tables.Queues.c.id)

            sel = sel.select_from(j)
            and_clause = self._and_stmt_with_ttl(queue_name, project)

            and_clause.append(tables.Messages.c.cid == (None))

            sel = sel.where(sa.and_(*and_clause))
            sel = sel.limit(limit)

            records = trans.execute(sel)
            now = timeutils.utcnow_ts()
            messages = []
            message_ids = []
            for id, body, ttl, created, cid in records:
                messages.append({
                    'id':
                    utils.msgid_encode(id),
                    'ttl':
                    ttl,
                    'age':
                    now - calendar.timegm(created.timetuple()),
                    'body':
                    utils.json_decode(body),
                    'claim_id':
                    utils.cid_encode(cid) if cid else None,
                })
                message_ids.append(id)

            statement = tables.Messages.delete()

            qid = utils.get_qid(self.driver, queue_name, project)

            and_stmt = [
                tables.Messages.c.id.in_(message_ids),
                tables.Messages.c.qid == qid
            ]

            trans.execute(statement.where(sa.and_(*and_stmt)))

            return messages
Exemplo n.º 37
0
    def _list(self,
              queue,
              project=None,
              marker=None,
              limit=storage.DEFAULT_MESSAGES_PER_PAGE,
              echo=False,
              client_uuid=None,
              include_claimed=False,
              to_basic=True):

        if not self._queue_ctrl.exists(queue, project):
            raise errors.QueueDoesNotExist(queue, project)

        msgset_key = utils.msgset_key(queue, project)
        client = self._client

        if not marker and not include_claimed:
            # NOTE(kgriffs): Skip unclaimed messages at the head
            # of the queue; otherwise we would just filter them all
            # out and likely end up with an empty list to return.
            marker = self._find_first_unclaimed(queue, project, limit)
            start = client.zrank(msgset_key, marker) or 0
        else:
            rank = client.zrank(msgset_key, marker)
            start = rank + 1 if rank else 0

        message_ids = client.zrange(msgset_key, start, start + (limit - 1))

        messages = Message.from_redis_bulk(message_ids, client)

        # NOTE(prashanthr_): Build a list of filters for checking
        # the following:
        #
        #     1. Message is expired
        #     2. Message is claimed
        #     3. Message should not be echoed
        #
        now = timeutils.utcnow_ts()
        filters = [functools.partial(utils.msg_expired_filter, now=now)]

        if not include_claimed:
            filters.append(functools.partial(utils.msg_claimed_filter,
                                             now=now))

        if not echo:
            filters.append(
                functools.partial(utils.msg_echo_filter,
                                  client_uuid=client_uuid))

        marker = {}

        yield _filter_messages(messages, filters, to_basic, marker)
        yield marker['next']
Exemplo n.º 38
0
    def get(self, key):
        """Retrieves the value for a key or None.

        This expunges expired keys during each get.
        """

        now = timeutils.utcnow_ts()
        for k in list(self.cache):
            (timeout, _value) = self.cache[k]
            if timeout and now >= timeout:
                del self.cache[k]

        return self.cache.get(key, (0, None))[1]
Exemplo n.º 39
0
Arquivo: claims.py Projeto: rose/zaqar
    def _gc(self, queue, project):
        """Garbage-collect expired claim data.

        Not all claim data can be automatically expired. This method
        cleans up the remainder.

        :returns: Number of claims removed
        """

        claims_set_key = utils.scope_claims_set(queue, project,
                                                QUEUE_CLAIMS_SUFFIX)
        now = timeutils.utcnow_ts()
        num_removed = self._client.zremrangebyscore(claims_set_key, 0, now)
        return num_removed
Exemplo n.º 40
0
    def first(self, queue, project=None, sort=1):
        if sort not in (1, -1):
            raise ValueError(u'sort must be either 1 (ascending) '
                             u'or -1 (descending)')

        message_id = self._get_first_message_id(queue, project, sort)
        if not message_id:
            raise errors.QueueIsEmpty(queue, project)

        message = Message.from_redis(message_id, self._client)
        if message is None:
            raise errors.QueueIsEmpty(queue, project)

        now = timeutils.utcnow_ts()
        return message.to_basic(now, include_created=True)
Exemplo n.º 41
0
    def bulk_get(self, queue, message_ids, project=None):
        if not self._queue_ctrl.exists(queue, project):
            raise errors.QueueDoesNotExist(queue, project)

        # NOTE(prashanthr_): Pipelining is used here purely
        # for performance.
        with self._client.pipeline() as pipe:
            for mid in message_ids:
                pipe.hgetall(mid)

            messages = pipe.execute()

        # NOTE(kgriffs): Skip messages that may have been deleted
        now = timeutils.utcnow_ts()
        return (Message.from_hmap(msg).to_basic(now) for msg in messages
                if msg)
Exemplo n.º 42
0
    def _unclaim(self, queue_name, claim_id, project=None):
        cid = utils.to_oid(claim_id)

        # NOTE(cpp-cabrera): early abort - avoid a DB query if we're handling
        # an invalid ID
        if cid is None:
            return

        # NOTE(cpp-cabrera):  unclaim by setting the claim ID to None
        # and the claim expiration time to now
        now = timeutils.utcnow_ts()
        scope = utils.scope_queue_name(queue_name, project)
        collection = self._collection(queue_name, project)

        collection.update({PROJ_QUEUE: scope, 'c.id': cid},
                          {'$set': {'c': {'id': None, 'e': now}}},
                          upsert=False, multi=True)
Exemplo n.º 43
0
    def test_basic_message(self):
        now = timeutils.utcnow_ts()
        body = {
            'msg': 'Hello Earthlings!',
            'unicode': u'ab\u00e7',
            'bytes': b'ab\xc3\xa7',
            b'ab\xc3\xa7': 'one, two, three',
            u'ab\u00e7': 'one, two, three',
        }

        msg = _create_sample_message(now=now, body=body)
        basic_msg = msg.to_basic(now + 5)

        self.assertEqual(basic_msg['id'], msg.id)
        self.assertEqual(basic_msg['age'], 5)
        self.assertEqual(basic_msg['body'], body)
        self.assertEqual(basic_msg['ttl'], msg.ttl)
Exemplo n.º 44
0
    def delete(self, queue_name, message_id, project=None, claim=None):
        # NOTE(cpp-cabrera): return early - this is an invalid message
        # id so we won't be able to find it any way
        mid = utils.to_oid(message_id)
        if mid is None:
            return

        collection = self._collection(queue_name, project)

        query = {
            '_id': mid,
            PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
        }

        cid = utils.to_oid(claim)
        if cid is None:
            raise errors.ClaimDoesNotExist(queue_name, project, claim)

        now = timeutils.utcnow_ts()
        cursor = collection.find(query).hint(ID_INDEX_FIELDS)

        try:
            message = next(cursor)
        except StopIteration:
            return

        if claim is None:
            if _is_claimed(message, now):
                raise errors.MessageIsClaimed(message_id)

        else:
            if message['c']['id'] != cid:
                # NOTE(kgriffs): Read from primary in case the message
                # was just barely claimed, and claim hasn't made it to
                # the secondary.
                pref = pymongo.read_preferences.ReadPreference.PRIMARY
                message = collection.find_one(query, read_preference=pref)

                if message['c']['id'] != cid:
                    if _is_claimed(message, now):
                        raise errors.MessageNotClaimedBy(message_id, claim)

                    raise errors.MessageNotClaimed(message_id)

        collection.remove(query['_id'], w=0)
Exemplo n.º 45
0
def _create_sample_message(now=None, claimed=False, body=None):
    if now is None:
        now = timeutils.utcnow_ts()

    if claimed:
        claim_id = uuid.uuid4()
        claim_expires = now + 300
    else:
        claim_id = None
        claim_expires = now

    if body is None:
        body = {}

    return messages.Message(ttl=60,
                            created=now,
                            client_uuid=uuid.uuid4(),
                            claim_id=claim_id,
                            claim_expires=claim_expires,
                            body=body)
Exemplo n.º 46
0
Arquivo: claims.py Projeto: rose/zaqar
    def _exists(self, queue, claim_id, project):
        client = self._client
        claims_set_key = utils.scope_claims_set(queue, project,
                                                QUEUE_CLAIMS_SUFFIX)

        # Return False if no such claim exists
        # TODO(prashanthr_): Discuss the feasibility of a bloom filter.
        if client.zscore(claims_set_key, claim_id) is None:
            return False

        expires = self._get_claim_info(claim_id, b'e')[0]
        now = timeutils.utcnow_ts()

        if expires <= now:
            # NOTE(kgriffs): Redis should automatically remove the
            # other records in the very near future. This one
            # has to be manually deleted, however.
            client.zrem(claims_set_key, claim_id)
            return False

        return True
Exemplo n.º 47
0
    def get(self, queue_name, message_id, project=None):
        mid = utils.to_oid(message_id)
        if mid is None:
            raise errors.MessageDoesNotExist(message_id, queue_name,
                                             project)

        now = timeutils.utcnow_ts()

        query = {
            '_id': mid,
            PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
        }

        collection = self._collection(queue_name, project)
        message = list(collection.find(query).limit(1).hint(ID_INDEX_FIELDS))

        if not message:
            raise errors.MessageDoesNotExist(message_id, queue_name,
                                             project)

        return _basic_message(message[0], now)
Exemplo n.º 48
0
    def _purge_expired(self):
        """Removes expired keys from the cache."""

        now = timeutils.utcnow_ts()
        for timeout in sorted(self._keys_expires.keys()):

            # NOTE(flaper87): If timeout is greater
            # than `now`, stop the iteration, remaining
            # keys have not expired.
            if now < timeout:
                break

            # NOTE(flaper87): Unset every key in
            # this set from the cache if its timeout
            # is equal to `timeout`. (The key might
            # have been updated)
            for subkey in self._keys_expires.pop(timeout):
                try:
                    if self._cache[subkey][0] == timeout:
                        del self._cache[subkey]
                except KeyError:
                    continue
Exemplo n.º 49
0
Arquivo: claims.py Projeto: rose/zaqar
    def get(self, queue, claim_id, project=None):
        msg_ctrl = self.driver.message_controller

        # Base query, always check expire time
        now = timeutils.utcnow_ts()
        cid = utils.to_oid(claim_id)
        if cid is None:
            raise errors.ClaimDoesNotExist(queue, project, claim_id)

        def messages(msg_iter):
            msg = next(msg_iter)
            yield msg.pop('claim')
            yield msg

            # Smoke it!
            for msg in msg_iter:
                del msg['claim']
                yield msg

        try:
            # Lets get claim's data
            # from the first message
            # in the iterator
            msgs = messages(msg_ctrl._claimed(queue, cid, now,
                                              project=project))
            claim = next(msgs)

            update_time = claim['e'] - claim['t']
            age = now - update_time

            claim_meta = {
                'age': int(age),
                'ttl': claim['t'],
                'id': str(claim['id']),
            }
        except StopIteration:
            raise errors.ClaimDoesNotExist(cid, queue, project)

        return (claim_meta, msgs)
Exemplo n.º 50
0
    def _find_first_unclaimed(self, queue, project, limit):
        """Find the first unclaimed message in the queue."""

        msgset_key = utils.msgset_key(queue, project)
        now = timeutils.utcnow_ts()

        # TODO(kgriffs): Generalize this paging pattern (DRY)
        offset = 0

        while True:
            msg_keys = self._client.zrange(msgset_key, offset,
                                           offset + limit - 1)
            if not msg_keys:
                return None

            offset += len(msg_keys)

            messages = [MessageEnvelope.from_redis(msg_key, self._client)
                        for msg_key in msg_keys]

            for msg in messages:
                if msg and not utils.msg_claimed_filter(msg, now):
                    return msg.id
Exemplo n.º 51
0
    def pop(self, queue_name, limit, project=None):
        query = {
            PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
        }

        # Only include messages that are not part of
        # any claim, or are part of an expired claim.
        now = timeutils.utcnow_ts()
        query['c.e'] = {'$lte': now}

        collection = self._collection(queue_name, project)
        fields = {'_id': 1, 't': 1, 'b': 1, 'c.id': 1}

        messages = (collection.find_and_modify(query,
                                               fields=fields,
                                               remove=True)
                    for _ in range(limit))

        final_messages = [_basic_message(message, now)
                          for message in messages
                          if message]

        return final_messages
Exemplo n.º 52
0
    def bulk_get(self, queue_name, message_ids, project=None):
        message_ids = [mid for mid in map(utils.to_oid, message_ids) if mid]
        if not message_ids:
            return iter([])

        now = timeutils.utcnow_ts()

        # Base query, always check expire time
        query = {
            '_id': {'$in': message_ids},
            PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
        }

        collection = self._collection(queue_name, project)

        # NOTE(flaper87): Should this query
        # be sorted?
        messages = collection.find(query).hint(ID_INDEX_FIELDS)

        def denormalizer(msg):
            return _basic_message(msg, now)

        return utils.HookedCursor(messages, denormalizer)
Exemplo n.º 53
0
    def test_gc(self):
        self.queue_ctrl.create(self.queue_name)

        for _ in range(100):
            self.message_ctrl.post(self.queue_name, [{
                'ttl': 300,
                'body': 'yo gabba'
            }],
                                   client_uuid=str(uuid.uuid4()))

        now = timeutils.utcnow_ts()
        timeutils_utcnow = 'oslo.utils.timeutils.utcnow_ts'

        # Test a single claim
        with mock.patch(timeutils_utcnow) as mock_utcnow:
            mock_utcnow.return_value = now - 1
            self.controller.create(self.queue_name, {'ttl': 1, 'grace': 60})

        num_removed = self.controller._gc(self.queue_name, None)
        self.assertEqual(num_removed, 1)

        # Test multiple claims
        with mock.patch(timeutils_utcnow) as mock_utcnow:
            mock_utcnow.return_value = now - 1

            for _ in range(5):
                self.controller.create(self.queue_name, {
                    'ttl': 1,
                    'grace': 60
                })

        # NOTE(kgriffs): These ones should not be cleaned up
        self.controller.create(self.queue_name, {'ttl': 60, 'grace': 60})
        self.controller.create(self.queue_name, {'ttl': 60, 'grace': 60})

        num_removed = self.controller._gc(self.queue_name, None)
        self.assertEqual(num_removed, 5)
Exemplo n.º 54
0
Arquivo: claims.py Projeto: rose/zaqar
    def create(self,
               queue,
               metadata,
               project=None,
               limit=storage.DEFAULT_MESSAGES_PER_CLAIM):

        claim_ttl = metadata['ttl']
        grace = metadata['grace']

        now = timeutils.utcnow_ts()
        msg_ttl = claim_ttl + grace
        claim_expires = now + claim_ttl
        msg_expires = claim_expires + grace

        claim_id = utils.generate_uuid()
        claimed_msgs = []

        # NOTE(kgriffs): Claim some messages
        msgset_key = utils.msgset_key(queue, project)
        claimed_ids = self._claim_messages(msgset_key, now, limit, claim_id,
                                           claim_expires, msg_ttl, msg_expires)

        if claimed_ids:
            claimed_msgs = messages.Message.from_redis_bulk(
                claimed_ids, self._client)
            claimed_msgs = [msg.to_basic(now) for msg in claimed_msgs]

            # NOTE(kgriffs): Perist claim records
            with self._client.pipeline() as pipe:
                claim_msgs_key = utils.scope_claim_messages(
                    claim_id, CLAIM_MESSAGES_SUFFIX)

                for mid in claimed_ids:
                    pipe.rpush(claim_msgs_key, mid)

                pipe.expire(claim_msgs_key, claim_ttl)

                claim_info = {
                    'id': claim_id,
                    't': claim_ttl,
                    'e': claim_expires,
                    'n': len(claimed_ids),
                }

                pipe.hmset(claim_id, claim_info)
                pipe.expire(claim_id, claim_ttl)

                # NOTE(kgriffs): Add the claim ID to a set so that
                # existence checks can be performed quickly. This
                # is also used as a watch key in order to gaurd
                # against race conditions.
                #
                # A sorted set is used to facilitate cleaning
                # up the IDs of expired claims.
                claims_set_key = utils.scope_claims_set(
                    queue, project, QUEUE_CLAIMS_SUFFIX)

                pipe.zadd(claims_set_key, claim_expires, claim_id)
                pipe.execute()

        return claim_id, claimed_msgs
Exemplo n.º 55
0
    def post(self, queue_name, messages, client_uuid, project=None):
        # NOTE(flaper87): This method should be safe to retry on
        # autoreconnect, since we've a 2-step insert for messages.
        # The worst-case scenario is that we'll increase the counter
        # several times and we'd end up with some non-active messages.

        if not self._queue_ctrl.exists(queue_name, project):
            raise errors.QueueDoesNotExist(queue_name, project)

        now = timeutils.utcnow_ts()
        now_dt = datetime.datetime.utcfromtimestamp(now)
        collection = self._collection(queue_name, project)

        # Set the next basis marker for the first attempt.
        #
        # Note that we don't increment the counter right away because
        # if 2 concurrent posts happen and the one with the higher counter
        # ends before the one with the lower counter, there's a window
        # where a client paging through the queue may get the messages
        # with the higher counter and skip the previous ones. This would
        # make our FIFO guarantee unsound.
        next_marker = self._queue_ctrl._get_counter(queue_name, project)

        # Unique transaction ID to facilitate atomic batch inserts
        transaction = objectid.ObjectId()

        prepared_messages = [
            {
                PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
                't': message['ttl'],
                'e': now_dt + datetime.timedelta(seconds=message['ttl']),
                'u': client_uuid,
                'c': {'id': None, 'e': now},
                'b': message['body'] if 'body' in message else {},
                'k': next_marker + index,
                'tx': transaction,
            }

            for index, message in enumerate(messages)
        ]

        # NOTE(kgriffs): Don't take the time to do a 2-phase insert
        # if there is no way for it to partially succeed.
        if len(prepared_messages) == 1:
            transaction = None
            prepared_messages[0]['tx'] = None

        # Use a retry range for sanity, although we expect
        # to rarely, if ever, reach the maximum number of
        # retries.
        #
        # NOTE(kgriffs): With the default configuration (100 ms
        # max sleep, 1000 max attempts), the max stall time
        # before the operation is abandoned is 49.95 seconds.
        for attempt in self._retry_range:
            try:
                ids = collection.insert(prepared_messages)

                # Log a message if we retried, for debugging perf issues
                if attempt != 0:
                    msgtmpl = _(u'%(attempts)d attempt(s) required to post '
                                u'%(num_messages)d messages to queue '
                                u'"%(queue)s" under project %(project)s')

                    LOG.debug(msgtmpl,
                              dict(queue=queue_name,
                                   attempts=attempt + 1,
                                   num_messages=len(ids),
                                   project=project))

                # Update the counter in preparation for the next batch
                #
                # NOTE(kgriffs): Due to the unique index on the messages
                # collection, competing inserts will fail as a whole,
                # and keep retrying until the counter is incremented
                # such that the competing marker's will start at a
                # unique number, 1 past the max of the messages just
                # inserted above.
                self._queue_ctrl._inc_counter(queue_name, project,
                                              amount=len(ids))

                # NOTE(kgriffs): Finalize the insert once we can say that
                # all the messages made it. This makes bulk inserts
                # atomic, assuming queries filter out any non-finalized
                # messages.
                if transaction is not None:
                    collection.update({'tx': transaction},
                                      {'$set': {'tx': None}},
                                      upsert=False, multi=True)

                return [str(id_) for id_ in ids]

            except pymongo.errors.DuplicateKeyError as ex:
                # TODO(kgriffs): Record stats of how often retries happen,
                # and how many attempts, on average, are required to insert
                # messages.

                # NOTE(kgriffs): This can be used in conjunction with the
                # log line, above, that is emitted after all messages have
                # been posted, to gauge how long it is taking for messages
                # to be posted to a given queue, or overall.
                #
                # TODO(kgriffs): Add transaction ID to help match up loglines
                if attempt == 0:
                    msgtmpl = _(u'First attempt failed while '
                                u'adding messages to queue '
                                u'"%(queue)s" under project %(project)s')

                    LOG.debug(msgtmpl, dict(queue=queue_name, project=project))

                # NOTE(kgriffs): Never retry past the point that competing
                # messages expire and are GC'd, since once they are gone,
                # the unique index no longer protects us from getting out
                # of order, which could cause an observer to miss this
                # message. The code below provides a sanity-check to ensure
                # this situation can not happen.
                elapsed = timeutils.utcnow_ts() - now
                if elapsed > MAX_RETRY_POST_DURATION:
                    msgtmpl = _(u'Exceeded maximum retry duration for queue '
                                u'"%(queue)s" under project %(project)s')

                    LOG.warning(msgtmpl,
                                dict(queue=queue_name, project=project))
                    break

                # Chill out for a moment to mitigate thrashing/thundering
                self._backoff_sleep(attempt)

                # NOTE(kgriffs): Perhaps we failed because a worker crashed
                # after inserting messages, but before incrementing the
                # counter; that would cause all future requests to stall,
                # since they would keep getting the same base marker that is
                # conflicting with existing messages, until the messages that
                # "won" expire, at which time we would end up reusing markers,
                # and that could make some messages invisible to an observer
                # that is querying with a marker that is large than the ones
                # being reused.
                #
                # To mitigate this, we apply a heuristic to determine whether
                # a counter has stalled. We attempt to increment the counter,
                # but only if it hasn't been updated for a few seconds, which
                # should mean that nobody is left to update it!
                #
                # Note that we increment one at a time until the logjam is
                # broken, since we don't know how many messages were posted
                # by the worker before it crashed.
                next_marker = self._queue_ctrl._inc_counter(
                    queue_name, project, window=COUNTER_STALL_WINDOW)

                # Retry the entire batch with a new sequence of markers.
                #
                # NOTE(kgriffs): Due to the unique index, and how
                # MongoDB works with batch requests, we will never
                # end up with a partially-successful update. The first
                # document in the batch will fail to insert, and the
                # remainder of the documents will not be attempted.
                if next_marker is None:
                    # NOTE(kgriffs): Usually we will end up here, since
                    # it should be rare that a counter becomes stalled.
                    next_marker = self._queue_ctrl._get_counter(
                        queue_name, project)
                else:
                    msgtmpl = (u'Detected a stalled message counter for '
                               u'queue "%(queue)s" under project %(project)s. '
                               u'The counter was incremented to %(value)d.')

                    LOG.warning(msgtmpl,
                                dict(queue=queue_name,
                                     project=project,
                                     value=next_marker))

                for index, message in enumerate(prepared_messages):
                    message['k'] = next_marker + index

            except Exception as ex:
                LOG.exception(ex)
                raise

        msgtmpl = _(u'Hit maximum number of attempts (%(max)s) for queue '
                    u'"%(queue)s" under project %(project)s')

        LOG.warning(msgtmpl,
                    dict(max=self.driver.mongodb_conf.max_attempts,
                         queue=queue_name,
                         project=project))

        raise errors.MessageConflict(queue_name, project)
Exemplo n.º 56
0
    def _list(self, queue_name, project=None, marker=None,
              echo=False, client_uuid=None, fields=None,
              include_claimed=False, sort=1, limit=None):
        """Message document listing helper.

        :param queue_name: Name of the queue to list
        :param project: (Default None) Project `queue_name` belongs to. If
            not specified, queries the "global" namespace/project.
        :param marker: (Default None) Message marker from which to start
            iterating. If not specified, starts with the first message
            available in the queue.
        :param echo: (Default False) Whether to return messages that match
            client_uuid
        :param client_uuid: (Default None) UUID for the client that
            originated this request
        :param fields: (Default None) Fields to include in emitted
            documents
        :param include_claimed: (Default False) Whether to include
            claimed messages, not just active ones
        :param sort: (Default 1) Sort order for the listing. Pass 1 for
            ascending (oldest message first), or -1 for descending (newest
            message first).
        :param limit: (Default None) The maximum number of messages
            to list. The results may include fewer messages than the
            requested `limit` if not enough are available. If limit is
            not specified

        :returns: Generator yielding up to `limit` messages.
        """

        if sort not in (1, -1):
            raise ValueError(u'sort must be either 1 (ascending) '
                             u'or -1 (descending)')

        now = timeutils.utcnow_ts()

        query = {
            # Messages must belong to this queue and project.
            PROJ_QUEUE: utils.scope_queue_name(queue_name, project),

            # NOTE(kgriffs): Messages must be finalized (i.e., must not
            # be part of an unfinalized transaction).
            #
            # See also the note wrt 'tx' within the definition
            # of ACTIVE_INDEX_FIELDS.
            'tx': None,
        }

        if not echo:
            query['u'] = {'$ne': client_uuid}

        if marker is not None:
            query['k'] = {'$gt': marker}

        collection = self._collection(queue_name, project)

        if not include_claimed:
            # Only include messages that are not part of
            # any claim, or are part of an expired claim.
            query['c.e'] = {'$lte': now}

        # Construct the request
        cursor = collection.find(query, fields=fields, sort=[('k', sort)])

        if limit is not None:
            cursor.limit(limit)

        # NOTE(flaper87): Suggest the index to use for this query to
        # ensure the most performant one is chosen.
        return cursor.hint(ACTIVE_INDEX_FIELDS)
Exemplo n.º 57
0
    def test_service_is_up(self):
        serv = self.useFixture(
            ServiceFixture(self._host, self._binary, self._topic)).serv
        serv.start()
        service_ref = db.service_get_by_args(self._ctx,
                                             self._host,
                                             self._binary)
        fake_now = 1000
        down_time = 15
        self.flags(service_down_time=down_time)
        self.mox.StubOutWithMock(timeutils, 'utcnow_ts')
        self.servicegroup_api = servicegroup.API()
        hostkey = str("%s:%s" % (self._topic, self._host))

        # Up (equal)
        timeutils.utcnow_ts().AndReturn(fake_now)
        timeutils.utcnow_ts().AndReturn(fake_now + down_time - 1)
        self.mox.ReplayAll()
        self.servicegroup_api._driver.mc.set(hostkey,
                                             timeutils.utcnow(),
                                             time=down_time)
        result = self.servicegroup_api.service_is_up(service_ref)
        self.assertTrue(result)

        self.mox.ResetAll()
        # Up
        timeutils.utcnow_ts().AndReturn(fake_now)
        timeutils.utcnow_ts().AndReturn(fake_now + down_time - 2)
        self.mox.ReplayAll()
        self.servicegroup_api._driver.mc.set(hostkey,
                                             timeutils.utcnow(),
                                             time=down_time)
        result = self.servicegroup_api.service_is_up(service_ref)
        self.assertTrue(result)

        self.mox.ResetAll()
        # Down
        timeutils.utcnow_ts().AndReturn(fake_now)
        timeutils.utcnow_ts().AndReturn(fake_now + down_time)
        self.mox.ReplayAll()
        self.servicegroup_api._driver.mc.set(hostkey,
                                             timeutils.utcnow(),
                                             time=down_time)
        result = self.servicegroup_api.service_is_up(service_ref)
        self.assertFalse(result)

        self.mox.ResetAll()
        # Down
        timeutils.utcnow_ts().AndReturn(fake_now)
        timeutils.utcnow_ts().AndReturn(fake_now + down_time + 1)
        self.mox.ReplayAll()
        self.servicegroup_api._driver.mc.set(hostkey,
                                             timeutils.utcnow(),
                                             time=down_time)
        result = self.servicegroup_api.service_is_up(service_ref)
        self.assertFalse(result)

        self.mox.ResetAll()
Exemplo n.º 58
0
Arquivo: claims.py Projeto: rose/zaqar
    def create(self,
               queue,
               metadata,
               project=None,
               limit=storage.DEFAULT_MESSAGES_PER_CLAIM):
        """Creates a claim.

        This implementation was done in a best-effort fashion.
        In order to create a claim we need to get a list
        of messages that can be claimed. Once we have that
        list we execute a query filtering by the ids returned
        by the previous query.

        Since there's a lot of space for race conditions here,
        we'll check if the number of updated records is equal to
        the max number of messages to claim. If the number of updated
        messages is lower than limit we'll try to claim the remaining
        number of messages.

        This 2 queries are required because there's no way, as for the
        time being, to execute an update on a limited number of records.
        """
        msg_ctrl = self.driver.message_controller

        ttl = metadata['ttl']
        grace = metadata['grace']
        oid = objectid.ObjectId()

        now = timeutils.utcnow_ts()
        claim_expires = now + ttl
        claim_expires_dt = datetime.datetime.utcfromtimestamp(claim_expires)

        message_ttl = ttl + grace
        message_expiration = datetime.datetime.utcfromtimestamp(claim_expires +
                                                                grace)

        meta = {
            'id': oid,
            't': ttl,
            'e': claim_expires,
        }

        # Get a list of active, not claimed nor expired
        # messages that could be claimed.
        msgs = msg_ctrl._active(queue,
                                fields={'_id': 1},
                                project=project,
                                limit=limit)

        messages = iter([])
        ids = [msg['_id'] for msg in msgs]

        if len(ids) == 0:
            return (None, messages)

        now = timeutils.utcnow_ts()

        # NOTE(kgriffs): Set the claim field for
        # the active message batch, while also
        # filtering out any messages that happened
        # to get claimed just now by one or more
        # parallel requests.
        #
        # Filtering by just 'c.e' works because
        # new messages have that field initialized
        # to the current time when the message is
        # posted. There is no need to check whether
        # 'c' exists or 'c.id' is None.
        collection = msg_ctrl._collection(queue, project)
        updated = collection.update({
            '_id': {
                '$in': ids
            },
            'c.e': {
                '$lte': now
            }
        }, {'$set': {
            'c': meta
        }},
                                    upsert=False,
                                    multi=True)['n']

        # NOTE(flaper87): Dirty hack!
        # This sets the expiration time to
        # `expires` on messages that would
        # expire before claim.
        new_values = {'e': message_expiration, 't': message_ttl}
        collection.update(
            {
                'p_q': utils.scope_queue_name(queue, project),
                'e': {
                    '$lt': claim_expires_dt
                },
                'c.id': oid
            }, {'$set': new_values},
            upsert=False,
            multi=True)

        if updated != 0:
            # NOTE(kgriffs): This extra step is necessary because
            # in between having gotten a list of active messages
            # and updating them, some of them may have been
            # claimed by a parallel request. Therefore, we need
            # to find out which messages were actually tagged
            # with the claim ID successfully.
            claim, messages = self.get(queue, oid, project=project)

        return (str(oid), messages)
Exemplo n.º 59
0
Arquivo: claims.py Projeto: rose/zaqar
    def update(self, queue, claim_id, metadata, project=None):
        cid = utils.to_oid(claim_id)
        if cid is None:
            raise errors.ClaimDoesNotExist(claim_id, queue, project)

        now = timeutils.utcnow_ts()
        grace = metadata['grace']
        ttl = metadata['ttl']
        claim_expires = now + ttl
        claim_expires_dt = datetime.datetime.utcfromtimestamp(claim_expires)
        message_ttl = ttl + grace
        message_expires = datetime.datetime.utcfromtimestamp(claim_expires +
                                                             grace)

        msg_ctrl = self.driver.message_controller
        claimed = msg_ctrl._claimed(queue,
                                    cid,
                                    expires=now,
                                    limit=1,
                                    project=project)

        try:
            next(claimed)
        except StopIteration:
            raise errors.ClaimDoesNotExist(claim_id, queue, project)

        meta = {
            'id': cid,
            't': ttl,
            'e': claim_expires,
        }

        # TODO(kgriffs): Create methods for these so we don't interact
        # with the messages collection directly (loose coupling)
        scope = utils.scope_queue_name(queue, project)
        collection = msg_ctrl._collection(queue, project)
        collection.update({
            'p_q': scope,
            'c.id': cid
        }, {'$set': {
            'c': meta
        }},
                          upsert=False,
                          multi=True)

        # NOTE(flaper87): Dirty hack!
        # This sets the expiration time to
        # `expires` on messages that would
        # expire before claim.
        collection.update(
            {
                'p_q': scope,
                'e': {
                    '$lt': claim_expires_dt
                },
                'c.id': cid
            }, {'$set': {
                'e': message_expires,
                't': message_ttl
            }},
            upsert=False,
            multi=True)