Example #1
0
    def mention_keys(self, mentions, min=None, max=None, limit=100, tag=None):
        if not mentions:
            return ()
        if tag and tag not in self._tag_mapping:
            return ()

        if mentions == str(mentions):
            # single mention optimization
            mention = mentions
            mapping = self._mentions_mapping.get(mention)
            if not mapping:
                return ()

        else:
            # collection of LLTreeSet
            treesets = (self._mentions_mapping.get(mention)
                        for mention in mentions
                        if mention in self._mentions_mapping.keys())
            mapping = reduce(LLBTree.union, treesets, LLBTree.TreeSet())

        # returns unchanged mapping if tag is None
        mapping = self._keys_tag(tag, mapping)
        mapping = self.secure(mapping)
        return longkeysortreverse(mapping,
                                  min, max, limit)
Example #2
0
    def user_keys(self, users, min=None, max=None, limit=100, tag=None):
        if not users:
            return ()
        if tag and tag not in self._tag_mapping:
            return ()

        if users == str(users):
            # single user optimization
            userid = users
            mapping = self._user_mapping.get(userid)
            if not mapping:
                return ()

        else:
            # collection of user LLTreeSet
            treesets = (self._user_mapping.get(userid)
                        for userid in users
                        if userid in self._user_mapping.keys())
            mapping = reduce(LLBTree.union, treesets, LLBTree.TreeSet())

        # returns unchanged mapping if tag is None
        mapping = self._keys_tag(tag, mapping)
        mapping = self.secure(mapping)
        return longkeysortreverse(mapping,
                                  min, max, limit)
Example #3
0
    def allowed_status_keys(self):
        """Return the subset of IStatusUpdate keys
        that are related to UUIDs of accessible contexts.
        I.e. blacklist all IStatusUpdate that has a context
        which we don't have permission to access.

        This is the key security protection used by all getters.
        Because it's called a lot we're caching results per user request.
        """
        uuid_blacklist = self._blacklist_microblogcontext_uuids()
        if not uuid_blacklist:
            return self._status_mapping.keys()
        else:
            # for each uid, expand uid into set of statusids
            blacklisted_treesets = (self._uuid_mapping.get(uuid)
                                    for uuid in uuid_blacklist
                                    if uuid in self._uuid_mapping.keys())
            # merge sets of blacklisted statusids into single blacklist
            blacklisted_statusids = reduce(LLBTree.union,
                                           blacklisted_treesets,
                                           LLBTree.TreeSet())
            # subtract blacklisted statusids from all statusids
            all_statusids = LLBTree.LLSet(self._status_mapping.keys())
            return LLBTree.difference(all_statusids,
                                      blacklisted_statusids)

        return self._allowed_status_keys()
def longkeysortreverse(btreeish, minv=None, maxv=None, limit=None):
    """Performance optimized keyspace accessor.
    Returns an iterable of btreeish keys, reverse sorted by key.
    Expects a btreeish with long(microsec) keys.
    """
    try:
        accessor = btreeish.keys
    except AttributeError:
        accessor = LLBTree.TreeSet(btreeish).keys

    i = 0

    if minv or maxv:
        # no optimization
        keys = [x for x in accessor(min=minv, max=maxv)]
        keys.sort()
        keys.reverse()
        for key in keys:
            yield key
            i += 1
            if i == limit:
                return

    else:

        # first run: last hour
        tmax = long(time.time() * 1e6)
        tmin = long(tmax - 3600 * 1e6)
        keys = [x for x in accessor(min=tmin, max=tmax)]
        keys.sort()
        keys.reverse()
        for key in keys:
            yield key
            i += 1
            if i == limit:
                return

        # second run: last day until last hour
        tmax = tmin
        tmin = long(tmax - 23 * 3600 * 1e6)
        keys = [x for x in accessor(min=tmin, max=tmax)]
        keys.sort()
        keys.reverse()
        for key in keys:
            yield key
            i += 1
            if i == limit:
                return

        # final run: everything else
        tmax = tmin
        keys = [x for x in accessor(max=tmax)]
        keys.sort()
        keys.reverse()
        for key in keys:
            yield key
            i += 1
            if i == limit:
                return
 def _allowed_status_keys(self, uuid_blacklist=[]):
     if not uuid_blacklist:
         return self._status_mapping.keys()
     else:
         # for each uid, expand uid into set of statusids
         blacklisted_treesets = (self._uuid_mapping.get(uuid)
                                 for uuid in uuid_blacklist
                                 if uuid in self._uuid_mapping.keys())
         # merge sets of blacklisted statusids into single blacklist
         blacklisted_statusids = reduce(LLBTree.union, blacklisted_treesets,
                                        LLBTree.TreeSet())
         # subtract blacklisted statusids from all statusids
         all_statusids = LLBTree.LLSet(self._status_mapping.keys())
         return LLBTree.difference(all_statusids, blacklisted_statusids)
Example #6
0
def longkeysortreverse(btreeish, minv=None, maxv=None, limit=None):
    """Performance optimized keyspace accessor.
    Returns an iterable of btreeish keys, reverse sorted by key.
    Expects a btreeish with long(microsec) keys.

    In case a limit, but neither minv nor maxv is given, optimizates
    by not sorting the whole keyspace, but instead heuristically chunk
    the keyspace and sort only chunks, until the limit is reached.

    The reason for this is that we want the most recent slice, which
    is last in the accessor, so we cannot just start iterating the slice.
    Basically we want to iterate backwards.
    """
    try:
        accessor = btreeish.keys
    except AttributeError:
        accessor = LLBTree.TreeSet(btreeish).keys

    if minv or limit is None:
        return _longkeysortreverse_direct(accessor, minv, maxv, limit)
    else:
        return _longkeysortreverse_optimized(accessor, maxv, limit)