def mention_keys(self, mentions, min=None, max=None, limit=100, tag=None):
if not mentions:
return ()
if tag and tag not in self._tag_mapping:
return ()
if mentions == str(mentions):
# single mention optimization
mention = mentions
mapping = self._mentions_mapping.get(mention)
if not mapping:
return ()
else:
# collection of LLTreeSet
treesets = (self._mentions_mapping.get(mention)
for mention in mentions
if mention in self._mentions_mapping.keys())
mapping = reduce(LLBTree.union, treesets, LLBTree.TreeSet())
# returns unchanged mapping if tag is None
mapping = self._keys_tag(tag, mapping)
mapping = self.secure(mapping)
return longkeysortreverse(mapping,
min, max, limit)
def user_keys(self, users, min=None, max=None, limit=100, tag=None):
if not users:
return ()
if tag and tag not in self._tag_mapping:
return ()
if users == str(users):
# single user optimization
userid = users
mapping = self._user_mapping.get(userid)
if not mapping:
return ()
else:
# collection of user LLTreeSet
treesets = (self._user_mapping.get(userid)
for userid in users
if userid in self._user_mapping.keys())
mapping = reduce(LLBTree.union, treesets, LLBTree.TreeSet())
# returns unchanged mapping if tag is None
mapping = self._keys_tag(tag, mapping)
mapping = self.secure(mapping)
return longkeysortreverse(mapping,
min, max, limit)
def allowed_status_keys(self):
"""Return the subset of IStatusUpdate keys
that are related to UUIDs of accessible contexts.
I.e. blacklist all IStatusUpdate that has a context
which we don't have permission to access.
This is the key security protection used by all getters.
Because it's called a lot we're caching results per user request.
"""
uuid_blacklist = self._blacklist_microblogcontext_uuids()
if not uuid_blacklist:
return self._status_mapping.keys()
else:
# for each uid, expand uid into set of statusids
blacklisted_treesets = (self._uuid_mapping.get(uuid)
for uuid in uuid_blacklist
if uuid in self._uuid_mapping.keys())
# merge sets of blacklisted statusids into single blacklist
blacklisted_statusids = reduce(LLBTree.union,
blacklisted_treesets,
LLBTree.TreeSet())
# subtract blacklisted statusids from all statusids
all_statusids = LLBTree.LLSet(self._status_mapping.keys())
return LLBTree.difference(all_statusids,
blacklisted_statusids)
return self._allowed_status_keys()
def longkeysortreverse(btreeish, minv=None, maxv=None, limit=None):
"""Performance optimized keyspace accessor.
Returns an iterable of btreeish keys, reverse sorted by key.
Expects a btreeish with long(microsec) keys.
"""
try:
accessor = btreeish.keys
except AttributeError:
accessor = LLBTree.TreeSet(btreeish).keys
i = 0
if minv or maxv:
# no optimization
keys = [x for x in accessor(min=minv, max=maxv)]
keys.sort()
keys.reverse()
for key in keys:
yield key
i += 1
if i == limit:
return
else:
# first run: last hour
tmax = long(time.time() * 1e6)
tmin = long(tmax - 3600 * 1e6)
keys = [x for x in accessor(min=tmin, max=tmax)]
keys.sort()
keys.reverse()
for key in keys:
yield key
i += 1
if i == limit:
return
# second run: last day until last hour
tmax = tmin
tmin = long(tmax - 23 * 3600 * 1e6)
keys = [x for x in accessor(min=tmin, max=tmax)]
keys.sort()
keys.reverse()
for key in keys:
yield key
i += 1
if i == limit:
return
# final run: everything else
tmax = tmin
keys = [x for x in accessor(max=tmax)]
keys.sort()
keys.reverse()
for key in keys:
yield key
i += 1
if i == limit:
return
def _allowed_status_keys(self, uuid_blacklist=[]):
if not uuid_blacklist:
return self._status_mapping.keys()
else:
# for each uid, expand uid into set of statusids
blacklisted_treesets = (self._uuid_mapping.get(uuid)
for uuid in uuid_blacklist
if uuid in self._uuid_mapping.keys())
# merge sets of blacklisted statusids into single blacklist
blacklisted_statusids = reduce(LLBTree.union, blacklisted_treesets,
LLBTree.TreeSet())
# subtract blacklisted statusids from all statusids
all_statusids = LLBTree.LLSet(self._status_mapping.keys())
return LLBTree.difference(all_statusids, blacklisted_statusids)
def longkeysortreverse(btreeish, minv=None, maxv=None, limit=None):
"""Performance optimized keyspace accessor.
Returns an iterable of btreeish keys, reverse sorted by key.
Expects a btreeish with long(microsec) keys.
In case a limit, but neither minv nor maxv is given, optimizates
by not sorting the whole keyspace, but instead heuristically chunk
the keyspace and sort only chunks, until the limit is reached.
The reason for this is that we want the most recent slice, which
is last in the accessor, so we cannot just start iterating the slice.
Basically we want to iterate backwards.
"""
try:
accessor = btreeish.keys
except AttributeError:
accessor = LLBTree.TreeSet(btreeish).keys
if minv or limit is None:
return _longkeysortreverse_direct(accessor, minv, maxv, limit)
else:
return _longkeysortreverse_optimized(accessor, maxv, limit)