def _promotion_stickers(following_ids):
feed_promotion_keys = [
UserFeedPromotionBuffer.get_key(user_id) for user_id in following_ids
]
return [(
util.loads(promotion),
score,
) for promotion, score in redis.zunion(
feed_promotion_keys, withscores=True, transaction=False)]
def _followee_quest_ids(user, since_timestamp=None):
buffer_keys = ['ugq_by_user:{}'.format(followee_id)
for followee_id in user.redis.new_following.zrange(0, -1)]
items = redis.zunion(buffer_keys, withscores=True, transaction=False)
if since_timestamp is not None:
items = [item for item in items if item[1] > since_timestamp]
items = sorted(items, key=lambda item: -item[1])
return [int(item[0]) for item in items]
def get_user_top_posted(user, manager, aslice, anonymous=False, namefriend=True):
manager = getattr(Comment, manager)
redis_top = (user.redis.top_anonymous_posts if anonymous else user.redis.top_posts)
if anonymous and namefriend:
keys = [user.redis.top_anonymous_posts.key, user.redis.top_posts.key]
redis_top = list(redis.zunion(keys, withscores=True, transaction=False))
redis_top = sorted(redis_top, key=lambda x: -x[1])
redis_top = [int(x) for x,y in redis_top]
return manager.in_bulk_list(redis_top[aslice])
return manager.filter(anonymous=anonymous).in_bulk_list(redis_top[aslice])
def _feed_items(user, earliest_timestamp_cutoff=None, items_to_skip=set()):
feed_source_keys = [UserFeedSourceBuffer.get_key(user_id)
for user_id in user.redis.new_following.zrange(0, -1)]
max_score = _tighten_earliest_timestamp_cutoff(earliest_timestamp_cutoff)
comments = [{'type': 'comment', 'comment_id': id_, 'ts': score}
for id_, score
in redis.zunion(feed_source_keys,
withscores=True, transaction=False, max_score=max_score)]
# Sort by recency.
items = sorted(itertools.chain(comments), key=lambda e: float(e['ts']), reverse=True)
# Skip items as requested and skip comments the user has hidden.
comments_to_skip = set(str(item['comment_id']) for item in items_to_skip)
comments_to_skip |= set(user.flags.all().values_list('id', flat=True))
# Remove dupes.
items = OrderedDict((str(item['comment_id']), item,) for item in items
if str(item['comment_id']) not in comments_to_skip).values()
return items
def feed_for_user(user,
earliest_timestamp_cutoff=None,
per_page=knobs.FEED_ITEMS_PER_PAGE,
items_to_skip=set()):
following_ids = user.redis.following.smembers()
followed_thread_ids = user.redis.followed_threads.smembers()
if not following_ids and not followed_thread_ids:
return []
feed_source_keys = [
UserFeedSourceBuffer.get_key(user_id) for user_id in following_ids
]
feed_thread_source_keys = [
ThreadFeedSourceBuffer.get_key(thread_id)
for thread_id in followed_thread_ids
]
thread_posts = [{
'type': 'thread',
'comment_id': id_,
'ts': score
} for id_, score in redis.zunion(
feed_thread_source_keys,
withscores=True,
transaction=False,
max_score=_tighten_earliest_timestamp_cutoff(
earliest_timestamp_cutoff))]
posts = [{
'type': 'post',
'comment_id': id_,
'ts': score
} for id_, score in redis.zunion(
feed_source_keys,
withscores=True,
transaction=False,
max_score=_tighten_earliest_timestamp_cutoff(
earliest_timestamp_cutoff))]
posts += thread_posts
promotions = promoted_comments(
user,
earliest_timestamp_cutoff=earliest_timestamp_cutoff,
comments_to_skip=set(post['comment_id'] for post in posts))
# Sort by recency.
items = sorted(itertools.chain(posts, promotions),
key=lambda e: float(e['ts']),
reverse=True)
# Skip items as requested and skip comments the user has hidden.
hidden_comments = user.redis.hidden_comments.smembers()
comments_to_skip = set(str(item['comment_id'])
for item in items_to_skip) | hidden_comments
# Remove dupes.
items = OrderedDict(
(
str(item['comment_id']),
item,
) for item in items
if str(item['comment_id']) not in comments_to_skip).values()
# Pagination.
items = items[:per_page]
# Promote comment_id to CommentDetails.
details = CommentDetails.from_ids([item['comment_id'] for item in items])
for i, item in enumerate(items):
item['comment'] = details[i]
# Hide hidden threads.
items = user.redis.hidden_threads.filter_comments(
items, comment_key=lambda item: item['comment'])
# Prune items that shouldn't show up in this feed.
items = filter(
partial(visible_in_feed,
earliest_timestamp_cutoff=earliest_timestamp_cutoff), items)
items = filter(partial(not_self_authored, username=user.username), items)
# Determine whether each item is dismissable by user.
def item_is_dismissable(item):
if item['type'] == 'promotion' and item['username'].lower(
) == 'canvas':
return False
return item['type'] != 'sticky_thread' and is_dismissable(
item['comment'], user)
for item in items:
item['is_dismissable'] = item_is_dismissable(item)
# Add viewer_sticker to items.
for item in items:
_add_viewer_sticker_to_item(item, user)
return items
def feed_for_user(user, earliest_timestamp_cutoff=None, per_page=knobs.FEED_ITEMS_PER_PAGE, items_to_skip=set()):
following_ids = user.redis.following.smembers()
followed_thread_ids = user.redis.followed_threads.smembers()
if not following_ids and not followed_thread_ids:
return []
feed_source_keys = [UserFeedSourceBuffer.get_key(user_id) for user_id in following_ids]
feed_thread_source_keys = [ThreadFeedSourceBuffer.get_key(thread_id) for thread_id in followed_thread_ids]
thread_posts = [{'type': 'thread', 'comment_id': id_, 'ts': score}
for id_, score in
redis.zunion(feed_thread_source_keys,
withscores=True, transaction=False,
max_score=_tighten_earliest_timestamp_cutoff(earliest_timestamp_cutoff))]
posts = [{'type': 'post', 'comment_id': id_, 'ts': score}
for id_, score in
redis.zunion(feed_source_keys,
withscores=True, transaction=False,
max_score=_tighten_earliest_timestamp_cutoff(earliest_timestamp_cutoff))]
posts += thread_posts
promotions = promoted_comments(user,
earliest_timestamp_cutoff=earliest_timestamp_cutoff,
comments_to_skip=set(post['comment_id'] for post in posts))
# Sort by recency.
items = sorted(itertools.chain(posts, promotions), key=lambda e: float(e['ts']), reverse=True)
# Skip items as requested and skip comments the user has hidden.
hidden_comments = user.redis.hidden_comments.smembers()
comments_to_skip = set(str(item['comment_id']) for item in items_to_skip) | hidden_comments
# Remove dupes.
items = OrderedDict((str(item['comment_id']), item,) for item in items
if str(item['comment_id']) not in comments_to_skip).values()
# Pagination.
items = items[:per_page]
# Promote comment_id to CommentDetails.
details = CommentDetails.from_ids([item['comment_id'] for item in items])
for i, item in enumerate(items):
item['comment'] = details[i]
# Hide hidden threads.
items = user.redis.hidden_threads.filter_comments(items, comment_key=lambda item: item['comment'])
# Prune items that shouldn't show up in this feed.
items = filter(partial(visible_in_feed, earliest_timestamp_cutoff=earliest_timestamp_cutoff), items)
items = filter(partial(not_self_authored, username=user.username), items)
# Determine whether each item is dismissable by user.
def item_is_dismissable(item):
if item['type'] == 'promotion' and item['username'].lower() == 'canvas':
return False
return item['type'] != 'sticky_thread' and is_dismissable(item['comment'], user)
for item in items:
item['is_dismissable'] = item_is_dismissable(item)
# Add viewer_sticker to items.
for item in items:
_add_viewer_sticker_to_item(item, user)
return items
def _promotion_stickers(following_ids):
feed_promotion_keys = [UserFeedPromotionBuffer.get_key(user_id) for user_id in following_ids]
return [(util.loads(promotion), score,)
for promotion, score
in redis.zunion(feed_promotion_keys, withscores=True, transaction=False)]