def __init__(self, user_id, redis=None, max_length=None):
'''
'''
RedisSortedSetCache.__init__(self, user_id, redis=redis)
#input validation
if not isinstance(user_id, int):
raise ValueError('user id should be an int, found %r' % user_id)
#support for different serialization schemes
self.serializer = self.serializer_class()
#support for pipelining redis
self.user_id = user_id
self.item_cache = LoveFeedItemCache('global')
self.key = self.key_format % user_id
self._max_length = max_length
def contains(self, activity):
'''
Uses zscore to see if the given activity is present in our sorted set
'''
result = RedisSortedSetCache.contains(self, activity.serialization_id)
activity_found = bool(result)
return activity_found
def __init__(self, user_id, redis=None):
'''
User id (the user for which we want to read/write notifications)
'''
RedisSortedSetCache.__init__(self, user_id, redis=redis)
# input validation
if not isinstance(user_id, int):
raise ValueError('user id should be an int, found %r' % user_id)
# support for different serialization schemes
self.serializer = self.get_serializer()
# support for pipelining redis
self.user_id = user_id
# write the key locations
self.format_dict = dict(user_id=user_id)
self.key = self.key_format % user_id
def contains(self, activity):
'''
Uses zscore to see if the given activity is present in our sorted set
'''
result = RedisSortedSetCache.contains(self, activity.serialization_id)
activity_found = bool(result)
return activity_found
def add_many(self, activities):
'''
We use pipelining for doing multiple adds
Alternatively we could also send multiple adds to one call.
Don't see a reason for that though
'''
value_score_pairs = []
key_value_pairs = []
for activity in activities:
value = self.serialize_activity(activity)
score = self.get_activity_score(activity)
#if its real data write the id to the redis hash cache
if not isinstance(activity, FeedEndMarker):
key_value_pairs.append((activity.serialization_id, value))
value_score_pairs.append((activity.serialization_id, score))
# we need to do this sequentially, otherwise there's a risk of broken reads
self.item_cache.set_many(key_value_pairs)
results = RedisSortedSetCache.add_many(self, value_score_pairs)
#make sure we trim to max length
self.trim()
return results
def __init__(self, user_id, redis=None):
'''
User id (the user for which we want to read/write notifications)
'''
RedisSortedSetCache.__init__(self, user_id, redis=redis)
# input validation
if not isinstance(user_id, int):
raise ValueError('user id should be an int, found %r' % user_id)
# support for different serialization schemes
self.serializer = self.get_serializer()
# support for pipelining redis
self.user_id = user_id
# write the key locations
self.format_dict = dict(user_id=user_id)
self.key = self.key_format % user_id
def add_many(self, activities, cache_item=True):
'''
We use pipelining for doing multiple adds
Alternatively we could also send multiple adds to one call.
Don't see a reason for that though
'''
value_score_pairs = []
key_value_pairs = []
for activity in activities:
value = self.serialize_activity(activity)
score = self.get_activity_score(activity)
#if its real data write the id to the redis hash cache
if not isinstance(activity, FeedEndMarker):
key_value_pairs.append((activity.serialization_id, value))
value_score_pairs.append((activity.serialization_id, score))
# we need to do this sequentially, otherwise there's a risk of broken reads
if cache_item:
self.item_cache.set_many(key_value_pairs)
else:
logger.debug('skipping item cache write')
results = RedisSortedSetCache.add_many(self, value_score_pairs)
#make sure we trim to max length
self.trim()
return results
def __init__(self, user_id, redis=None, max_length=None):
'''
'''
from feedly.feed_managers.love_feedly import LoveFeedly
self.manager = LoveFeedly
RedisSortedSetCache.__init__(self, user_id, redis=redis)
#input validation
if not isinstance(user_id, int):
raise ValueError('user id should be an int, found %r' % user_id)
#support for different serialization schemes
self.serializer = self.serializer_class()
#support for pipelining redis
self.user_id = user_id
self.item_cache = LoveFeedItemCache('global')
self.key = self.key_format % user_id
self._max_length = max_length
def remove_many(self, activities):
'''
Efficiently remove many activities
'''
values = []
for activity in activities:
values.append(activity.serialization_id)
results = RedisSortedSetCache.remove_many(self, values)
return results
def remove_many(self, activities):
'''
Efficiently remove many activities
'''
values = []
for activity in activities:
values.append(activity.serialization_id)
results = RedisSortedSetCache.remove_many(self, values)
return results
def remove_many(self, aggregated_activities):
'''
Efficiently remove many activities
'''
scores = []
for activity in aggregated_activities:
if not isinstance(activity, AggregatedActivity):
raise ValueError('we can only remove aggregated activities')
score = self.get_activity_score(activity)
scores.append(score)
results = RedisSortedSetCache.remove_by_scores(self, scores)
return results
def remove_many(self, aggregated_activities):
'''
Efficiently remove many activities
'''
scores = []
for activity in aggregated_activities:
if not isinstance(activity, AggregatedActivity):
raise ValueError('we can only remove aggregated activities')
score = self.get_activity_score(activity)
scores.append(score)
results = RedisSortedSetCache.remove_by_scores(self, scores)
return results
def mark_all(self, seen=True, read=None):
'''
Mark all the entries as seen or read
'''
# TODO refactor this code
with self.redis.lock(self.lock_key, timeout=2):
# get the current aggregated activities
activities = self[:self.max_length]
# create the update dict
update_dict = {}
for activity in activities:
changed = False
old_activity = copy.deepcopy(activity)
if seen is True and not activity.is_seen():
activity.seen_at = datetime.datetime.today()
changed = True
if read is True and not activity.is_read():
activity.read_at = datetime.datetime.today()
changed = True
if changed:
update_dict[old_activity] = activity
# now add the new ones and remove the old ones in one atomic operation
to_delete = []
to_add = []
for old, new in update_dict.items():
new_value = self.serialize_activity(new)
new_score = self.get_activity_score(new)
to_delete.append(old)
to_add.append((new_value, new_score))
# pipeline all our writes to improve performance
# Update: removed self.map(), multithreaded behaviour seems bugged
if to_delete:
delete_results = self.remove_many(to_delete)
# add the data in batch
if to_add:
add_results = RedisSortedSetCache.add_many(self, to_add)
# denormalize the count
count = self.denormalize_count(activities)
# return the new activities
return activities
def mark_all(self, seen=True, read=None):
'''
Mark all the entries as seen or read
'''
# TODO refactor this code
with self.redis.lock(self.lock_key, timeout=2):
# get the current aggregated activities
activities = self[:self.max_length]
# create the update dict
update_dict = {}
for activity in activities:
changed = False
old_activity = copy.deepcopy(activity)
if seen is True and not activity.is_seen():
activity.seen_at = datetime.datetime.today()
changed = True
if read is True and not activity.is_read():
activity.read_at = datetime.datetime.today()
changed = True
if changed:
update_dict[old_activity] = activity
# now add the new ones and remove the old ones in one atomic operation
to_delete = []
to_add = []
for old, new in update_dict.items():
new_value = self.serialize_activity(new)
new_score = self.get_activity_score(new)
to_delete.append(old)
to_add.append((new_value, new_score))
# pipeline all our writes to improve performance
# Update: removed self.map(), multithreaded behaviour seems bugged
if to_delete:
delete_results = self.remove_many(to_delete)
# add the data in batch
if to_add:
add_results = RedisSortedSetCache.add_many(self, to_add)
# denormalize the count
count = self.denormalize_count(activities)
# return the new activities
return activities
def get_results(self, start, stop):
redis_results = RedisSortedSetCache.get_results(self, start, stop)
enriched_results = self.deserialize_activities(redis_results)
return enriched_results
def add_many(self, activities):
'''
Note this function is very specific to notifications, this won't
get you good performance characteristics in applications with longer
lists
Add many works as follows:
- retrieve all aggregated activities
- add the new activities to the existing ones
- update the values in Redis by sending several deletes and adds
Trim the sorted set to max length
Denormalize the unseen count
Send a pubsub publish
'''
value_score_pairs = []
remove_activities = {}
aggregator = self.get_aggregator()
# first stick the new activities in groups
aggregated_activities = aggregator.aggregate(activities)
# get the current aggregated activities
current_activities = self[:self.max_length]
current_activities_dict = dict([(a.group, a)
for a in current_activities])
# see what we need to update
for activity in aggregated_activities:
if activity.group in current_activities_dict:
# update existing
current_activity = current_activities_dict[activity.group]
old_activity = copy.deepcopy(current_activity)
for a in activity.activities:
current_activity.append(a)
new_activity = current_activity
# we should only do this the first time, verify things go well
if old_activity.group in remove_activities:
raise ValueError('Thierry didnt expect this to happen')
remove_activities[old_activity.group] = old_activity
else:
# create a new activity
new_activity = activity
current_activities.append(new_activity)
# add the data to the to write list
value = self.serialize_activity(new_activity)
score = self.get_activity_score(new_activity)
value_score_pairs.append((value, score))
# pipeline all our writes to improve performance
# TODO: removed map just to be sure
# first remove the old notifications
delete_results = self.remove_many(remove_activities.values())
# add the data in batch
add_results = RedisSortedSetCache.add_many(self, value_score_pairs)
# make sure we trim to max length
trim_result = self.trim()
# return the current state of the notification feed
return current_activities
def get_results(self, start, stop):
redis_results = RedisSortedSetCache.get_results(self, start, stop)
enriched_results = self.deserialize_activities(redis_results)
return enriched_results
def add_many(self, activities):
'''
Note this function is very specific to notifications, this won't
get you good performance characteristics in applications with longer
lists
Add many works as follows:
- retrieve all aggregated activities
- add the new activities to the existing ones
- update the values in Redis by sending several deletes and adds
Trim the sorted set to max length
Denormalize the unseen count
Send a pubsub publish
'''
value_score_pairs = []
remove_activities = {}
aggregator = self.get_aggregator()
# first stick the new activities in groups
aggregated_activities = aggregator.aggregate(activities)
# get the current aggregated activities
current_activities = self[:self.max_length]
current_activities_dict = dict(
[(a.group, a) for a in current_activities])
# see what we need to update
for activity in aggregated_activities:
if activity.group in current_activities_dict:
# update existing
current_activity = current_activities_dict[activity.group]
old_activity = copy.deepcopy(current_activity)
for a in activity.activities:
current_activity.append(a)
new_activity = current_activity
# we should only do this the first time, verify things go well
if old_activity.group in remove_activities:
raise ValueError('Thierry didnt expect this to happen')
remove_activities[old_activity.group] = old_activity
else:
# create a new activity
new_activity = activity
current_activities.append(new_activity)
# add the data to the to write list
value = self.serialize_activity(new_activity)
score = self.get_activity_score(new_activity)
value_score_pairs.append((value, score))
# pipeline all our writes to improve performance
# TODO: removed map just to be sure
# first remove the old notifications
delete_results = self.remove_many(remove_activities.values())
# add the data in batch
add_results = RedisSortedSetCache.add_many(self, value_score_pairs)
# make sure we trim to max length
trim_result = self.trim()
# return the current state of the notification feed
return current_activities
