def __init__(self, user_id, redis=None, max_length=None): ''' ''' RedisSortedSetCache.__init__(self, user_id, redis=redis) #input validation if not isinstance(user_id, int): raise ValueError('user id should be an int, found %r' % user_id) #support for different serialization schemes self.serializer = self.serializer_class() #support for pipelining redis self.user_id = user_id self.item_cache = LoveFeedItemCache('global') self.key = self.key_format % user_id self._max_length = max_length
def contains(self, activity): ''' Uses zscore to see if the given activity is present in our sorted set ''' result = RedisSortedSetCache.contains(self, activity.serialization_id) activity_found = bool(result) return activity_found
def __init__(self, user_id, redis=None): ''' User id (the user for which we want to read/write notifications) ''' RedisSortedSetCache.__init__(self, user_id, redis=redis) # input validation if not isinstance(user_id, int): raise ValueError('user id should be an int, found %r' % user_id) # support for different serialization schemes self.serializer = self.get_serializer() # support for pipelining redis self.user_id = user_id # write the key locations self.format_dict = dict(user_id=user_id) self.key = self.key_format % user_id
def add_many(self, activities): ''' We use pipelining for doing multiple adds Alternatively we could also send multiple adds to one call. Don't see a reason for that though ''' value_score_pairs = [] key_value_pairs = [] for activity in activities: value = self.serialize_activity(activity) score = self.get_activity_score(activity) #if its real data write the id to the redis hash cache if not isinstance(activity, FeedEndMarker): key_value_pairs.append((activity.serialization_id, value)) value_score_pairs.append((activity.serialization_id, score)) # we need to do this sequentially, otherwise there's a risk of broken reads self.item_cache.set_many(key_value_pairs) results = RedisSortedSetCache.add_many(self, value_score_pairs) #make sure we trim to max length self.trim() return results
def add_many(self, activities, cache_item=True): ''' We use pipelining for doing multiple adds Alternatively we could also send multiple adds to one call. Don't see a reason for that though ''' value_score_pairs = [] key_value_pairs = [] for activity in activities: value = self.serialize_activity(activity) score = self.get_activity_score(activity) #if its real data write the id to the redis hash cache if not isinstance(activity, FeedEndMarker): key_value_pairs.append((activity.serialization_id, value)) value_score_pairs.append((activity.serialization_id, score)) # we need to do this sequentially, otherwise there's a risk of broken reads if cache_item: self.item_cache.set_many(key_value_pairs) else: logger.debug('skipping item cache write') results = RedisSortedSetCache.add_many(self, value_score_pairs) #make sure we trim to max length self.trim() return results
def __init__(self, user_id, redis=None, max_length=None): ''' ''' from feedly.feed_managers.love_feedly import LoveFeedly self.manager = LoveFeedly RedisSortedSetCache.__init__(self, user_id, redis=redis) #input validation if not isinstance(user_id, int): raise ValueError('user id should be an int, found %r' % user_id) #support for different serialization schemes self.serializer = self.serializer_class() #support for pipelining redis self.user_id = user_id self.item_cache = LoveFeedItemCache('global') self.key = self.key_format % user_id self._max_length = max_length
def remove_many(self, activities): ''' Efficiently remove many activities ''' values = [] for activity in activities: values.append(activity.serialization_id) results = RedisSortedSetCache.remove_many(self, values) return results
def remove_many(self, aggregated_activities): ''' Efficiently remove many activities ''' scores = [] for activity in aggregated_activities: if not isinstance(activity, AggregatedActivity): raise ValueError('we can only remove aggregated activities') score = self.get_activity_score(activity) scores.append(score) results = RedisSortedSetCache.remove_by_scores(self, scores) return results
def mark_all(self, seen=True, read=None): ''' Mark all the entries as seen or read ''' # TODO refactor this code with self.redis.lock(self.lock_key, timeout=2): # get the current aggregated activities activities = self[:self.max_length] # create the update dict update_dict = {} for activity in activities: changed = False old_activity = copy.deepcopy(activity) if seen is True and not activity.is_seen(): activity.seen_at = datetime.datetime.today() changed = True if read is True and not activity.is_read(): activity.read_at = datetime.datetime.today() changed = True if changed: update_dict[old_activity] = activity # now add the new ones and remove the old ones in one atomic operation to_delete = [] to_add = [] for old, new in update_dict.items(): new_value = self.serialize_activity(new) new_score = self.get_activity_score(new) to_delete.append(old) to_add.append((new_value, new_score)) # pipeline all our writes to improve performance # Update: removed self.map(), multithreaded behaviour seems bugged if to_delete: delete_results = self.remove_many(to_delete) # add the data in batch if to_add: add_results = RedisSortedSetCache.add_many(self, to_add) # denormalize the count count = self.denormalize_count(activities) # return the new activities return activities
def get_results(self, start, stop): redis_results = RedisSortedSetCache.get_results(self, start, stop) enriched_results = self.deserialize_activities(redis_results) return enriched_results
def add_many(self, activities): ''' Note this function is very specific to notifications, this won't get you good performance characteristics in applications with longer lists Add many works as follows: - retrieve all aggregated activities - add the new activities to the existing ones - update the values in Redis by sending several deletes and adds Trim the sorted set to max length Denormalize the unseen count Send a pubsub publish ''' value_score_pairs = [] remove_activities = {} aggregator = self.get_aggregator() # first stick the new activities in groups aggregated_activities = aggregator.aggregate(activities) # get the current aggregated activities current_activities = self[:self.max_length] current_activities_dict = dict([(a.group, a) for a in current_activities]) # see what we need to update for activity in aggregated_activities: if activity.group in current_activities_dict: # update existing current_activity = current_activities_dict[activity.group] old_activity = copy.deepcopy(current_activity) for a in activity.activities: current_activity.append(a) new_activity = current_activity # we should only do this the first time, verify things go well if old_activity.group in remove_activities: raise ValueError('Thierry didnt expect this to happen') remove_activities[old_activity.group] = old_activity else: # create a new activity new_activity = activity current_activities.append(new_activity) # add the data to the to write list value = self.serialize_activity(new_activity) score = self.get_activity_score(new_activity) value_score_pairs.append((value, score)) # pipeline all our writes to improve performance # TODO: removed map just to be sure # first remove the old notifications delete_results = self.remove_many(remove_activities.values()) # add the data in batch add_results = RedisSortedSetCache.add_many(self, value_score_pairs) # make sure we trim to max length trim_result = self.trim() # return the current state of the notification feed return current_activities
def add_many(self, activities): ''' Note this function is very specific to notifications, this won't get you good performance characteristics in applications with longer lists Add many works as follows: - retrieve all aggregated activities - add the new activities to the existing ones - update the values in Redis by sending several deletes and adds Trim the sorted set to max length Denormalize the unseen count Send a pubsub publish ''' value_score_pairs = [] remove_activities = {} aggregator = self.get_aggregator() # first stick the new activities in groups aggregated_activities = aggregator.aggregate(activities) # get the current aggregated activities current_activities = self[:self.max_length] current_activities_dict = dict( [(a.group, a) for a in current_activities]) # see what we need to update for activity in aggregated_activities: if activity.group in current_activities_dict: # update existing current_activity = current_activities_dict[activity.group] old_activity = copy.deepcopy(current_activity) for a in activity.activities: current_activity.append(a) new_activity = current_activity # we should only do this the first time, verify things go well if old_activity.group in remove_activities: raise ValueError('Thierry didnt expect this to happen') remove_activities[old_activity.group] = old_activity else: # create a new activity new_activity = activity current_activities.append(new_activity) # add the data to the to write list value = self.serialize_activity(new_activity) score = self.get_activity_score(new_activity) value_score_pairs.append((value, score)) # pipeline all our writes to improve performance # TODO: removed map just to be sure # first remove the old notifications delete_results = self.remove_many(remove_activities.values()) # add the data in batch add_results = RedisSortedSetCache.add_many(self, value_score_pairs) # make sure we trim to max length trim_result = self.trim() # return the current state of the notification feed return current_activities