def add_many(self, activities, batch_interface=None, trim=True, *args, **kwargs): ''' Add many activities :param activities: a list of activities :param batch_interface: the batch interface ''' validate_list_of_strict(activities, (self.activity_class, FakeActivity)) add_count = self.timeline_storage.add_many( self.key, activities, batch_interface=batch_interface, *args, **kwargs) # trim the feed sometimes if trim and random.random() <= self.trim_chance: self.trim() self.on_update_feed(new=activities, deleted=[]) return add_count
def remove_many(self, activities, batch_interface=None, trim=True, *args, **kwargs): ''' Removes many activities from the feed :param activities: the list of activities to remove ''' validate_list_of_strict(activities, (self.activity_class, FakeActivity)) # trim to make sure nothing we don't need is stored after the max # length self.trim() # now we only have to look at max length current_activities = self.get_activity_slice(stop=self.max_length, rehydrate=False) # setup our variables new, deleted, changed = [], [], [] activities_to_remove = set(a.serialization_id for a in activities) activity_dict = dict((a.serialization_id, a) for a in activities) # first built the activity lookup dict activity_remove_dict = defaultdict(list) for aggregated in current_activities: for activity_id in aggregated.activity_ids: if activity_id in activities_to_remove: activity_remove_dict[aggregated].append(activity_id) activities_to_remove.discard(activity_id) # stop searching when we have all of the activities to remove if not activities_to_remove: break # stick the activities to remove in changed or remove hydrated_aggregated = activity_remove_dict.keys() if self.needs_hydration(hydrated_aggregated): hydrated_aggregated = self.hydrate_activities(hydrated_aggregated) hydrate_dict = dict((a.group, a) for a in hydrated_aggregated) for aggregated, activity_ids_to_remove in activity_remove_dict.items(): aggregated = hydrate_dict.get(aggregated.group) if len(aggregated) == len(activity_ids_to_remove): deleted.append(aggregated) else: original = copy.deepcopy(aggregated) activities_to_remove = map(activity_dict.get, activity_ids_to_remove) aggregated.remove_many(activities_to_remove) changed.append((original, aggregated)) # new ones we insert, changed we do a delete and insert new_aggregated = self._update_from_diff(new, changed, deleted) return new_aggregated
def add_many_aggregated(self, aggregated, *args, **kwargs): ''' Adds the list of aggregated activities :param aggregated: the list of aggregated activities to add ''' validate_list_of_strict( aggregated, (self.aggregated_activity_class, FakeAggregatedActivity)) self.timeline_storage.add_many(self.key, aggregated, *args, **kwargs)
def remove_many(self, activities, batch_interface=None, trim=True, *args, **kwargs): ''' Removes many activities from the feed :param activities: the list of activities to remove ''' validate_list_of_strict( activities, (self.activity_class, FakeActivity, long)) # trim to make sure nothing we don't need is stored after the max # length self.trim() # now we only have to look at max length current_activities = self.get_activity_slice( stop=self.max_length, rehydrate=False) # setup our variables new, deleted, changed = [], [], [] getid = lambda a: getattr(a, 'serialization_id', a) activities_to_remove = set(getid(a) for a in activities) activity_dict = dict((getid(a), a) for a in activities) # first built the activity lookup dict activity_remove_dict = defaultdict(list) for aggregated in current_activities: for activity_id in aggregated.activity_ids: if activity_id in activities_to_remove: activity_remove_dict[aggregated].append(activity_id) activities_to_remove.discard(activity_id) # stop searching when we have all of the activities to remove if not activities_to_remove: break # stick the activities to remove in changed or remove hydrated_aggregated = activity_remove_dict.keys() if self.needs_hydration(hydrated_aggregated): hydrated_aggregated = self.hydrate_activities(hydrated_aggregated) hydrate_dict = dict((a.group, a) for a in hydrated_aggregated) for aggregated, activity_ids_to_remove in activity_remove_dict.items(): aggregated = hydrate_dict.get(aggregated.group) if len(aggregated) == len(activity_ids_to_remove): deleted.append(aggregated) else: original = copy.deepcopy(aggregated) activities_to_remove = map( activity_dict.get, activity_ids_to_remove) aggregated.remove_many(activities_to_remove) changed.append((original, aggregated)) # new ones we insert, changed we do a delete and insert new_aggregated = self._update_from_diff(new, changed, deleted) return new_aggregated
def add_many(self, activities, batch_interface=None, trim=True, *args, **kwargs): ''' Add many activities :param activities: a list of activities :param batch_interface: the batch interface ''' validate_list_of_strict(activities, (self.activity_class, FakeActivity)) add_count = self.timeline_storage.add_many( self.key, activities, batch_interface=batch_interface, *args, **kwargs) # trim the feed sometimes if trim and random.random() <= self.trim_chance: self.trim() return add_count
def add_many(self, activities, trim=True, current_activities=None, *args, **kwargs): ''' Adds many activities to the feed Unfortunately we can't support the batch interface. The writes depend on the reads. Also subsequent writes will depend on these writes. So no batching is possible at all. :param activities: the list of activities ''' validate_list_of_strict(activities, (self.activity_class, FakeActivity)) # start by getting the aggregator aggregator = self.get_aggregator() t = timer() # get the current aggregated activities if current_activities is None: current_activities = self[:self.merge_max_length] msg_format = 'reading %s items took %s' logger.debug(msg_format, self.merge_max_length, t.next()) # merge the current activities with the new ones new, changed, deleted = aggregator.merge(current_activities, activities) logger.debug('merge took %s', t.next()) # new ones we insert, changed we do a delete and insert new_aggregated = self._update_from_diff(new, changed, deleted) new_aggregated = aggregator.rank(new_aggregated) # trim every now and then if trim and random.random() <= self.trim_chance: self.timeline_storage.trim(self.key, self.max_length) return new_aggregated
def add_many(self, activities, trim=True, current_activities=None, *args, **kwargs): ''' Adds many activities to the feed Unfortunately we can't support the batch interface. The writes depend on the reads. Also subsequent writes will depend on these writes. So no batching is possible at all. :param activities: the list of activities ''' validate_list_of_strict( activities, (self.activity_class, FakeActivity)) # start by getting the aggregator aggregator = self.get_aggregator() t = timer() # get the current aggregated activities if current_activities is None: current_activities = self[:self.merge_max_length] msg_format = 'reading %s items took %s' logger.debug(msg_format, self.merge_max_length, t.next()) # merge the current activities with the new ones new, changed, deleted = aggregator.merge( current_activities, activities) logger.debug('merge took %s', t.next()) # new ones we insert, changed we do a delete and insert new_aggregated = self._update_from_diff(new, changed, deleted) new_aggregated = aggregator.rank(new_aggregated) # trim every now and then if trim and random.random() <= self.trim_chance: self.timeline_storage.trim(self.key, self.max_length) return new_aggregated