def add_many(self,
activities,
batch_interface=None,
trim=True,
*args,
**kwargs):
'''
Add many activities
:param activities: a list of activities
:param batch_interface: the batch interface
'''
validate_list_of_strict(activities,
(self.activity_class, FakeActivity))
add_count = self.timeline_storage.add_many(
self.key,
activities,
batch_interface=batch_interface,
*args,
**kwargs)
# trim the feed sometimes
if trim and random.random() <= self.trim_chance:
self.trim()
self.on_update_feed(new=activities, deleted=[])
return add_count
def remove_many(self,
activities,
batch_interface=None,
trim=True,
*args,
**kwargs):
'''
Removes many activities from the feed
:param activities: the list of activities to remove
'''
validate_list_of_strict(activities,
(self.activity_class, FakeActivity))
# trim to make sure nothing we don't need is stored after the max
# length
self.trim()
# now we only have to look at max length
current_activities = self.get_activity_slice(stop=self.max_length,
rehydrate=False)
# setup our variables
new, deleted, changed = [], [], []
activities_to_remove = set(a.serialization_id for a in activities)
activity_dict = dict((a.serialization_id, a) for a in activities)
# first built the activity lookup dict
activity_remove_dict = defaultdict(list)
for aggregated in current_activities:
for activity_id in aggregated.activity_ids:
if activity_id in activities_to_remove:
activity_remove_dict[aggregated].append(activity_id)
activities_to_remove.discard(activity_id)
# stop searching when we have all of the activities to remove
if not activities_to_remove:
break
# stick the activities to remove in changed or remove
hydrated_aggregated = activity_remove_dict.keys()
if self.needs_hydration(hydrated_aggregated):
hydrated_aggregated = self.hydrate_activities(hydrated_aggregated)
hydrate_dict = dict((a.group, a) for a in hydrated_aggregated)
for aggregated, activity_ids_to_remove in activity_remove_dict.items():
aggregated = hydrate_dict.get(aggregated.group)
if len(aggregated) == len(activity_ids_to_remove):
deleted.append(aggregated)
else:
original = copy.deepcopy(aggregated)
activities_to_remove = map(activity_dict.get,
activity_ids_to_remove)
aggregated.remove_many(activities_to_remove)
changed.append((original, aggregated))
# new ones we insert, changed we do a delete and insert
new_aggregated = self._update_from_diff(new, changed, deleted)
return new_aggregated
def add_many_aggregated(self, aggregated, *args, **kwargs):
'''
Adds the list of aggregated activities
:param aggregated: the list of aggregated activities to add
'''
validate_list_of_strict(
aggregated, (self.aggregated_activity_class, FakeAggregatedActivity))
self.timeline_storage.add_many(self.key, aggregated, *args, **kwargs)
def add_many_aggregated(self, aggregated, *args, **kwargs):
'''
Adds the list of aggregated activities
:param aggregated: the list of aggregated activities to add
'''
validate_list_of_strict(
aggregated, (self.aggregated_activity_class, FakeAggregatedActivity))
self.timeline_storage.add_many(self.key, aggregated, *args, **kwargs)
def remove_many(self, activities, batch_interface=None, trim=True, *args, **kwargs):
'''
Removes many activities from the feed
:param activities: the list of activities to remove
'''
validate_list_of_strict(
activities, (self.activity_class, FakeActivity, long))
# trim to make sure nothing we don't need is stored after the max
# length
self.trim()
# now we only have to look at max length
current_activities = self.get_activity_slice(
stop=self.max_length, rehydrate=False)
# setup our variables
new, deleted, changed = [], [], []
getid = lambda a: getattr(a, 'serialization_id', a)
activities_to_remove = set(getid(a) for a in activities)
activity_dict = dict((getid(a), a) for a in activities)
# first built the activity lookup dict
activity_remove_dict = defaultdict(list)
for aggregated in current_activities:
for activity_id in aggregated.activity_ids:
if activity_id in activities_to_remove:
activity_remove_dict[aggregated].append(activity_id)
activities_to_remove.discard(activity_id)
# stop searching when we have all of the activities to remove
if not activities_to_remove:
break
# stick the activities to remove in changed or remove
hydrated_aggregated = activity_remove_dict.keys()
if self.needs_hydration(hydrated_aggregated):
hydrated_aggregated = self.hydrate_activities(hydrated_aggregated)
hydrate_dict = dict((a.group, a) for a in hydrated_aggregated)
for aggregated, activity_ids_to_remove in activity_remove_dict.items():
aggregated = hydrate_dict.get(aggregated.group)
if len(aggregated) == len(activity_ids_to_remove):
deleted.append(aggregated)
else:
original = copy.deepcopy(aggregated)
activities_to_remove = map(
activity_dict.get, activity_ids_to_remove)
aggregated.remove_many(activities_to_remove)
changed.append((original, aggregated))
# new ones we insert, changed we do a delete and insert
new_aggregated = self._update_from_diff(new, changed, deleted)
return new_aggregated
def add_many(self, activities, batch_interface=None, trim=True, *args, **kwargs):
'''
Add many activities
:param activities: a list of activities
:param batch_interface: the batch interface
'''
validate_list_of_strict(activities, (self.activity_class, FakeActivity))
add_count = self.timeline_storage.add_many(
self.key, activities, batch_interface=batch_interface, *args, **kwargs)
# trim the feed sometimes
if trim and random.random() <= self.trim_chance:
self.trim()
return add_count
def add_many(self,
activities,
trim=True,
current_activities=None,
*args,
**kwargs):
'''
Adds many activities to the feed
Unfortunately we can't support the batch interface.
The writes depend on the reads.
Also subsequent writes will depend on these writes.
So no batching is possible at all.
:param activities: the list of activities
'''
validate_list_of_strict(activities,
(self.activity_class, FakeActivity))
# start by getting the aggregator
aggregator = self.get_aggregator()
t = timer()
# get the current aggregated activities
if current_activities is None:
current_activities = self[:self.merge_max_length]
msg_format = 'reading %s items took %s'
logger.debug(msg_format, self.merge_max_length, t.next())
# merge the current activities with the new ones
new, changed, deleted = aggregator.merge(current_activities,
activities)
logger.debug('merge took %s', t.next())
# new ones we insert, changed we do a delete and insert
new_aggregated = self._update_from_diff(new, changed, deleted)
new_aggregated = aggregator.rank(new_aggregated)
# trim every now and then
if trim and random.random() <= self.trim_chance:
self.timeline_storage.trim(self.key, self.max_length)
return new_aggregated
def add_many(self, activities, trim=True, current_activities=None, *args, **kwargs):
'''
Adds many activities to the feed
Unfortunately we can't support the batch interface.
The writes depend on the reads.
Also subsequent writes will depend on these writes.
So no batching is possible at all.
:param activities: the list of activities
'''
validate_list_of_strict(
activities, (self.activity_class, FakeActivity))
# start by getting the aggregator
aggregator = self.get_aggregator()
t = timer()
# get the current aggregated activities
if current_activities is None:
current_activities = self[:self.merge_max_length]
msg_format = 'reading %s items took %s'
logger.debug(msg_format, self.merge_max_length, t.next())
# merge the current activities with the new ones
new, changed, deleted = aggregator.merge(
current_activities, activities)
logger.debug('merge took %s', t.next())
# new ones we insert, changed we do a delete and insert
new_aggregated = self._update_from_diff(new, changed, deleted)
new_aggregated = aggregator.rank(new_aggregated)
# trim every now and then
if trim and random.random() <= self.trim_chance:
self.timeline_storage.trim(self.key, self.max_length)
return new_aggregated
