# MAX_VALUES_PER_PROPERTY items.sort() for time, value, counts in items[:pivot]: del values[value] # drop timestamps per PROPERTIES_RETENTION for time, value, counts in items[pivot:]: counts.drop_chunks(lambda x: now - x[0] <= PROPERTIES_RETENTION) if not counts: del values[value] if not values: empty_keys.append(key) for key in empty_keys: del properties[key] head = itemgetter(0) for profile, daily_events in profile_events(): properties = profile.setdefault('properties', {}) events = profile.setdefault('events', {}) hourly = sorted(parse(daily_events), key=head) for hour, hourly_events in groupby(hourly, head): event_counter = Counter() prop_counter = Counter() for hour, event_type, event_properties in hourly_events: event_counter[event_type] += 1 prop_counter.update((unicode(k), unicode(v))\ for k, v in event_properties.iteritems()) for event_type, count in event_counter.iteritems(): push(events, event_type, hour, count) for (prop_key, prop_value), count in prop_counter.iteritems(): push(properties.setdefault(prop_key, {}), prop_value, hour, count) drop_old_properties(hour, profile)
import time import json from bitdeli import profile_events from bitdeli.protocol import params from bitdeli.chunkedlist import ChunkedList PROFILE_RETENTION = params()['plan']['retention-days'] now = time.time() * 1000 for profile, sessions in profile_events(): psessions = profile.get('sessions') if psessions == None: psessions = profile['sessions'] = ChunkedList() psessions.drop_chunks(lambda x:\ now - json.loads(x.data)['t'] <= PROFILE_RETENTION) psessions.push(s.object for s in sessions) profile.set_expire(PROFILE_RETENTION)
from bitdeli import profile_events from bitdeli.chunkedlist import ChunkedList for profile, events in profile_events(): pevents = profile.get('commits') if not pevents: profile['commits'] = pevents = ChunkedList() pevents.push(e.object for e in events)
from bitdeli import profile_events from bitdeli.protocol import params PARAMS = params() PROFILE_RETENTION = PARAMS['plan']['retention-days'] MAX_EVENTS = PARAMS['plan']['num-raw-events'] for profile, events in profile_events(): pevents = profile.get('events', []) pevents.extend((e.timestamp, e.groupkey, e.ip, e.object) for e in events) profile['events'] = pevents[-MAX_EVENTS:] profile.set_expire(PROFILE_RETENTION)