import time import json from bitdeli import profile_events from bitdeli.protocol import params from bitdeli.chunkedlist import ChunkedList PROFILE_RETENTION = params()['plan']['retention-days'] now = time.time() * 1000 for profile, sessions in profile_events(): psessions = profile.get('sessions') if psessions == None: psessions = profile['sessions'] = ChunkedList() psessions.drop_chunks(lambda x:\ now - json.loads(x.data)['t'] <= PROFILE_RETENTION) psessions.push(s.object for s in sessions) profile.set_expire(PROFILE_RETENTION)
from bitdeli import profile_events from bitdeli.protocol import params, done from bitdeli.chunkedlist import ChunkedList from collections import Counter from itertools import groupby, islice from operator import itemgetter import json PARAMS = params() PROPERTIES_RETENTION = PARAMS['plan']['retention-days'] * 24 PROFILE_RETENTION = PARAMS['plan']['retention-days'] DROP_PROPERTIES_INTERVAL = 5 MAX_VALUES_PER_PROPERTY = 10 def parse(events): for event in events: e = json.loads(event.object.data) prop = e['properties'] prop.pop('distinct_id', None) hour = prop.pop('time') / 3600 yield hour, e['event'], prop def push(d, key, hour, count): key = key if isinstance(key, basestring) else str(key) if key in d: c = d[key] else: c = d[key] = ChunkedList() c.push([(hour, count)]) def drop_old_properties(now, profile):
from bitdeli import profile_events from bitdeli.protocol import params PARAMS = params() PROFILE_RETENTION = PARAMS['plan']['retention-days'] MAX_EVENTS = PARAMS['plan']['num-raw-events'] for profile, events in profile_events(): pevents = profile.get('events', []) pevents.extend((e.timestamp, e.groupkey, e.ip, e.object) for e in events) profile['events'] = pevents[-MAX_EVENTS:] profile.set_expire(PROFILE_RETENTION)