import time
import json
from bitdeli import profile_events
from bitdeli.protocol import params
from bitdeli.chunkedlist import ChunkedList
PROFILE_RETENTION = params()['plan']['retention-days']
now = time.time() * 1000
for profile, sessions in profile_events():
psessions = profile.get('sessions')
if psessions == None:
psessions = profile['sessions'] = ChunkedList()
psessions.drop_chunks(lambda x:\
now - json.loads(x.data)['t'] <= PROFILE_RETENTION)
psessions.push(s.object for s in sessions)
profile.set_expire(PROFILE_RETENTION)
from bitdeli import profile_events
from bitdeli.protocol import params, done
from bitdeli.chunkedlist import ChunkedList
from collections import Counter
from itertools import groupby, islice
from operator import itemgetter
import json
PARAMS = params()
PROPERTIES_RETENTION = PARAMS['plan']['retention-days'] * 24
PROFILE_RETENTION = PARAMS['plan']['retention-days']
DROP_PROPERTIES_INTERVAL = 5
MAX_VALUES_PER_PROPERTY = 10
def parse(events):
for event in events:
e = json.loads(event.object.data)
prop = e['properties']
prop.pop('distinct_id', None)
hour = prop.pop('time') / 3600
yield hour, e['event'], prop
def push(d, key, hour, count):
key = key if isinstance(key, basestring) else str(key)
if key in d:
c = d[key]
else:
c = d[key] = ChunkedList()
c.push([(hour, count)])
def drop_old_properties(now, profile):
from bitdeli import profile_events
from bitdeli.protocol import params
PARAMS = params()
PROFILE_RETENTION = PARAMS['plan']['retention-days']
MAX_EVENTS = PARAMS['plan']['num-raw-events']
for profile, events in profile_events():
pevents = profile.get('events', [])
pevents.extend((e.timestamp, e.groupkey, e.ip, e.object) for e in events)
profile['events'] = pevents[-MAX_EVENTS:]
profile.set_expire(PROFILE_RETENTION)
