def flush_to_redis(dict_, name, time_): p = redis.pipeline() sum_keys = ['ts:%i:%i:%s:sum' % (interval, time_//interval, name) for interval in HASH_INTERVALS] len_keys = ['ts:%i:%i:%s:len' % (interval, time_//interval, name) for interval in HASH_INTERVALS] if len(dict_) > 0: hmincrbyfloat(sum_keys, dict_, client=p) hmincr(len_keys, dict_.keys(), client=p) for interval in HASH_INTERVALS: basekey = 'ts:%i:%i:%s' % (interval, time_//interval, name) if len(dict_) > 0: p.expire(basekey + ':sum', interval) p.expire(basekey + ':len', interval) prev = time_//interval - 1 prev_key = 'ts:%i:%i:%s' % (interval, prev, name) if redis.incr(prev_key + ':lock') == 1: hdivh(prev_key, prev_key + ':sum', prev_key + ':len', range(10240), format='%.2g', client=p) keys = setavgmax(prev_key, client=p) for k in keys: p.expire(k, HASH_EXPIRE*interval) p.expire(prev_key, HASH_EXPIRE*interval) p.expire(prev_key + ':lock', interval) p.execute()
def flush_cache(cache, cache_set, cache_nhit, cache_pmt, time): # for docs on redis pipeline see http://redis.io/topics/pipelining p = redis.pipeline() for name, hash in cache.items(): keys = ['ts:%i:%i:%s' % (interval, time//interval, name) for interval in INTERVALS] if len(hash) > 0: hmincrby(keys, hash, client=p) for key, interval in zip(keys,INTERVALS): p.expire(key,interval*EXPIRE) for interval in INTERVALS: for name, hash in cache_set.items(): key = 'ts:%i:%i:%s' % (interval, time//interval, name) if len(hash) > 0: p.hmset(key, hash) p.expire(key, interval*EXPIRE) keys = ['ts:%i:%i:occupancy:hits' % (interval, time//interval) for interval in HASH_INTERVALS] if len(cache_pmt) > 0: hmincrby(keys, cache_pmt, client=p) for interval in HASH_INTERVALS: key = 'ts:%i:%i:occupancy' % (interval, time//interval) p.incrby(key + ':count', cache['trig']['TOTAL']) # expire after just interval, because these will # occupancy will be set as hits/count p.expire(key + ':hits', interval) p.expire(key + ':count', interval) prev_key = 'ts:%i:%i:occupancy' % (interval,time//interval-1) if redis.incr(prev_key + ':lock') == 1: hdivk(prev_key, prev_key + ':hits', prev_key + ':count', range(10240), format='%.2g', client=p) keys = setavgmax(prev_key, client=p) for k in keys: p.expire(k, HASH_EXPIRE*interval) p.expire(prev_key, HASH_EXPIRE*interval) p.expire(prev_key + ':lock', interval) if len(cache_nhit) > 0: # nhit distribution if len(cache_nhit) > 100: # if there are more than 100 events this second # randomly sample the nhit from 100 events cache_nhit = random.sample(cache_nhit,100) # see http://flask.pocoo.org/snippets/71/ for this design pattern p.lpush('ts:1:%i:nhit' % time, *cache_nhit) p.expire('ts:1:%i:nhit' % time, 3600) p.execute()
def flush_cache(cache, cache_set, cache_nhit, cache_pmt, time): # for docs on redis pipeline see http://redis.io/topics/pipelining p = redis.pipeline() for name, hash in cache.items(): keys = [ 'ts:%i:%i:%s' % (interval, time // interval, name) for interval in INTERVALS ] if len(hash) > 0: hmincrby(keys, hash, client=p) for key, interval in zip(keys, INTERVALS): p.expire(key, interval * EXPIRE) for interval in INTERVALS: for name, hash in cache_set.items(): key = 'ts:%i:%i:%s' % (interval, time // interval, name) if len(hash) > 0: p.hmset(key, hash) p.expire(key, interval * EXPIRE) keys = [ 'ts:%i:%i:occupancy:hits' % (interval, time // interval) for interval in HASH_INTERVALS ] if len(cache_pmt) > 0: hmincrby(keys, cache_pmt, client=p) for interval in HASH_INTERVALS: key = 'ts:%i:%i:occupancy' % (interval, time // interval) p.incrby(key + ':count', cache['trig']['TOTAL']) # expire after just interval, because these will # occupancy will be set as hits/count p.expire(key + ':hits', interval) p.expire(key + ':count', interval) prev_key = 'ts:%i:%i:occupancy' % (interval, time // interval - 1) if redis.incr(prev_key + ':lock') == 1: hdivk(prev_key, prev_key + ':hits', prev_key + ':count', range(10240), format='%.2g', client=p) keys = setavgmax(prev_key, client=p) for k in keys: p.expire(k, HASH_EXPIRE * interval) p.expire(prev_key, HASH_EXPIRE * interval) p.expire(prev_key + ':lock', interval) if len(cache_nhit) > 0: # nhit distribution if len(cache_nhit) > 100: # if there are more than 100 events this second # randomly sample the nhit from 100 events cache_nhit = random.sample(cache_nhit, 100) # see http://flask.pocoo.org/snippets/71/ for this design pattern p.lpush('ts:1:%i:nhit' % time, *cache_nhit) p.expire('ts:1:%i:nhit' % time, 3600) p.execute()