def _get_newuser_bitmap(self, day=None, Type='dau'): """ 返回新用户当日登陆记录bitmap对象 适配记录每日新用户id偏移量的数据存储方式 """ day = day or self.baseDay offsets = 0 dauBitmap = Bitmap() if day: if Type == 'mau': day = datetime(day.year, day.month, 1) hKey = (self.config.dau_keys_conf['newuser'], day.strftime(self.config.DATE_FORMAT)) # dauBitmap = self._get_cache(hKey) or dauBitmap # cached = self._get_cache(hKey) # if cached is None or True: if True: offsets = self.REDIS.hget(*hKey) if not offsets: return dauBitmap offsets = int(offsets) bmp = ((day == self.baseDay and Type == 'dau') and Bitmap(self.baseBitmap) or self._make_bitmap(day, Type)) s = time.time() dauBitmap = Bitmap(bmp) # 生成新的实例,避免被篡改 dauBitmap[:offsets] = False # self._cache(hKey, dauBitmap) else: dauBitmap = cached logging.debug('get nu bitmap: %s Sec' % (time.time() - s)) return dauBitmap
def _get_filter_bitmap(self, redis_cli, filtername, filterclass): """ 由数据源获取筛选条件 BitMap """ logging.debug(self.redis_cli) redis_cli = redis_cli or self.redis_cli if not isinstance(redis_cli, redis.client.Redis): raise TypeError, "Need redis connection but not found" fKey_format = self.config.filter_keys_conf.get(filtername) if not fKey_format: raise ValueError, "Can not find the key \'%s\' in self.config.filter_keys_conf" % k logging.debug('%s, %s, %s', fKey_format, filtername, filterclass) fKey = fKey_format.format(**{filtername: filterclass}) cached = self._cache.get_cache(fKey) if cached is None: fBits = redis_cli.get(fKey) fBm = Bitmap() if fBits: fBm.frombytes(fBits) else: fBm = Bitmap('0') self._cache.set(fKey, fBm) else: fBm = cached return fBm
def _make_bitmap(self, day=None, Type='dau'): """ initial and return dwarf.Bitmap object """ # logging.info('make_bitmap:%s %s', day, Type) s = time.time() dauBitmap = Bitmap() if day: DAU_KEY = self.config.dau_keys_conf[Type] if Type in ('mau', 'mnu', 'mru'): dauKey = DAU_KEY.format( month=day.strftime(self.config.MONTH_FORMAT)) else: dauKey = DAU_KEY.format( date=day.strftime(self.config.DATE_FORMAT)) dauBitmap = Bitmap() cached = self._get_cache(dauKey) if cached: bitsDau = cached if cached is 'None': bitsDau = None else: logging.debug('no cache:%s', dauKey) bitsDau = self.REDIS.get(dauKey) self._cache(dauKey, bitsDau or 'None') if bitsDau: dauBitmap.frombytes(bitsDau) # logging.debug('Init bitmap:Count: %s' % (dauBitmap.count())) if self.filters: dauBitmap.filter(self.filters) # logging.info('Filter bitmap: f-%s b-%s' % (self.filters.count(), dauBitmap.count())) # self._cache(dauKey, dauBitmap) logging.debug('_make_bitmap Handler:%s %s - %s Sec' % (day, Type, time.time() - s)) return dauBitmap
def __init__(self, baseday=None, redis_cli=None, filters=None, cache=True, config=None): s = time.time() # logging.info('init austat: baseday %s, config: %s', baseday, config and config.dau_keys_conf or '') if not redis_cli: raise KeyError, 'Redis connection not found' if not config: self.config = dauconfig else: self.config = config self.cache = _Cache() # self._cache_dict = {} # self._max_cache_lens = 1024 self.REDIS = redis_cli self.baseDay = baseday self.filters = filters self._is_cache = cache self.baseBitmap = self._make_bitmap(baseday) self.newUserBitmap = Bitmap()