def _get_newuser_bitmap(self, day=None, Type='dau'):
"""
返回新用户当日登陆记录bitmap对象
适配记录每日新用户id偏移量的数据存储方式
"""
day = day or self.baseDay
offsets = 0
dauBitmap = Bitmap()
if day:
if Type == 'mau':
day = datetime(day.year, day.month, 1)
hKey = (self.config.dau_keys_conf['newuser'],
day.strftime(self.config.DATE_FORMAT))
# dauBitmap = self._get_cache(hKey) or dauBitmap
# cached = self._get_cache(hKey)
# if cached is None or True:
if True:
offsets = self.REDIS.hget(*hKey)
if not offsets:
return dauBitmap
offsets = int(offsets)
bmp = ((day == self.baseDay and Type == 'dau')
and Bitmap(self.baseBitmap)
or self._make_bitmap(day, Type))
s = time.time()
dauBitmap = Bitmap(bmp) # 生成新的实例，避免被篡改
dauBitmap[:offsets] = False
# self._cache(hKey, dauBitmap)
else:
dauBitmap = cached
logging.debug('get nu bitmap: %s Sec' % (time.time() - s))
return dauBitmap
def _get_filter_bitmap(self, redis_cli, filtername, filterclass):
"""
由数据源获取筛选条件 BitMap
"""
logging.debug(self.redis_cli)
redis_cli = redis_cli or self.redis_cli
if not isinstance(redis_cli, redis.client.Redis):
raise TypeError, "Need redis connection but not found"
fKey_format = self.config.filter_keys_conf.get(filtername)
if not fKey_format:
raise ValueError, "Can not find the key \'%s\' in self.config.filter_keys_conf" % k
logging.debug('%s, %s, %s', fKey_format, filtername, filterclass)
fKey = fKey_format.format(**{filtername: filterclass})
cached = self._cache.get_cache(fKey)
if cached is None:
fBits = redis_cli.get(fKey)
fBm = Bitmap()
if fBits:
fBm.frombytes(fBits)
else:
fBm = Bitmap('0')
self._cache.set(fKey, fBm)
else:
fBm = cached
return fBm
def _make_bitmap(self, day=None, Type='dau'):
"""
initial and return dwarf.Bitmap object
"""
# logging.info('make_bitmap:%s %s', day, Type)
s = time.time()
dauBitmap = Bitmap()
if day:
DAU_KEY = self.config.dau_keys_conf[Type]
if Type in ('mau', 'mnu', 'mru'):
dauKey = DAU_KEY.format(
month=day.strftime(self.config.MONTH_FORMAT))
else:
dauKey = DAU_KEY.format(
date=day.strftime(self.config.DATE_FORMAT))
dauBitmap = Bitmap()
cached = self._get_cache(dauKey)
if cached:
bitsDau = cached
if cached is 'None':
bitsDau = None
else:
logging.debug('no cache:%s', dauKey)
bitsDau = self.REDIS.get(dauKey)
self._cache(dauKey, bitsDau or 'None')
if bitsDau:
dauBitmap.frombytes(bitsDau)
# logging.debug('Init bitmap:Count: %s' % (dauBitmap.count()))
if self.filters:
dauBitmap.filter(self.filters)
# logging.info('Filter bitmap: f-%s b-%s' % (self.filters.count(), dauBitmap.count()))
# self._cache(dauKey, dauBitmap)
logging.debug('_make_bitmap Handler:%s %s - %s Sec' %
(day, Type, time.time() - s))
return dauBitmap
def __init__(self,
baseday=None,
redis_cli=None,
filters=None,
cache=True,
config=None):
s = time.time()
# logging.info('init austat: baseday %s, config: %s', baseday, config and config.dau_keys_conf or '')
if not redis_cli:
raise KeyError, 'Redis connection not found'
if not config:
self.config = dauconfig
else:
self.config = config
self.cache = _Cache()
# self._cache_dict = {}
# self._max_cache_lens = 1024
self.REDIS = redis_cli
self.baseDay = baseday
self.filters = filters
self._is_cache = cache
self.baseBitmap = self._make_bitmap(baseday)
self.newUserBitmap = Bitmap()
