def _get_filter_bitmap(self, redis_cli, filtername, filterclass):
"""
由数据源获取筛选条件 BitMap
"""
logging.debug(self.redis_cli)
redis_cli = redis_cli or self.redis_cli
if not isinstance(redis_cli, redis.client.Redis):
raise TypeError, "Need redis connection but not found"
fKey_format = self.config.filter_keys_conf.get(filtername)
if not fKey_format:
raise ValueError, "Can not find the key \'%s\' in self.config.filter_keys_conf" % k
logging.debug('%s, %s, %s',fKey_format,filtername,filterclass)
fKey = fKey_format.format(**{filtername:filterclass})
cached = self._cache.get_cache(fKey)
if cached is None:
fBits = redis_cli.get(fKey)
fBm = Bitmap()
if fBits:
fBm.frombytes(fBits)
else:
fBm = Bitmap('0')
self._cache.set(fKey, fBm)
else:
fBm = cached
return fBm
def _get_filter_bitmap(self, redis_cli, filtername, filterclass):
"""
由数据源获取筛选条件 BitMap
"""
logging.debug(self.redis_cli)
redis_cli = redis_cli or self.redis_cli
if not isinstance(redis_cli, redis.client.Redis):
raise TypeError, "Need redis connection but not found"
fKey_format = self.config.filter_keys_conf.get(filtername)
if not fKey_format:
raise ValueError, "Can not find the key \'%s\' in self.config.filter_keys_conf" % k
logging.debug('%s, %s, %s', fKey_format, filtername, filterclass)
fKey = fKey_format.format(**{filtername: filterclass})
cached = self._cache.get_cache(fKey)
if cached is None:
fBits = redis_cli.get(fKey)
fBm = Bitmap()
if fBits:
fBm.frombytes(fBits)
else:
fBm = Bitmap('0')
self._cache.set(fKey, fBm)
else:
fBm = cached
return fBm
def _make_bitmap(self, day=None, Type='dau'):
"""
initial and return dwarf.Bitmap object
"""
# logging.info('make_bitmap:%s %s', day, Type)
s = time.time()
dauBitmap = Bitmap()
if day:
DAU_KEY = self.config.dau_keys_conf[Type]
if Type in ('mau','mnu','mru'):
dauKey = DAU_KEY.format(month=day.strftime(self.config.MONTH_FORMAT))
else:
dauKey = DAU_KEY.format(date=day.strftime(self.config.DATE_FORMAT))
dauBitmap = Bitmap()
cached = self._get_cache(dauKey)
if cached:
bitsDau = cached
if cached is 'None':
bitsDau = None
else:
logging.debug('no cache:%s',dauKey)
bitsDau = self.REDIS.get(dauKey)
self._cache(dauKey, bitsDau or 'None')
if bitsDau:
dauBitmap.frombytes(bitsDau)
# logging.debug('Init bitmap:Count: %s' % (dauBitmap.count()))
if self.filters:
dauBitmap.filter(self.filters)
# logging.info('Filter bitmap: f-%s b-%s' % (self.filters.count(), dauBitmap.count()))
# self._cache(dauKey, dauBitmap)
logging.debug('_make_bitmap Handler:%s %s - %s Sec' % (day,Type,time.time()-s))
return dauBitmap
def _make_bitmap(self, day=None, Type='dau'):
"""
initial and return dwarf.Bitmap object
"""
# logging.info('make_bitmap:%s %s', day, Type)
s = time.time()
dauBitmap = Bitmap()
if day:
DAU_KEY = self.config.dau_keys_conf[Type]
if Type in ('mau', 'mnu', 'mru'):
dauKey = DAU_KEY.format(
month=day.strftime(self.config.MONTH_FORMAT))
else:
dauKey = DAU_KEY.format(
date=day.strftime(self.config.DATE_FORMAT))
dauBitmap = Bitmap()
cached = self._get_cache(dauKey)
if cached:
bitsDau = cached
if cached is 'None':
bitsDau = None
else:
logging.debug('no cache:%s', dauKey)
bitsDau = self.REDIS.get(dauKey)
self._cache(dauKey, bitsDau or 'None')
if bitsDau:
dauBitmap.frombytes(bitsDau)
# logging.debug('Init bitmap:Count: %s' % (dauBitmap.count()))
if self.filters:
dauBitmap.filter(self.filters)
# logging.info('Filter bitmap: f-%s b-%s' % (self.filters.count(), dauBitmap.count()))
# self._cache(dauKey, dauBitmap)
logging.debug('_make_bitmap Handler:%s %s - %s Sec' %
(day, Type, time.time() - s))
return dauBitmap
