def es_shops(shopids, date=None):
try:
if date is None:
date = defaultdate
si = ShopIndex(date)
for shopid in shopids:
try:
es_shop(si, date, shopid)
except:
traceback.print_exc()
flush()
except:
traceback.print_exc()
def es_brands(brands, date=None):
try:
if date is None:
date = defaultdate
bi = BrandIndex(date)
for brand in brands:
try:
es_brand(bi, date, brand)
except:
traceback.print_exc()
flush()
except:
traceback.print_exc()
def es_brands(brands, date=None):
try:
if date is None:
date = defaultdate
bi = BrandIndex(date)
for brand in brands:
try:
es_brand(bi, date, brand)
except:
traceback.print_exc()
flush()
except:
traceback.print_exc()
def es_shops(shopids, date=None):
try:
if date is None:
date = defaultdate
si = ShopIndex(date)
for shopid in shopids:
try:
es_shop(si, date, shopid)
except:
traceback.print_exc()
flush()
except:
traceback.print_exc()
class ShopESProcess(Process):
def __init__(self, date=None):
super(ShopESProcess, self).__init__('shopes')
if ENV == 'DEV':
self.step = 2**64 / 5
self.max_workers = 5
else:
self.step = 2**64 / 10000
self.max_workers = 100
self.date = date
def generate_tasks(self):
self.clear_redis()
ts = ShopIndex(self.date).allshopids
for bucket in ts.conn.smembers(ts.bucketskey):
shopids = ts.conn.smembers(bucket)
shopids = [int(id) for id in shopids]
self.add_task('aggregator.shopes.es_shops',
shopids,
date=self.date)
self.finish_generation()
sep = ShopESProcess()
if __name__ == '__main__':
#sep.date = '2013-12-18'
#sep.start()
es_shops([60463921], date='2014-01-14')
flush()
self.date = date
def generate_tasks(self):
self.clear_redis()
bi = BrandIndex(self.date)
ci = CategoryIndex(self.date)
# from aggregator.brands import brands as brands1
# brands2 = set(b.decode('utf-8') for b in bi.getbrands())
# brands = list(brands1 & brands2)
# for i in range(len(brands)/self.step):
# self.add_task('aggregator.brandes.es_brands', brands[self.step*i:self.step*(i+1)], date=self.date)
allbrands = set()
for cate1, cate2 in l1l2s:
brands1 = ci.getbrandnames(cate1, cate2)
brands = list(brands1 - allbrands)
allbrands.update(brands1)
for i in range(1 + len(brands) / self.step):
bs = brands[i * self.step : (i + 1) * self.step]
self.add_task("aggregator.brandes.es_brands", bs, date=self.date)
self.finish_generation()
bep = BrandESProcess()
if __name__ == "__main__":
# bep.date = '2013-12-25'
# bep.start()
es_brands(["SAMSUNG/三星"], date="2014-01-14")
flush()
refresh()