示例#1
0
def es_shops(shopids, date=None):
    try:
        if date is None:
            date = defaultdate
        si = ShopIndex(date)
        for shopid in shopids:
            try:
                es_shop(si, date, shopid)
            except:
                traceback.print_exc()
        flush()
    except:
        traceback.print_exc()
示例#2
0
def es_brands(brands, date=None):
    try:
        if date is None:
            date = defaultdate
        bi = BrandIndex(date)
        for brand in brands:
            try:
                es_brand(bi, date, brand)
            except:
                traceback.print_exc()
        flush()
    except:
        traceback.print_exc()
示例#3
0
def es_brands(brands, date=None):
    try:
        if date is None:
            date = defaultdate
        bi = BrandIndex(date)
        for brand in brands:
            try:
                es_brand(bi, date, brand)
            except:
                traceback.print_exc()
        flush()
    except:
        traceback.print_exc()
示例#4
0
def es_shops(shopids, date=None):
    try:
        if date is None:
            date = defaultdate
        si = ShopIndex(date)
        for shopid in shopids:
            try:
                es_shop(si, date, shopid)
            except:
                traceback.print_exc()
        flush()
    except:
        traceback.print_exc()
示例#5
0
class ShopESProcess(Process):
    def __init__(self, date=None):
        super(ShopESProcess, self).__init__('shopes')
        if ENV == 'DEV':
            self.step = 2**64 / 5
            self.max_workers = 5
        else:
            self.step = 2**64 / 10000
            self.max_workers = 100
        self.date = date

    def generate_tasks(self):
        self.clear_redis()
        ts = ShopIndex(self.date).allshopids
        for bucket in ts.conn.smembers(ts.bucketskey):
            shopids = ts.conn.smembers(bucket)
            shopids = [int(id) for id in shopids]
            self.add_task('aggregator.shopes.es_shops',
                          shopids,
                          date=self.date)
        self.finish_generation()


sep = ShopESProcess()

if __name__ == '__main__':
    #sep.date = '2013-12-18'
    #sep.start()
    es_shops([60463921], date='2014-01-14')
    flush()
示例#6
0
        self.date = date

    def generate_tasks(self):
        self.clear_redis()
        bi = BrandIndex(self.date)
        ci = CategoryIndex(self.date)
        # from aggregator.brands import brands as brands1
        # brands2 = set(b.decode('utf-8') for b in bi.getbrands())
        # brands = list(brands1 & brands2)
        # for i in range(len(brands)/self.step):
        #     self.add_task('aggregator.brandes.es_brands', brands[self.step*i:self.step*(i+1)], date=self.date)
        allbrands = set()
        for cate1, cate2 in l1l2s:
            brands1 = ci.getbrandnames(cate1, cate2)
            brands = list(brands1 - allbrands)
            allbrands.update(brands1)
            for i in range(1 + len(brands) / self.step):
                bs = brands[i * self.step : (i + 1) * self.step]
                self.add_task("aggregator.brandes.es_brands", bs, date=self.date)
        self.finish_generation()


bep = BrandESProcess()

if __name__ == "__main__":
    # bep.date = '2013-12-25'
    # bep.start()
    es_brands(["SAMSUNG/三星"], date="2014-01-14")
    flush()
    refresh()