async def exec_data(item, cids, semaphore): async with semaphore: goods_id = item.get('product_id') if not goods_id: return sell_num = int(item.get('sell_num')) shop_id = item.get('shop_id') await check_shop(shop_id) goods_price = item.get('discount_price') / 100 goods_name = item.get('name') cid = item.get('third_cid') if not cids.__contains__(cid): cid = item.get('second_cid') goods_picture_url = item.get('img') goods_url = 'https://haohuo.snssdk.com/views/product/item?id=' + goods_id is_add = False goods = await Goods.find_one('goods_id=?', goods_id) if goods: # 修改 time_now = datetime.datetime.now().strftime("%Y-%m-%d") time_last_edit = goods.edit_time.strftime("%Y-%m-%d") # 较上次增量 add_num = sell_num - goods.sell_num goods.shop_id = shop_id goods.cid = cid goods.goods_name = goods_name goods.goods_url = goods_url goods.goods_picture_url = goods_picture_url goods.goods_price = goods_price if time_now != time_last_edit: goods.add_num = 0 elif add_num >= 0: goods.add_num = goods.add_num + add_num goods.sell_num = sell_num if goods.item_last_sell_num is None: goods.item_last_sell_num = goods.sell_num goods.edit_time = datetime.datetime.now() item_add_num = sell_num - goods.item_last_sell_num if item_add_num > 100: goods.item_last_sell_num = sell_num await goods.update() else: # 新增 is_add = True item_add_num = 0 goods = Goods() goods.shop_id = shop_id goods.goods_id = goods_id goods.goods_name = goods_name goods.goods_url = goods_url goods.goods_picture_url = goods_picture_url goods.goods_price = goods_price goods.cid = cid goods.add_num = 0 goods.sell_num = sell_num goods.item_last_sell_num = sell_num goods.id = await goods.save() if goods.id == 0: is_add = False if is_add or item_add_num >= 100: goods_item = Goods_Item() goods_item.goods_id = goods.id goods_item.sell_num = sell_num goods_item.add_num = item_add_num await goods_item.save() if goods.add_num > 0: await Goods_Tmp.del_by('goods_id=?', goods.id) tmp = Goods_Tmp() tmp.goods_id = goods.id tmp.add_num = goods.add_num tmp.sell_num = goods.sell_num tmp.edit_time = datetime.datetime.now() await tmp.save()
tmp.sell_num = goods.sell_num tmp.edit_time = datetime.datetime.now() await tmp.save() # 按照给定时间更新 if __name__ == '__main__': query_time = str(sys.argv[1]) # query_time = '2019-06-14 09:00:00' print(query_time) start = datetime.datetime.now() loop = asyncio.get_event_loop() loop.run_until_complete(orm.create_pool(loop=loop, **configs.db)) # and is_selling=? ,True goods = loop.run_until_complete(Goods.findAll('edit_time<?', query_time)) q_goods = queue.Queue(maxsize=0) for good in goods: q_goods.put_nowait(good.goods_id) print("商品总数%s" % q_goods.qsize()) goods_id_object = tools.list_to_dict(goods, "goods_id") category_cids = loop.run_until_complete(Category_Cid.findAll()) cids = [] for category_cid in category_cids: if not cids.__contains__(category_cid.cid): cids.append(category_cid.cid) # 初始化 q_data = queue.Queue(maxsize=30000) global_goods_ids = []
shops = loop.run_until_complete(Shop.findAll()) # shops = loop.run_until_complete(Shop.findAll('shop_id=?', 'hmSuxrl')) q_shops = queue.Queue(maxsize=0) for shop in shops: q_shops.put_nowait(shop) print("店铺总数%s" % q_shops.qsize()) category_cids = loop.run_until_complete(Category_Cid.findAll()) cids = [] for category_cid in category_cids: if not cids.__contains__(category_cid.cid): cids.append(category_cid.cid) shop_id_object = tools.list_to_dict(shops, 'shop_id') all_goods = loop.run_until_complete(Goods.findAll()) goods_id_object = tools.list_to_dict(all_goods, "goods_id") # 初始化 q_data = queue.Queue(maxsize=30000) global_goods_ids = [] event = threading.Event() lock = threading.Lock() if event.isSet: event.clear() for i in range(400): p = shop_producer.Producer(i, q_shops, q_data, event, global_goods_ids) p.start()
tmp = Goods_Tmp() tmp.goods_id = goods.id tmp.add_num = goods.add_num tmp.sell_num = goods.sell_num tmp.edit_time = datetime.datetime.now() await tmp.save() # 更新当日热销 if __name__ == '__main__': start = datetime.datetime.now() loop = asyncio.get_event_loop() loop.run_until_complete(orm.create_pool(loop=loop, **configs.db)) goods = loop.run_until_complete( Goods.find_inner(tools.get_temp_table(), 'goods_id')) q_goods = queue.Queue(maxsize=0) for good in goods: q_goods.put_nowait(good.goods_id) print("商品总数%s" % q_goods.qsize()) goods_id_object = tools.list_to_dict(goods, "goods_id") category_cids = loop.run_until_complete(Category_Cid.findAll()) cids = [] for category_cid in category_cids: if not cids.__contains__(category_cid.cid): cids.append(category_cid.cid) # 初始化 q_data = queue.Queue(maxsize=30000) global_goods_ids = []