async def exec_data(item, cids, semaphore):
async with semaphore:
goods_id = item.get('product_id')
if not goods_id:
return
sell_num = int(item.get('sell_num'))
shop_id = item.get('shop_id')
await check_shop(shop_id)
goods_price = item.get('discount_price') / 100
goods_name = item.get('name')
cid = item.get('third_cid')
if not cids.__contains__(cid):
cid = item.get('second_cid')
goods_picture_url = item.get('img')
goods_url = 'https://haohuo.snssdk.com/views/product/item?id=' + goods_id
is_add = False
goods = await Goods.find_one('goods_id=?', goods_id)
if goods:
# 修改
time_now = datetime.datetime.now().strftime("%Y-%m-%d")
time_last_edit = goods.edit_time.strftime("%Y-%m-%d")
# 较上次增量
add_num = sell_num - goods.sell_num
goods.shop_id = shop_id
goods.cid = cid
goods.goods_name = goods_name
goods.goods_url = goods_url
goods.goods_picture_url = goods_picture_url
goods.goods_price = goods_price
if time_now != time_last_edit:
goods.add_num = 0
elif add_num >= 0:
goods.add_num = goods.add_num + add_num
goods.sell_num = sell_num
if goods.item_last_sell_num is None:
goods.item_last_sell_num = goods.sell_num
goods.edit_time = datetime.datetime.now()
item_add_num = sell_num - goods.item_last_sell_num
if item_add_num > 100:
goods.item_last_sell_num = sell_num
await goods.update()
else:
# 新增
is_add = True
item_add_num = 0
goods = Goods()
goods.shop_id = shop_id
goods.goods_id = goods_id
goods.goods_name = goods_name
goods.goods_url = goods_url
goods.goods_picture_url = goods_picture_url
goods.goods_price = goods_price
goods.cid = cid
goods.add_num = 0
goods.sell_num = sell_num
goods.item_last_sell_num = sell_num
goods.id = await goods.save()
if goods.id == 0:
is_add = False
if is_add or item_add_num >= 100:
goods_item = Goods_Item()
goods_item.goods_id = goods.id
goods_item.sell_num = sell_num
goods_item.add_num = item_add_num
await goods_item.save()
if goods.add_num > 0:
await Goods_Tmp.del_by('goods_id=?', goods.id)
tmp = Goods_Tmp()
tmp.goods_id = goods.id
tmp.add_num = goods.add_num
tmp.sell_num = goods.sell_num
tmp.edit_time = datetime.datetime.now()
await tmp.save()
tmp.sell_num = goods.sell_num
tmp.edit_time = datetime.datetime.now()
await tmp.save()
# 按照给定时间更新
if __name__ == '__main__':
query_time = str(sys.argv[1])
# query_time = '2019-06-14 09:00:00'
print(query_time)
start = datetime.datetime.now()
loop = asyncio.get_event_loop()
loop.run_until_complete(orm.create_pool(loop=loop, **configs.db))
# and is_selling=? ,True
goods = loop.run_until_complete(Goods.findAll('edit_time<?', query_time))
q_goods = queue.Queue(maxsize=0)
for good in goods:
q_goods.put_nowait(good.goods_id)
print("商品总数%s" % q_goods.qsize())
goods_id_object = tools.list_to_dict(goods, "goods_id")
category_cids = loop.run_until_complete(Category_Cid.findAll())
cids = []
for category_cid in category_cids:
if not cids.__contains__(category_cid.cid):
cids.append(category_cid.cid)
# 初始化
q_data = queue.Queue(maxsize=30000)
global_goods_ids = []
shops = loop.run_until_complete(Shop.findAll())
# shops = loop.run_until_complete(Shop.findAll('shop_id=?', 'hmSuxrl'))
q_shops = queue.Queue(maxsize=0)
for shop in shops:
q_shops.put_nowait(shop)
print("店铺总数%s" % q_shops.qsize())
category_cids = loop.run_until_complete(Category_Cid.findAll())
cids = []
for category_cid in category_cids:
if not cids.__contains__(category_cid.cid):
cids.append(category_cid.cid)
shop_id_object = tools.list_to_dict(shops, 'shop_id')
all_goods = loop.run_until_complete(Goods.findAll())
goods_id_object = tools.list_to_dict(all_goods, "goods_id")
# 初始化
q_data = queue.Queue(maxsize=30000)
global_goods_ids = []
event = threading.Event()
lock = threading.Lock()
if event.isSet:
event.clear()
for i in range(400):
p = shop_producer.Producer(i, q_shops, q_data, event, global_goods_ids)
p.start()
tmp = Goods_Tmp()
tmp.goods_id = goods.id
tmp.add_num = goods.add_num
tmp.sell_num = goods.sell_num
tmp.edit_time = datetime.datetime.now()
await tmp.save()
# 更新当日热销
if __name__ == '__main__':
start = datetime.datetime.now()
loop = asyncio.get_event_loop()
loop.run_until_complete(orm.create_pool(loop=loop, **configs.db))
goods = loop.run_until_complete(
Goods.find_inner(tools.get_temp_table(), 'goods_id'))
q_goods = queue.Queue(maxsize=0)
for good in goods:
q_goods.put_nowait(good.goods_id)
print("商品总数%s" % q_goods.qsize())
goods_id_object = tools.list_to_dict(goods, "goods_id")
category_cids = loop.run_until_complete(Category_Cid.findAll())
cids = []
for category_cid in category_cids:
if not cids.__contains__(category_cid.cid):
cids.append(category_cid.cid)
# 初始化
q_data = queue.Queue(maxsize=30000)
global_goods_ids = []