await tmp.save()
# 按照分类更新数据
if __name__ == '__main__':
start = datetime.datetime.now()
loop = asyncio.get_event_loop()
loop.run_until_complete(orm.create_pool(loop=loop, **configs.db))
categorys = loop.run_until_complete(
Categorys.findAll('parent_id != ?', "0"))
categoryDtos = []
cids = []
for category in categorys:
categoryDto = {}
category_cids = loop.run_until_complete(
Category_Cid.findAll('category_id=?', category.id))
for category_cid in category_cids:
if not cids.__contains__(category_cid.cid):
cids.append(category_cid.cid)
categoryDto["id"] = category.id
categoryDto["parent_id"] = category.parent_id
num_list_new = [str(x) for x in cids]
categoryDto["cids"] = ','.join(num_list_new)
categoryDtos.append(categoryDto)
q_category = queue.Queue(maxsize=0)
for categoryDto in categoryDtos:
q_category.put_nowait(categoryDto)
print("分类总数%s" % q_category.qsize())
# 初始化
if material_id:
material_ids.append(int(material_id))
else:
list = data.get('list')
for element in list:
material_id = element.get('goods').get('id')
material_ids.append(int(material_id))
if len(material_ids) <= 0:
sys.exit()
q_material = queue.Queue(maxsize=0)
for material_id in material_ids:
q_material.put_nowait(material_id)
print("模块总数%s" % q_material.qsize())
category_cids = loop.run_until_complete(Category_Cid.findAll())
cids = []
for category_cid in category_cids:
if not cids.__contains__(category_cid.cid):
cids.append(category_cid.cid)
# 初始化
q_data = queue.Queue(maxsize=30000)
global_goods_ids = []
event = threading.Event()
lock = threading.Lock()
if event.isSet:
event.clear()
for i in range(5):
p = top100_producer.Producer(i, q_material, q_data, event, global_goods_ids, activity_id)
async def parse_page(category_id_object, category_cids):
item = tools.get_category_all()
for category_cid in category_cids:
await Category_Cid.remove(category_cid)
categorys = item.get('data')
for fist_level in categorys:
fist_category = fist_level.get('category')
fist_category_id = fist_category.get("id")
fist_category_image = fist_category.get("image")
fist_category_name = fist_category.get("name")
fist_category_priority = fist_category.get("priority")
if category_id_object.keys().__contains__(int(fist_category_id)):
#修改
category = category_id_object.get(int(fist_category_id))
category.id = fist_category_id
category.parent_id = "0"
category.name = fist_category_name
category.priority = fist_category_priority
category.image = fist_category_image
await category.update()
else:
category = Categorys()
category.id = fist_category_id
category.parent_id = "0"
category.name = fist_category_name
category.priority = fist_category_priority
category.image = fist_category_image
await category.save()
second_categorys = fist_level.get('sub_category')
for second_category in second_categorys:
second_category_id = second_category.get("id")
second_category_image = second_category.get("image")
second_category_name = second_category.get("name")
second_category_priority = second_category.get("priority")
second_category_parent_id = second_category.get("parent_id")
second_category_content = second_category.get("content")
if category_id_object.keys().__contains__(int(second_category_id)):
#修改
category = category_id_object.get(int(second_category_id))
category.id = second_category_id
category.parent_id = second_category_parent_id
category.name = second_category_name
category.priority = second_category_priority
category.image = second_category_image
await category.update()
else:
category = Categorys()
category.id = second_category_id
category.parent_id = second_category_parent_id
category.name = second_category_name
category.priority = second_category_priority
category.image = second_category_image
await category.save()
if second_category_content is not None or second_category_content != '':
contents = second_category_content.split(',')
for content in contents:
category_cid = Category_Cid()
category_cid.cid = content
category_cid.category_id = second_category_id
await category_cid.save()