await tmp.save() # 按照分类更新数据 if __name__ == '__main__': start = datetime.datetime.now() loop = asyncio.get_event_loop() loop.run_until_complete(orm.create_pool(loop=loop, **configs.db)) categorys = loop.run_until_complete( Categorys.findAll('parent_id != ?', "0")) categoryDtos = [] cids = [] for category in categorys: categoryDto = {} category_cids = loop.run_until_complete( Category_Cid.findAll('category_id=?', category.id)) for category_cid in category_cids: if not cids.__contains__(category_cid.cid): cids.append(category_cid.cid) categoryDto["id"] = category.id categoryDto["parent_id"] = category.parent_id num_list_new = [str(x) for x in cids] categoryDto["cids"] = ','.join(num_list_new) categoryDtos.append(categoryDto) q_category = queue.Queue(maxsize=0) for categoryDto in categoryDtos: q_category.put_nowait(categoryDto) print("分类总数%s" % q_category.qsize()) # 初始化
if material_id: material_ids.append(int(material_id)) else: list = data.get('list') for element in list: material_id = element.get('goods').get('id') material_ids.append(int(material_id)) if len(material_ids) <= 0: sys.exit() q_material = queue.Queue(maxsize=0) for material_id in material_ids: q_material.put_nowait(material_id) print("模块总数%s" % q_material.qsize()) category_cids = loop.run_until_complete(Category_Cid.findAll()) cids = [] for category_cid in category_cids: if not cids.__contains__(category_cid.cid): cids.append(category_cid.cid) # 初始化 q_data = queue.Queue(maxsize=30000) global_goods_ids = [] event = threading.Event() lock = threading.Lock() if event.isSet: event.clear() for i in range(5): p = top100_producer.Producer(i, q_material, q_data, event, global_goods_ids, activity_id)
async def parse_page(category_id_object, category_cids): item = tools.get_category_all() for category_cid in category_cids: await Category_Cid.remove(category_cid) categorys = item.get('data') for fist_level in categorys: fist_category = fist_level.get('category') fist_category_id = fist_category.get("id") fist_category_image = fist_category.get("image") fist_category_name = fist_category.get("name") fist_category_priority = fist_category.get("priority") if category_id_object.keys().__contains__(int(fist_category_id)): #修改 category = category_id_object.get(int(fist_category_id)) category.id = fist_category_id category.parent_id = "0" category.name = fist_category_name category.priority = fist_category_priority category.image = fist_category_image await category.update() else: category = Categorys() category.id = fist_category_id category.parent_id = "0" category.name = fist_category_name category.priority = fist_category_priority category.image = fist_category_image await category.save() second_categorys = fist_level.get('sub_category') for second_category in second_categorys: second_category_id = second_category.get("id") second_category_image = second_category.get("image") second_category_name = second_category.get("name") second_category_priority = second_category.get("priority") second_category_parent_id = second_category.get("parent_id") second_category_content = second_category.get("content") if category_id_object.keys().__contains__(int(second_category_id)): #修改 category = category_id_object.get(int(second_category_id)) category.id = second_category_id category.parent_id = second_category_parent_id category.name = second_category_name category.priority = second_category_priority category.image = second_category_image await category.update() else: category = Categorys() category.id = second_category_id category.parent_id = second_category_parent_id category.name = second_category_name category.priority = second_category_priority category.image = second_category_image await category.save() if second_category_content is not None or second_category_content != '': contents = second_category_content.split(',') for content in contents: category_cid = Category_Cid() category_cid.cid = content category_cid.category_id = second_category_id await category_cid.save()