async def save(self, **kwargs):
# 存进数据库
res_dic = kwargs.get('res_dic')
try:
motor_db = MotorBaseOld().db
await motor_db.all_novels.update_one({
'novel_url': res_dic['novel_url'], 'novel_author': res_dic['novel_name']},
{'$set': res_dic},
upsert=True)
except Exception as e:
self.logger.exception(e)
async def save(self, res_dic):
try:
motor_db = MotorBaseOld().db
await motor_db.novels_ranking.update_one({
'target_url': res_dic['target_url']},
{'$set': {
'data': res_dic['data'],
'spider': res_dic['spider'],
'type': res_dic['type'],
'finished_at': time.strftime("%Y-%m-%d %X", time.localtime())
}},
upsert=True)
except Exception as e:
self.logger.exception(e)
async def parse(self, res):
motor_db = MotorBaseOld().db
item = await QidianNovelInfoItem.get_item(html=res.html)
item_data = {
'novel_name': item.novel_name,
'author': item.author,
'cover': item.cover,
'abstract': item.abstract,
'status': item.status,
'novels_type': item.novels_type,
'latest_chapter': item.latest_chapter,
'latest_chapter_time': item.latest_chapter_time,
'spider': 'qidian',
'target_url': res.url,
'updated_at': time.strftime("%Y-%m-%d %X", time.localtime())
}
print('获取 {} 小说信息成功'.format(item.novel_name))
await motor_db.all_novels_info.update_one(
{
'novel_name': item_data['novel_name'],
'spider': item_data['spider']
}, {'$set': item_data},
upsert=True)