def fav_update(session, novel):
send_notice = False
logger.info("更新用户收藏小说[{0}]开始...".format(novel.novel_name))
start_time = time.time()
try:
parser_obj = novel_parser[novel.spider_name]
except KeyError as e:
logger.error("不存在的解析器:{0}".format(novel.spider_name))
logger.info("更新用户收藏小说[{0}]结束, 耗时:{1}".format(novel.novel_name,
time.time() - start_time))
return
try:
chapters = Chapter.select(
Chapter.chapter_url).where(Chapter.novel_id == novel.id)
html = parser_obj.get_html(session,
novel.url,
delay_time=settings.DOWNLOAD_DELAY,
headers=headers)
urls = parser_obj.parse_novel(html, novel.novel_name)
filter_urls = set([chapter.chapter_url for chapter in chapters])
for url in urls:
url = parse.urljoin(novel.url, url)
if url in filter_urls:
continue
html = parser_obj.get_html(session,
url,
delay_time=settings.DOWNLOAD_DELAY,
headers=headers)
chapter_name = parser_obj.parse_chapter(html, novel.novel_name)
chapter_index = utils.get_index_by_chapter(url)
try:
Chapter.create(chapter_url=url,
chapter_name=chapter_name,
chapter_index=chapter_index,
novel_id=novel.id)
logger.info("保存[{0}:{1}]到数据库成功.".format(
novel.novel_name, chapter_name))
send_notice = True
except Exception as e:
logger.error("保存[{0}:{1}:{2}]到数据库失败, 原因:{3}".format(
novel.novel_name, chapter_index, url, e))
except Exception as e:
logger.error("更新用户收藏小说[{0}]失败, 原因:{1}".format(novel.novel_name, e))
logger.info("更新用户收藏小说[{0}]结束, 耗时:{1}".format(novel.novel_name,
time.time() - start_time))
try:
if send_notice:
update_notice(novel)
else:
logger.info("小说[{0}]无更新.".format(novel.novel_name))
except Exception as e:
logger.error("发送小说[{0}]更新同时失败, 原因:{1}".format(novel.novel_name, e))
async def novel_update(novel):
async with async_semaphore:
send_notice = False
logger.info("更新小说【{0}】开始...".format(novel.novel_name))
start_time = time.time()
try:
parser_obj = novel_parser[novel.spider_name]
except KeyError as e:
logger.error("不存在的解析器:{0}".format(novel.spider_name))
logger.info("更新小说【{0}】结束, 耗时:{1}".format(novel.novel_name,
time.time() - start_time))
return
try:
chapters = await objects.execute(
Chapter.select(
Chapter.chapter_url).where(Chapter.novel_id == novel.id))
filter_urls = set([chapter.chapter_url for chapter in chapters])
html = await parser_obj.async_get_html(
novel.url, delay_time=settings.DOWNLOAD_DELAY, headers=headers)
urls = parser_obj.parse_novel(html)
for url in urls:
url = parse.urljoin(novel.url, url)
if url in filter_urls:
continue
try:
html = await parser_obj.async_get_html(
url,
delay_time=settings.DOWNLOAD_DELAY,
headers=headers)
chapter_name = parser_obj.parse_chapter(html)
except Exception as e:
logger.error(
f"解析小说【{novel.novel_name}】章节【{url}】失败, 原因:{e}")
continue
chapter_index = get_index_by_chapter(url)
try:
await objects.create(Chapter,
chapter_url=url,
chapter_name=chapter_name,
chapter_index=chapter_index,
novel_id=novel.id)
logger.info("保存【{0}:{1}】到数据库成功.".format(
novel.novel_name, chapter_name))
except Exception as e:
logger.error("保存[{0}:{1}:{2}]到数据库失败, 原因:{3}".format(
novel.novel_name, chapter_index, url, e))
except Exception as e:
logger.error("更新小说【{0}】失败, 原因:{1}".format(novel.novel_name, e))
logger.info("更新小说【{0}】结束, 耗时:{1}".format(novel.novel_name,
time.time() - start_time))
