def fav_update(session, novel): send_notice = False logger.info("更新用户收藏小说[{0}]开始...".format(novel.novel_name)) start_time = time.time() try: parser_obj = novel_parser[novel.spider_name] except KeyError as e: logger.error("不存在的解析器:{0}".format(novel.spider_name)) logger.info("更新用户收藏小说[{0}]结束, 耗时:{1}".format(novel.novel_name, time.time() - start_time)) return try: chapters = Chapter.select( Chapter.chapter_url).where(Chapter.novel_id == novel.id) html = parser_obj.get_html(session, novel.url, delay_time=settings.DOWNLOAD_DELAY, headers=headers) urls = parser_obj.parse_novel(html, novel.novel_name) filter_urls = set([chapter.chapter_url for chapter in chapters]) for url in urls: url = parse.urljoin(novel.url, url) if url in filter_urls: continue html = parser_obj.get_html(session, url, delay_time=settings.DOWNLOAD_DELAY, headers=headers) chapter_name = parser_obj.parse_chapter(html, novel.novel_name) chapter_index = utils.get_index_by_chapter(url) try: Chapter.create(chapter_url=url, chapter_name=chapter_name, chapter_index=chapter_index, novel_id=novel.id) logger.info("保存[{0}:{1}]到数据库成功.".format( novel.novel_name, chapter_name)) send_notice = True except Exception as e: logger.error("保存[{0}:{1}:{2}]到数据库失败, 原因:{3}".format( novel.novel_name, chapter_index, url, e)) except Exception as e: logger.error("更新用户收藏小说[{0}]失败, 原因:{1}".format(novel.novel_name, e)) logger.info("更新用户收藏小说[{0}]结束, 耗时:{1}".format(novel.novel_name, time.time() - start_time)) try: if send_notice: update_notice(novel) else: logger.info("小说[{0}]无更新.".format(novel.novel_name)) except Exception as e: logger.error("发送小说[{0}]更新同时失败, 原因:{1}".format(novel.novel_name, e))
async def novel_update(novel): async with async_semaphore: send_notice = False logger.info("更新小说【{0}】开始...".format(novel.novel_name)) start_time = time.time() try: parser_obj = novel_parser[novel.spider_name] except KeyError as e: logger.error("不存在的解析器:{0}".format(novel.spider_name)) logger.info("更新小说【{0}】结束, 耗时:{1}".format(novel.novel_name, time.time() - start_time)) return try: chapters = await objects.execute( Chapter.select( Chapter.chapter_url).where(Chapter.novel_id == novel.id)) filter_urls = set([chapter.chapter_url for chapter in chapters]) html = await parser_obj.async_get_html( novel.url, delay_time=settings.DOWNLOAD_DELAY, headers=headers) urls = parser_obj.parse_novel(html) for url in urls: url = parse.urljoin(novel.url, url) if url in filter_urls: continue try: html = await parser_obj.async_get_html( url, delay_time=settings.DOWNLOAD_DELAY, headers=headers) chapter_name = parser_obj.parse_chapter(html) except Exception as e: logger.error( f"解析小说【{novel.novel_name}】章节【{url}】失败, 原因:{e}") continue chapter_index = get_index_by_chapter(url) try: await objects.create(Chapter, chapter_url=url, chapter_name=chapter_name, chapter_index=chapter_index, novel_id=novel.id) logger.info("保存【{0}:{1}】到数据库成功.".format( novel.novel_name, chapter_name)) except Exception as e: logger.error("保存[{0}:{1}:{2}]到数据库失败, 原因:{3}".format( novel.novel_name, chapter_index, url, e)) except Exception as e: logger.error("更新小说【{0}】失败, 原因:{1}".format(novel.novel_name, e)) logger.info("更新小说【{0}】结束, 耗时:{1}".format(novel.novel_name, time.time() - start_time))