async def owllook_search(request): start = time.time() name = str(request.args.get('wd', '')).strip() novels_keyword = name.split(' ')[0] motor_db = motor_base.get_db() if not name: return redirect('/') else: # 记录搜索小说名 try: await motor_db.search_records.update_one({'keyword': name}, {'$inc': {'count': 1}}, upsert=True) except Exception as e: LOGGER.exception(e) # 通过搜索引擎获取检索结果 parse_result = None if name.startswith('!baidu'): novels_keyword = name.split('baidu')[1].strip() novels_name = 'intitle:{name} 小说 阅读'.format(name=novels_keyword) parse_result = await cache_owllook_baidu_novels_result(novels_name) elif name.startswith('!360'): novels_keyword = name.split('360')[1].strip() novels_name = "{name} 小说 最新章节".format(name=novels_keyword) parse_result = await cache_owllook_so_novels_result(novels_name) elif name.startswith('!bing'): novels_keyword = name.split('bing')[1].strip() novels_name = "{name} 小说 阅读 最新章节".format(name=novels_keyword) parse_result = await cache_owllook_bing_novels_result(novels_name) # elif name.startswith('!duck_go'): # novels_keyword = name.split('duck_go')[1].strip() # novels_name = '{name} 小说 阅读 最新章节'.format(name=novels_keyword) # parse_result = await cache_owllook_duck_novels_result(novels_name) else: for each_engine in ENGINE_PRIORITY: # for bing if each_engine == "bing": novels_name = "{name} 小说 阅读 最新章节".format(name=name) parse_result = await cache_owllook_bing_novels_result(novels_name) if parse_result: break # for 360 so if each_engine == "360": novels_name = "{name} 小说 最新章节".format(name=name) parse_result = await cache_owllook_so_novels_result(novels_name) if parse_result: break # for baidu if each_engine == "baidu": novels_name = 'intitle:{name} 小说 阅读'.format(name=name) parse_result = await cache_owllook_baidu_novels_result(novels_name) if parse_result: break # for duckduckgo if each_engine == "duck_go": novels_name = '{name} 小说 阅读 最新章节'.format(name=name) parse_result = await cache_owllook_duck_novels_result(novels_name) if parse_result: break if parse_result: # result_sorted = sorted( # parse_result, reverse=True, key=lambda res: res['timestamp']) if ':baidu' not in name else parse_result # 优先依靠是否解析进行排序 其次以更新时间进行排序 result_sorted = sorted( parse_result, reverse=True, key=itemgetter('is_recommend', 'is_parse', 'timestamp')) user = request['session'].get('user', None) if user: try: time_current = get_time() res = await motor_db.user_message.update_one({'user': user}, {'$set': {'last_update_time': time_current}}, upsert=True) # 此处语法操作过多 下次看一遍mongo再改 if res: is_ok = await motor_db.user_message.update_one( {'user': user, 'search_records.keyword': {'$ne': novels_keyword}}, {'$push': {'search_records': {'keyword': novels_keyword, 'counts': 1}}}, ) if is_ok: await motor_db.user_message.update_one( {'user': user, 'search_records.keyword': novels_keyword}, {'$inc': {'search_records.$.counts': 1}} ) except Exception as e: LOGGER.exception(e) return template( 'result.html', is_login=1, user=user, name=novels_keyword, time='%.2f' % (time.time() - start), result=result_sorted, count=len(parse_result)) else: return template( 'result.html', is_login=0, name=novels_keyword, time='%.2f' % (time.time() - start), result=result_sorted, count=len(parse_result)) else: return html("No Result!请将小说名反馈给本站,谢谢!")
async def get_the_latest_chapter(chapter_url): try: with async_timeout.timeout(20): url = parse_qs(urlparse(chapter_url).query).get('url', '') novels_name = parse_qs(urlparse(chapter_url).query).get( 'novels_name', '') data = None if url and novels_name: url = url[0] novels_name = novels_name[0] netloc = urlparse(url).netloc if netloc in LATEST_RULES.keys(): async with aiohttp.ClientSession() as client: try: html = await target_fetch(client=client, url=url) if html is None: html = requests_target_fetch(url=url) except TypeError: html = requests_target_fetch(url=url) except Exception as e: LOGGER.exception(e) return None try: soup = BeautifulSoup(html, 'html5lib') except Exception as e: LOGGER.exception(e) return None latest_chapter_name, latest_chapter_url = None, None if LATEST_RULES[netloc].plan: meta_value = LATEST_RULES[netloc].meta_value latest_chapter_name = soup.select( 'meta[property="{0}"]'.format( meta_value["latest_chapter_name"])) latest_chapter_name = latest_chapter_name[0].get( 'content', None) if latest_chapter_name else None latest_chapter_url = soup.select( 'meta[property="{0}"]'.format( meta_value["latest_chapter_url"])) latest_chapter_url = latest_chapter_url[0].get( 'content', None) if latest_chapter_url else None else: selector = LATEST_RULES[netloc].selector content_url = selector.get('content_url') if selector.get('id', None): latest_chapter_soup = soup.find_all( id=selector['id']) elif selector.get('class', None): latest_chapter_soup = soup.find_all( class_=selector['class']) else: latest_chapter_soup = soup.select( selector.get('tag')) if latest_chapter_soup: if content_url == '1': # TODO pass elif content_url == '0': # TODO pass else: latest_chapter_url = content_url + latest_chapter_soup[ 0].get('href', None) latest_chapter_name = latest_chapter_soup[ 0].get('title', None) if latest_chapter_name and latest_chapter_url: time_current = get_time() data = { "latest_chapter_name": latest_chapter_name, "latest_chapter_url": latest_chapter_url, "owllook_chapter_url": chapter_url, "owllook_content_url": "/owllook_content?url={latest_chapter_url}&name={name}&chapter_url={chapter_url}&novels_name={novels_name}" .format( latest_chapter_url=latest_chapter_url, name=latest_chapter_name, chapter_url=url, novels_name=novels_name, ), } # 存储最新章节 motor_db = MotorBase().db await motor_db.latest_chapter.update_one( { "novels_name": novels_name, 'owllook_chapter_url': chapter_url }, { '$set': { 'data': data, "finished_at": time_current } }, upsert=True) return data except Exception as e: LOGGER.exception(e) return None
async def owllook_content(request): """ 返回小说章节内容页 : content_url 这决定当前U页面url的生成方式 : url 章节内容页源url : chapter_url 小说目录源url : novels_name 小说名称 :return: 小说章节内容页 """ url = request.args.get('url', None) chapter_url = request.args.get('chapter_url', None) novels_name = request.args.get('novels_name', None) name = request.args.get('name', '') is_ajax = request.args.get('is_ajax', '') # 当小说内容url不在解析规则内 跳转到原本url netloc = get_netloc(url) if netloc not in RULES.keys(): return redirect(url) user = request['session'].get('user', None) # 拼接小说目录url book_url = "/chapter?url={chapter_url}&novels_name={novels_name}".format( chapter_url=chapter_url, novels_name=novels_name) motor_db = motor_base.get_db() if url == chapter_url: # 阅读到最后章节时候 在数据库中保存最新阅读章节 if user and is_ajax == "owl_cache": owl_referer = request.headers.get('Referer', '').split('owllook_content')[1] if owl_referer: latest_read = "/owllook_content" + owl_referer await motor_db.user_message.update_one( {'user': user, 'books_url.book_url': book_url}, {'$set': {'books_url.$.last_read_url': latest_read}}) return redirect(book_url) content_url = RULES[netloc].content_url content_data = await cache_owllook_novels_content(url=url, netloc=netloc) if content_data: try: content = content_data.get('content', '获取失败') next_chapter = content_data.get('next_chapter', []) title = content_data.get('title', '').replace(novels_name, '') name = title if title else name # 拼接小说书签url bookmark_url = "{path}?url={url}&name={name}&chapter_url={chapter_url}&novels_name={novels_name}".format( path=request.path, url=url, name=name, chapter_url=chapter_url, novels_name=novels_name ) # 破坏广告链接 content = str(content).strip('[]Jjs,').replace('http', 'hs') if user: bookmark = await motor_db.user_message.find_one({'user': user, 'bookmarks.bookmark': bookmark_url}) book = await motor_db.user_message.find_one({'user': user, 'books_url.book_url': book_url}) bookmark = 1 if bookmark else 0 if book: # 当书架中存在该书源 book = 1 # 保存最后一次阅读记录 if is_ajax == "owl_cache": owl_referer = request.headers.get('Referer', bookmark_url).split('owllook_content')[1] latest_read = "/owllook_content" + owl_referer await motor_db.user_message.update_one( {'user': user, 'books_url.book_url': book_url}, {'$set': {'books_url.$.last_read_url': latest_read}}) else: book = 0 if is_ajax == "owl_cache": owl_cache_dict = dict( is_login=1, user=user, name=name, url=url, bookmark=bookmark, book=book, content_url=content_url, chapter_url=chapter_url, novels_name=novels_name, next_chapter=next_chapter, soup=content ) return json(owl_cache_dict) return template( 'content.html', is_login=1, user=user, name=name, url=url, bookmark=bookmark, book=book, content_url=content_url, chapter_url=chapter_url, novels_name=novels_name, next_chapter=next_chapter, soup=content) else: if is_ajax == "owl_cache": owl_cache_dict = dict( is_login=0, name=name, url=url, bookmark=0, book=0, content_url=content_url, chapter_url=chapter_url, novels_name=novels_name, next_chapter=next_chapter, soup=content ) return json(owl_cache_dict) return template( 'content.html', is_login=0, name=name, url=url, bookmark=0, book=0, content_url=content_url, chapter_url=chapter_url, novels_name=novels_name, next_chapter=next_chapter, soup=content) except Exception as e: LOGGER.exception(e) return redirect(book_url) else: if user: is_login = 1 user = user return template('parse_error.html', url=url, is_login=is_login, user=user) else: is_login = 0 return template('parse_error.html', url=url, is_login=is_login)
async def author_notification(request): """ 作者新书通知 :param request: :return: : -1 用户session失效 需要重新登录 : 2 无该作者信息 : 3 作者已经添加 : 4 超过添加的上限 : 0 操作失败 : 1 操作成功 """ user = request['session'].get('user', None) user_data = parse_qs(str(request.body, encoding='utf-8')) if user: try: motor_db = motor_base.get_db() all_authors = await motor_db.user_message.find_one( {'user': user}, { 'author_latest': 1, '_id': 0 }) count = len(all_authors.get('author_latest', [])) if count == CONFIG.WEBSITE.get("AUTHOR_LATEST_COUNT", 5): return json({'status': 4}) author_name = user_data.get('author_name', None)[0] data = [] author_cursor = motor_db.all_books.find({'author': author_name}, { 'name': 1, 'url': 1, '_id': 0 }) async for document in author_cursor: data.append(document) if data: time = get_time() res = await motor_db.user_message.update_one( {'user': user}, {'$set': { 'last_update_time': time }}, upsert=True) is_exist = await motor_db.user_message.find_one({ 'user': user, 'author_latest.author_name': author_name }) if is_exist: return json({'status': 3}) if res: await motor_db.user_message.update_one( { 'user': user, 'author_latest.author_name': { '$ne': author_name } }, { '$push': { 'author_latest': { 'author_name': author_name, 'add_time': time } } }) is_author_exist = await motor_db.author_message.find_one( {'name': author_name}) if not is_author_exist: author_data = { "author_name": author_name, "nums": len(data), "updated_time": get_time(), } await motor_db.author_message.save(author_data) LOGGER.info('作者添加成功') return json({'status': 1}) else: return json({'status': 2}) else: return json({'status': 2}) except Exception as e: LOGGER.exception(e) return json({'status': 0}) else: return json({'status': -1})
async def search_user(request): user = request['session'].get('user', None) name = request.args.get('ss', None) if user and name: motor_db = MotorBase().db try: data = await motor_db.user_message.find_one({'user': name}) books_url = data.get('books_url', None) if data else None if books_url: result = [] for i in books_url: item_result = {} book_url = i.get('book_url', None) last_read_url = i.get("last_read_url", "") book_query = parse_qs(urlparse(book_url).query) last_read_chapter_name = parse_qs(last_read_url).get( 'name', ['暂无'])[0] item_result['novels_name'] = book_query.get( 'novels_name', '')[0] if book_query.get( 'novels_name', '') else '' item_result['book_url'] = book_url latest_data = await motor_db.latest_chapter.find_one( {'owllook_chapter_url': book_url}) if latest_data: item_result['latest_chapter_name'] = latest_data[ 'data']['latest_chapter_name'] item_result['owllook_content_url'] = latest_data[ 'data']['owllook_content_url'] else: get_latest_data = await get_the_latest_chapter(book_url ) or {} item_result[ 'latest_chapter_name'] = get_latest_data.get( 'latest_chapter_name', '暂未获取,请反馈') item_result[ 'owllook_content_url'] = get_latest_data.get( 'owllook_content_url', '') item_result['add_time'] = i.get('add_time', '') item_result[ "last_read_url"] = last_read_url if last_read_url else book_url item_result[ "last_read_chapter_name"] = last_read_chapter_name result.append(item_result) return template('search_user.html', title='{name}的书架 - owllook'.format(name=name), is_login=1, user=user, username=name, is_bookmark=1, result=result[::-1]) else: return template('search_user.html', title='{name}的书架 - owllook'.format(name=name), is_login=1, user=user, is_bookmark=0) except Exception as e: LOGGER.error(e) return redirect('/') else: return redirect('/')
async def owllook_content(request): """ 返回小说章节内容页 : content_url 这决定当前U页面url的生成方式 : url 章节内容页源url : chapter_url 小说目录源url : novels_name 小说名称 :return: 小说章节内容页 """ url = request.args.get('url', None) chapter_url = request.args.get('chapter_url', None) novels_name = request.args.get('novels_name', None) name = request.args.get('name', '') # 当小说内容url不在解析规则内 跳转到原本url netloc = get_netloc(url) if netloc not in RULES.keys(): return redirect(url) # 拼接小说目录url book_url = "/chapter?url={chapter_url}&novels_name={novels_name}".format( chapter_url=chapter_url, novels_name=novels_name) if url == chapter_url: return redirect(book_url) content_url = RULES[netloc].content_url content_data = await cache_owllook_novels_content(url=url, netloc=netloc) if content_data: user = request['session'].get('user', None) try: content = content_data.get('content', '获取失败') next_chapter = content_data.get('next_chapter', []) title = content_data.get('title', '').replace(novels_name, '') name = title if title else name # 拼接小说书签url bookmark_url = "{path}?url={url}&name={name}&chapter_url={chapter_url}&novels_name={novels_name}".format( path=request.path, url=url, name=name, chapter_url=chapter_url, novels_name=novels_name ) # 破坏广告链接 content = str(content).strip('[]Jjs,').replace('http', 'hs') if user: motor_db = motor_base.db bookmark = await motor_db.user_message.find_one({'user': user, 'bookmarks.bookmark': bookmark_url}) book = await motor_db.user_message.find_one({'user': user, 'books_url.book_url': book_url}) bookmark = 1 if bookmark else 0 if book: # 当书架中存在该书源 book = 1 # 保存最后一次阅读记录 await motor_db.user_message.update_one( {'user': user, 'books_url.book_url': book_url}, {'$set': {'books_url.$.last_read_url': bookmark_url}}) else: book = 0 return template( 'content.html', is_login=1, user=user, name=name, url=url, bookmark=bookmark, book=book, content_url=content_url, chapter_url=chapter_url, novels_name=novels_name, next_chapter=next_chapter, soup=content) else: return template( 'content.html', is_login=0, name=name, url=url, bookmark=0, book=0, content_url=content_url, chapter_url=chapter_url, novels_name=novels_name, next_chapter=next_chapter, soup=content) except Exception as e: LOGGER.exception(e) return redirect(book_url) else: return text('解析失败或者是没有下一页了,请将失败页面反馈给本站,请重新刷新一次,或者访问源网页:{url}'.format(url=url))
}, { 'class': "aiocache.plugins.TimingPlugin" }] } }) # aiocache.settings.set_defaults( # class_="aiocache.RedisCache", # endpoint=REDIS_DICT.get('REDIS_ENDPOINT', 'localhost'), # port=REDIS_DICT.get('REDIS_PORT', 6379), # db=REDIS_DICT.get('CACHE_DB', 0), # password=REDIS_DICT.get('REDIS_PASSWORD', None), # loop=loop, # ) LOGGER.info("Starting redis pool") redis_session = RedisSession() # redis instance for app app.get_redis_pool = redis_session.get_redis_pool # pass the getter method for the connection pool into the session app.session_interface = RedisSessionInterface(app.get_redis_pool, cookie_name="owl_sid", expiry=30 * 24 * 60 * 60) @app.middleware('request') async def add_session_to_request(request): # before each request initialize a session # using the client's request host = request.headers.get('host', None) user_agent = request.headers.get('user-agent', None)
async def index(request): motor_db = MotorBase().db ranking_cursor = motor_db.novels_ranking.find({}) async for document in ranking_cursor: LOGGER.info(document) return response.text('Ranking test')