async def owllook_search(request):
start = time.time()
name = str(request.args.get('wd', '')).strip()
novels_keyword = name.split(' ')[0]
motor_db = motor_base.get_db()
if not name:
return redirect('/')
else:
# 记录搜索小说名
try:
await motor_db.search_records.update_one({'keyword': name}, {'$inc': {'count': 1}}, upsert=True)
except Exception as e:
LOGGER.exception(e)
# 通过搜索引擎获取检索结果
parse_result = None
if name.startswith('!baidu'):
novels_keyword = name.split('baidu')[1].strip()
novels_name = 'intitle:{name} 小说 阅读'.format(name=novels_keyword)
parse_result = await cache_owllook_baidu_novels_result(novels_name)
elif name.startswith('!360'):
novels_keyword = name.split('360')[1].strip()
novels_name = "{name} 小说 最新章节".format(name=novels_keyword)
parse_result = await cache_owllook_so_novels_result(novels_name)
elif name.startswith('!bing'):
novels_keyword = name.split('bing')[1].strip()
novels_name = "{name} 小说 阅读 最新章节".format(name=novels_keyword)
parse_result = await cache_owllook_bing_novels_result(novels_name)
# elif name.startswith('!duck_go'):
# novels_keyword = name.split('duck_go')[1].strip()
# novels_name = '{name} 小说 阅读 最新章节'.format(name=novels_keyword)
# parse_result = await cache_owllook_duck_novels_result(novels_name)
else:
for each_engine in ENGINE_PRIORITY:
# for bing
if each_engine == "bing":
novels_name = "{name} 小说 阅读 最新章节".format(name=name)
parse_result = await cache_owllook_bing_novels_result(novels_name)
if parse_result:
break
# for 360 so
if each_engine == "360":
novels_name = "{name} 小说 最新章节".format(name=name)
parse_result = await cache_owllook_so_novels_result(novels_name)
if parse_result:
break
# for baidu
if each_engine == "baidu":
novels_name = 'intitle:{name} 小说 阅读'.format(name=name)
parse_result = await cache_owllook_baidu_novels_result(novels_name)
if parse_result:
break
# for duckduckgo
if each_engine == "duck_go":
novels_name = '{name} 小说 阅读 最新章节'.format(name=name)
parse_result = await cache_owllook_duck_novels_result(novels_name)
if parse_result:
break
if parse_result:
# result_sorted = sorted(
# parse_result, reverse=True, key=lambda res: res['timestamp']) if ':baidu' not in name else parse_result
# 优先依靠是否解析进行排序 其次以更新时间进行排序
result_sorted = sorted(
parse_result,
reverse=True,
key=itemgetter('is_recommend', 'is_parse', 'timestamp'))
user = request['session'].get('user', None)
if user:
try:
time_current = get_time()
res = await motor_db.user_message.update_one({'user': user},
{'$set': {'last_update_time': time_current}},
upsert=True)
# 此处语法操作过多 下次看一遍mongo再改
if res:
is_ok = await motor_db.user_message.update_one(
{'user': user, 'search_records.keyword': {'$ne': novels_keyword}},
{'$push': {'search_records': {'keyword': novels_keyword, 'counts': 1}}},
)
if is_ok:
await motor_db.user_message.update_one(
{'user': user, 'search_records.keyword': novels_keyword},
{'$inc': {'search_records.$.counts': 1}}
)
except Exception as e:
LOGGER.exception(e)
return template(
'result.html',
is_login=1,
user=user,
name=novels_keyword,
time='%.2f' % (time.time() - start),
result=result_sorted,
count=len(parse_result))
else:
return template(
'result.html',
is_login=0,
name=novels_keyword,
time='%.2f' % (time.time() - start),
result=result_sorted,
count=len(parse_result))
else:
return html("No Result！请将小说名反馈给本站，谢谢！")
async def get_the_latest_chapter(chapter_url):
try:
with async_timeout.timeout(20):
url = parse_qs(urlparse(chapter_url).query).get('url', '')
novels_name = parse_qs(urlparse(chapter_url).query).get(
'novels_name', '')
data = None
if url and novels_name:
url = url[0]
novels_name = novels_name[0]
netloc = urlparse(url).netloc
if netloc in LATEST_RULES.keys():
async with aiohttp.ClientSession() as client:
try:
html = await target_fetch(client=client, url=url)
if html is None:
html = requests_target_fetch(url=url)
except TypeError:
html = requests_target_fetch(url=url)
except Exception as e:
LOGGER.exception(e)
return None
try:
soup = BeautifulSoup(html, 'html5lib')
except Exception as e:
LOGGER.exception(e)
return None
latest_chapter_name, latest_chapter_url = None, None
if LATEST_RULES[netloc].plan:
meta_value = LATEST_RULES[netloc].meta_value
latest_chapter_name = soup.select(
'meta[property="{0}"]'.format(
meta_value["latest_chapter_name"]))
latest_chapter_name = latest_chapter_name[0].get(
'content',
None) if latest_chapter_name else None
latest_chapter_url = soup.select(
'meta[property="{0}"]'.format(
meta_value["latest_chapter_url"]))
latest_chapter_url = latest_chapter_url[0].get(
'content',
None) if latest_chapter_url else None
else:
selector = LATEST_RULES[netloc].selector
content_url = selector.get('content_url')
if selector.get('id', None):
latest_chapter_soup = soup.find_all(
id=selector['id'])
elif selector.get('class', None):
latest_chapter_soup = soup.find_all(
class_=selector['class'])
else:
latest_chapter_soup = soup.select(
selector.get('tag'))
if latest_chapter_soup:
if content_url == '1':
# TODO
pass
elif content_url == '0':
# TODO
pass
else:
latest_chapter_url = content_url + latest_chapter_soup[
0].get('href', None)
latest_chapter_name = latest_chapter_soup[
0].get('title', None)
if latest_chapter_name and latest_chapter_url:
time_current = get_time()
data = {
"latest_chapter_name":
latest_chapter_name,
"latest_chapter_url":
latest_chapter_url,
"owllook_chapter_url":
chapter_url,
"owllook_content_url":
"/owllook_content?url={latest_chapter_url}&name={name}&chapter_url={chapter_url}&novels_name={novels_name}"
.format(
latest_chapter_url=latest_chapter_url,
name=latest_chapter_name,
chapter_url=url,
novels_name=novels_name,
),
}
# 存储最新章节
motor_db = MotorBase().db
await motor_db.latest_chapter.update_one(
{
"novels_name": novels_name,
'owllook_chapter_url': chapter_url
}, {
'$set': {
'data': data,
"finished_at": time_current
}
},
upsert=True)
return data
except Exception as e:
LOGGER.exception(e)
return None
async def owllook_content(request):
"""
返回小说章节内容页
: content_url 这决定当前U页面url的生成方式
: url 章节内容页源url
: chapter_url 小说目录源url
: novels_name 小说名称
:return: 小说章节内容页
"""
url = request.args.get('url', None)
chapter_url = request.args.get('chapter_url', None)
novels_name = request.args.get('novels_name', None)
name = request.args.get('name', '')
is_ajax = request.args.get('is_ajax', '')
# 当小说内容url不在解析规则内 跳转到原本url
netloc = get_netloc(url)
if netloc not in RULES.keys():
return redirect(url)
user = request['session'].get('user', None)
# 拼接小说目录url
book_url = "/chapter?url={chapter_url}&novels_name={novels_name}".format(
chapter_url=chapter_url,
novels_name=novels_name)
motor_db = motor_base.get_db()
if url == chapter_url:
# 阅读到最后章节时候 在数据库中保存最新阅读章节
if user and is_ajax == "owl_cache":
owl_referer = request.headers.get('Referer', '').split('owllook_content')[1]
if owl_referer:
latest_read = "/owllook_content" + owl_referer
await motor_db.user_message.update_one(
{'user': user, 'books_url.book_url': book_url},
{'$set': {'books_url.$.last_read_url': latest_read}})
return redirect(book_url)
content_url = RULES[netloc].content_url
content_data = await cache_owllook_novels_content(url=url, netloc=netloc)
if content_data:
try:
content = content_data.get('content', '获取失败')
next_chapter = content_data.get('next_chapter', [])
title = content_data.get('title', '').replace(novels_name, '')
name = title if title else name
# 拼接小说书签url
bookmark_url = "{path}?url={url}&name={name}&chapter_url={chapter_url}&novels_name={novels_name}".format(
path=request.path,
url=url,
name=name,
chapter_url=chapter_url,
novels_name=novels_name
)
# 破坏广告链接
content = str(content).strip('[]Jjs,').replace('http', 'hs')
if user:
bookmark = await motor_db.user_message.find_one({'user': user, 'bookmarks.bookmark': bookmark_url})
book = await motor_db.user_message.find_one({'user': user, 'books_url.book_url': book_url})
bookmark = 1 if bookmark else 0
if book:
# 当书架中存在该书源
book = 1
# 保存最后一次阅读记录
if is_ajax == "owl_cache":
owl_referer = request.headers.get('Referer', bookmark_url).split('owllook_content')[1]
latest_read = "/owllook_content" + owl_referer
await motor_db.user_message.update_one(
{'user': user, 'books_url.book_url': book_url},
{'$set': {'books_url.$.last_read_url': latest_read}})
else:
book = 0
if is_ajax == "owl_cache":
owl_cache_dict = dict(
is_login=1,
user=user,
name=name,
url=url,
bookmark=bookmark,
book=book,
content_url=content_url,
chapter_url=chapter_url,
novels_name=novels_name,
next_chapter=next_chapter,
soup=content
)
return json(owl_cache_dict)
return template(
'content.html',
is_login=1,
user=user,
name=name,
url=url,
bookmark=bookmark,
book=book,
content_url=content_url,
chapter_url=chapter_url,
novels_name=novels_name,
next_chapter=next_chapter,
soup=content)
else:
if is_ajax == "owl_cache":
owl_cache_dict = dict(
is_login=0,
name=name,
url=url,
bookmark=0,
book=0,
content_url=content_url,
chapter_url=chapter_url,
novels_name=novels_name,
next_chapter=next_chapter,
soup=content
)
return json(owl_cache_dict)
return template(
'content.html',
is_login=0,
name=name,
url=url,
bookmark=0,
book=0,
content_url=content_url,
chapter_url=chapter_url,
novels_name=novels_name,
next_chapter=next_chapter,
soup=content)
except Exception as e:
LOGGER.exception(e)
return redirect(book_url)
else:
if user:
is_login = 1
user = user
return template('parse_error.html', url=url, is_login=is_login, user=user)
else:
is_login = 0
return template('parse_error.html', url=url, is_login=is_login)
async def author_notification(request):
"""
作者新书通知
:param request:
:return:
: -1 用户session失效 需要重新登录
: 2 无该作者信息
: 3 作者已经添加
: 4 超过添加的上限
: 0 操作失败
: 1 操作成功
"""
user = request['session'].get('user', None)
user_data = parse_qs(str(request.body, encoding='utf-8'))
if user:
try:
motor_db = motor_base.get_db()
all_authors = await motor_db.user_message.find_one(
{'user': user}, {
'author_latest': 1,
'_id': 0
})
count = len(all_authors.get('author_latest', []))
if count == CONFIG.WEBSITE.get("AUTHOR_LATEST_COUNT", 5):
return json({'status': 4})
author_name = user_data.get('author_name', None)[0]
data = []
author_cursor = motor_db.all_books.find({'author': author_name}, {
'name': 1,
'url': 1,
'_id': 0
})
async for document in author_cursor:
data.append(document)
if data:
time = get_time()
res = await motor_db.user_message.update_one(
{'user': user}, {'$set': {
'last_update_time': time
}},
upsert=True)
is_exist = await motor_db.user_message.find_one({
'user':
user,
'author_latest.author_name':
author_name
})
if is_exist:
return json({'status': 3})
if res:
await motor_db.user_message.update_one(
{
'user': user,
'author_latest.author_name': {
'$ne': author_name
}
}, {
'$push': {
'author_latest': {
'author_name': author_name,
'add_time': time
}
}
})
is_author_exist = await motor_db.author_message.find_one(
{'name': author_name})
if not is_author_exist:
author_data = {
"author_name": author_name,
"nums": len(data),
"updated_time": get_time(),
}
await motor_db.author_message.save(author_data)
LOGGER.info('作者添加成功')
return json({'status': 1})
else:
return json({'status': 2})
else:
return json({'status': 2})
except Exception as e:
LOGGER.exception(e)
return json({'status': 0})
else:
return json({'status': -1})
async def search_user(request):
user = request['session'].get('user', None)
name = request.args.get('ss', None)
if user and name:
motor_db = MotorBase().db
try:
data = await motor_db.user_message.find_one({'user': name})
books_url = data.get('books_url', None) if data else None
if books_url:
result = []
for i in books_url:
item_result = {}
book_url = i.get('book_url', None)
last_read_url = i.get("last_read_url", "")
book_query = parse_qs(urlparse(book_url).query)
last_read_chapter_name = parse_qs(last_read_url).get(
'name', ['暂无'])[0]
item_result['novels_name'] = book_query.get(
'novels_name', '')[0] if book_query.get(
'novels_name', '') else ''
item_result['book_url'] = book_url
latest_data = await motor_db.latest_chapter.find_one(
{'owllook_chapter_url': book_url})
if latest_data:
item_result['latest_chapter_name'] = latest_data[
'data']['latest_chapter_name']
item_result['owllook_content_url'] = latest_data[
'data']['owllook_content_url']
else:
get_latest_data = await get_the_latest_chapter(book_url
) or {}
item_result[
'latest_chapter_name'] = get_latest_data.get(
'latest_chapter_name', '暂未获取，请反馈')
item_result[
'owllook_content_url'] = get_latest_data.get(
'owllook_content_url', '')
item_result['add_time'] = i.get('add_time', '')
item_result[
"last_read_url"] = last_read_url if last_read_url else book_url
item_result[
"last_read_chapter_name"] = last_read_chapter_name
result.append(item_result)
return template('search_user.html',
title='{name}的书架 - owllook'.format(name=name),
is_login=1,
user=user,
username=name,
is_bookmark=1,
result=result[::-1])
else:
return template('search_user.html',
title='{name}的书架 - owllook'.format(name=name),
is_login=1,
user=user,
is_bookmark=0)
except Exception as e:
LOGGER.error(e)
return redirect('/')
else:
return redirect('/')
async def owllook_content(request):
"""
返回小说章节内容页
: content_url 这决定当前U页面url的生成方式
: url 章节内容页源url
: chapter_url 小说目录源url
: novels_name 小说名称
:return: 小说章节内容页
"""
url = request.args.get('url', None)
chapter_url = request.args.get('chapter_url', None)
novels_name = request.args.get('novels_name', None)
name = request.args.get('name', '')
# 当小说内容url不在解析规则内 跳转到原本url
netloc = get_netloc(url)
if netloc not in RULES.keys():
return redirect(url)
# 拼接小说目录url
book_url = "/chapter?url={chapter_url}&novels_name={novels_name}".format(
chapter_url=chapter_url,
novels_name=novels_name)
if url == chapter_url:
return redirect(book_url)
content_url = RULES[netloc].content_url
content_data = await cache_owllook_novels_content(url=url, netloc=netloc)
if content_data:
user = request['session'].get('user', None)
try:
content = content_data.get('content', '获取失败')
next_chapter = content_data.get('next_chapter', [])
title = content_data.get('title', '').replace(novels_name, '')
name = title if title else name
# 拼接小说书签url
bookmark_url = "{path}?url={url}&name={name}&chapter_url={chapter_url}&novels_name={novels_name}".format(
path=request.path,
url=url,
name=name,
chapter_url=chapter_url,
novels_name=novels_name
)
# 破坏广告链接
content = str(content).strip('[]Jjs,').replace('http', 'hs')
if user:
motor_db = motor_base.db
bookmark = await motor_db.user_message.find_one({'user': user, 'bookmarks.bookmark': bookmark_url})
book = await motor_db.user_message.find_one({'user': user, 'books_url.book_url': book_url})
bookmark = 1 if bookmark else 0
if book:
# 当书架中存在该书源
book = 1
# 保存最后一次阅读记录
await motor_db.user_message.update_one(
{'user': user, 'books_url.book_url': book_url},
{'$set': {'books_url.$.last_read_url': bookmark_url}})
else:
book = 0
return template(
'content.html',
is_login=1,
user=user,
name=name,
url=url,
bookmark=bookmark,
book=book,
content_url=content_url,
chapter_url=chapter_url,
novels_name=novels_name,
next_chapter=next_chapter,
soup=content)
else:
return template(
'content.html',
is_login=0,
name=name,
url=url,
bookmark=0,
book=0,
content_url=content_url,
chapter_url=chapter_url,
novels_name=novels_name,
next_chapter=next_chapter,
soup=content)
except Exception as e:
LOGGER.exception(e)
return redirect(book_url)
else:
return text('解析失败或者是没有下一页了，请将失败页面反馈给本站，请重新刷新一次，或者访问源网页：{url}'.format(url=url))
}, {
'class': "aiocache.plugins.TimingPlugin"
}]
}
})
# aiocache.settings.set_defaults(
# class_="aiocache.RedisCache",
# endpoint=REDIS_DICT.get('REDIS_ENDPOINT', 'localhost'),
# port=REDIS_DICT.get('REDIS_PORT', 6379),
# db=REDIS_DICT.get('CACHE_DB', 0),
# password=REDIS_DICT.get('REDIS_PASSWORD', None),
# loop=loop,
# )
LOGGER.info("Starting redis pool")
redis_session = RedisSession()
# redis instance for app
app.get_redis_pool = redis_session.get_redis_pool
# pass the getter method for the connection pool into the session
app.session_interface = RedisSessionInterface(app.get_redis_pool,
cookie_name="owl_sid",
expiry=30 * 24 * 60 * 60)
@app.middleware('request')
async def add_session_to_request(request):
# before each request initialize a session
# using the client's request
host = request.headers.get('host', None)
user_agent = request.headers.get('user-agent', None)
async def index(request):
motor_db = MotorBase().db
ranking_cursor = motor_db.novels_ranking.find({})
async for document in ranking_cursor:
LOGGER.info(document)
return response.text('Ranking test')
