async def change_email(request):
"""
修改用户邮箱
:param request:
:return:
: -1 用户session失效 需要重新登录
: 0 修改邮箱失败
: 1 添加邮箱成功
"""
user = request['session'].get('user', None)
data = parse_qs(str(request.body, encoding='utf-8'))
if user:
try:
email = data.get('email', None)[0]
motor_db = motor_base.get_db()
await motor_db.user.update_one({'user': user},
{'$set': {
'email': email
}})
LOGGER.info('修改邮箱成功')
return json({'status': 1})
except Exception as e:
LOGGER.exception(e)
return json({'status': 0})
else:
return json({'status': -1})
async def auth_param(request, *args, **kwargs):
request_params = {}
# POST request
if request.method == 'POST' or request.method == 'DELETE':
try:
post_data = json_loads(str(request.body, encoding='utf-8'))
except Exception as e:
LOGGER.exception(e)
return response_handle(request,
UniResponse.PARAM_PARSE_ERR,
status=400)
else:
request_params.update(post_data)
params = [key for key, value in post_data.items() if value]
elif request.method == 'GET':
request_params.update(request.args)
params = [key for key, value in request.args.items() if value]
else:
# TODO
return response_handle(request,
UniResponse.PARAM_UNKNOWN_ERR,
status=400)
if set(keys).issubset(set(params)):
try:
kwargs['request_params'] = request_params
response = await func(request, *args, **kwargs)
return response
except Exception as e:
LOGGER.exception(e)
return response_handle(request,
UniResponse.SERVER_UNKNOWN_ERR, 500)
else:
return response_handle(request,
UniResponse.PARAM_ERR,
status=400)
async def update_all_books(loop, timeout=15):
try:
motor_db = MotorBase().get_db()
# 获取所有书架链接游标
books_url_cursor = motor_db.user_message.find({}, {
'books_url.book_url': 1,
'_id': 0
})
book_urls = []
already_urls = set()
async for document in books_url_cursor:
if document:
books_url = document['books_url']
for book_url in books_url:
chapter_url = book_url['book_url']
if chapter_url not in already_urls:
try:
await get_the_latest_chapter(chapter_url, timeout)
except Exception as e:
LOGGER.exception(e)
already_urls.add(chapter_url)
# 一组书架链接列表数据
# book_urls += [book_url['book_url'] for book_url in books_url]
# url_tasks = [get_the_latest_chapter(each_url, loop) for each_url in set(book_urls)]
# tasks = [asyncio.ensure_future(i) for i in url_tasks]
# try:
# await asyncio.gather(*tasks)
# except asyncio.TimeoutError as e:
# pass
except Exception as e:
LOGGER.exception(e)
return False
async def owl_novels_chapters(request, **kwargs):
"""
返回章节目录 基本达到通用
:param request:
:param chapter_url: 章节源目录页url
:param novels_name: 小说名称
:return: 小说目录信息
"""
request_params = kwargs["request_params"]
chapters_url = request_params.get('chapters_url', None)
novels_name = request_params.get('novels_name', None)
netloc = get_netloc(chapters_url)
try:
res = await cache_owllook_novels_chapter(url=chapters_url,
netloc=netloc)
chapters_sorted = []
if res:
chapters_sorted = extract_chapters(chapters_url, res)
UniResponse.SUCCESS.update({
ResponseField.DATA: {
'novels_name': novels_name,
'chapter_url': chapters_url,
'all_chapters': chapters_sorted
},
ResponseField.FINISH_AT: get_time()
})
return response_handle(request, UniResponse.SUCCESS, 200)
except Exception as e:
LOGGER.exception(e)
return response_handle(request, UniResponse.SERVER_UNKNOWN_ERR, 500)
async def owllook_delete_bookmark(request):
"""
删除书签
:param request:
:return:
: -1 用户session失效 需要重新登录
: 0 删除书签失败
: 1 删除书签成功
"""
user = request['session'].get('user', None)
data = parse_qs(str(request.body, encoding='utf-8'))
bookmarkurl = data.get('bookmarkurl', '')
if user and bookmarkurl:
bookmark = unquote(bookmarkurl[0])
try:
motor_db = motor_base.get_db()
await motor_db.user_message.update_one(
{'user': user},
{'$pull': {
'bookmarks': {
"bookmark": bookmark
}
}})
LOGGER.info('删除书签成功')
return json({'status': 1})
except Exception as e:
LOGGER.exception(e)
return json({'status': 0})
else:
return json({'status': -1})
def extract_pre_next_chapter(chapter_url, html):
"""
获取单章节上一页下一页
:param chapter_url:
:param html:
:return:
"""
next_chapter = OrderedDict()
try:
# 参考https://greasyfork.org/zh-CN/scripts/292-my-novel-reader
next_reg = r'(<a\s+.*?>.*[第上前下后][一]?[0-9]{0,6}?[页张个篇章节步].*?</a>)'
judge_reg = r'[第上前下后][一]?[0-9]{0,6}?[页张个篇章节步]'
# 这里同样需要利用bs再次解析
next_res = re.findall(next_reg,
html.replace('<<', '').replace('>>', ''), re.I)
str_next_res = '\n'.join(next_res)
next_res_soup = BeautifulSoup(str_next_res, 'html5lib')
for link in next_res_soup.find_all('a'):
text = link.text or ''
text = text.replace(' ', '')
if novels_list(text):
is_next = re.search(judge_reg, text)
# is_ok = is_chapter(text)
if is_next:
url = urljoin(chapter_url, link.get('href')) or ''
next_chapter[text[:5]] = url
# nextDic = [{v[0]: v[1]} for v in sorted(next_chapter.items(), key=lambda d: d[1])]
return next_chapter
except Exception as e:
LOGGER.exception(e)
return next_chapter
async def owllook_delete_book(request):
"""
删除书架
:param request:
:return:
: -1 用户session失效 需要重新登录
: 0 删除书架失败
: 1 删除书架成功
"""
user = request['session'].get('user', None)
data = parse_qs(str(request.body, encoding='utf-8'))
if user:
if data.get('book_url', None):
book_url = data.get('book_url', None)[0]
else:
novels_name = data.get('novels_name', '')
chapter_url = data.get('chapter_url', '')
book_url = "/chapter?url={chapter_url}&novels_name={novels_name}".format(
chapter_url=chapter_url[0], novels_name=novels_name[0])
try:
motor_db = motor_base.get_db()
await motor_db.user_message.update_one(
{'user': user},
{'$pull': {
'books_url': {
"book_url": unquote(book_url)
}
}})
LOGGER.info('删除书架成功')
return json({'status': 1})
except Exception as e:
LOGGER.exception(e)
return json({'status': 0})
else:
return json({'status': -1})
def get_html_by_requests(url, headers, timeout=15):
"""
:param url:
:return:
"""
try:
response = requests.get(url=url, headers=headers, verify=False, timeout=timeout)
response.raise_for_status()
content = response.content
charset = cchardet.detect(content)
text = content.decode(charset['encoding'])
return text
except Exception as e:
LOGGER.exception(e)
return None
async def owllook_add_book(request):
"""
添加书架
:param request:
:return:
: -1 用户session失效 需要重新登录
: 0 添加书架失败
: 1 添加书架成功
"""
user = request['session'].get('user', None)
data = parse_qs(str(request.body, encoding='utf-8'))
novels_name = data.get('novels_name', '')
chapter_url = data.get('chapter_url', '')
last_read_url = data.get('last_read_url', '')
if user and novels_name and chapter_url:
url = "/chapter?url={chapter_url}&novels_name={novels_name}".format(
chapter_url=chapter_url[0], novels_name=novels_name[0])
time = get_time()
try:
motor_db = motor_base.get_db()
res = await motor_db.user_message.update_one(
{'user': user}, {'$set': {
'last_update_time': time
}},
upsert=True)
if res:
await motor_db.user_message.update_one(
{
'user': user,
'books_url.book_url': {
'$ne': url
}
}, {
'$push': {
'books_url': {
'book_url': url,
'add_time': time,
'last_read_url': unquote(last_read_url[0])
}
}
})
LOGGER.info('书架添加成功')
return json({'status': 1})
except Exception as e:
LOGGER.exception(e)
return json({'status': 0})
else:
return json({'status': -1})
async def owllook_add_bookmark(request):
"""
添加书签
:param request:
:return:
: -1 用户session失效 需要重新登录
: 0 添加书签失败
: 1 添加书签成功
"""
user = request['session'].get('user', None)
data = parse_qs(str(request.body, encoding='utf-8'))
bookmark_url = data.get('bookmark_url', '')
if user and bookmark_url:
url = unquote(bookmark_url[0])
time = get_time()
try:
motor_db = motor_base.get_db()
res = await motor_db.user_message.update_one(
{'user': user}, {'$set': {
'last_update_time': time
}},
upsert=True)
if res:
await motor_db.user_message.update_one(
{
'user': user,
'bookmarks.bookmark': {
'$ne': url
}
}, {
'$push': {
'bookmarks': {
'bookmark': url,
'add_time': time
}
}
})
LOGGER.info('书签添加成功')
return json({'status': 1})
except Exception as e:
LOGGER.exception(e)
return json({'status': 0})
else:
return json({'status': -1})
async def change_pass(request):
"""
修改用户密码
:param request:
:return:
: -1 用户session失效 需要重新登录
: 0 修改密码失败
: 1 添加密码成功
: -2 原始密码错误
"""
user = request['session'].get('user', None)
data = parse_qs(str(request.body, encoding='utf-8'))
if user:
try:
new_pass = data.get('new_pass', None)[0]
old_pass = data.get('old_pass', None)[0]
motor_db = motor_base.get_db()
user_data = await motor_db.user.find_one({'user': user})
if user_data:
pass_first = hashlib.md5(
(CONFIG.WEBSITE["TOKEN"] +
old_pass).encode("utf-8")).hexdigest()
pass_second = hashlib.md5(
(CONFIG.WEBSITE["TOKEN"] +
new_pass).encode("utf-8")).hexdigest()
new_password = hashlib.md5(
pass_second.encode("utf-8")).hexdigest()
password = hashlib.md5(pass_first.encode("utf-8")).hexdigest()
if password == user_data.get('password'):
await motor_db.user.update_one(
{'user': user}, {'$set': {
'password': new_password
}})
LOGGER.info('修改密码成功')
return json({'status': 1})
else:
return json({'status': -2})
except Exception as e:
LOGGER.exception(e)
return json({'status': 0})
else:
return json({'status': -1})
async def owl_so_novels(request, name):
"""
360小说信息接口
:param request:
:param name: 小说名
:return: 小说相关信息
"""
name = unquote(name)
novels_name = '{name} 小说 免费阅读'.format(name=name)
try:
res = await get_novels_info(class_name='baidu',
novels_name=novels_name)
parse_result = []
if res:
parse_result = [i for i in res if i]
UniResponse.SUCCESS.update({
ResponseField.DATA: parse_result,
ResponseField.FINISH_AT: get_time()
})
return response_handle(request, UniResponse.SUCCESS, 200)
except Exception as e:
LOGGER.exception(e)
return response_handle(request, UniResponse.SERVER_UNKNOWN_ERR, 500)
async def fetch_url(self, url, params, headers):
"""
公共抓取函数
:param client:
:param url:
:param params:
:return:
"""
with async_timeout.timeout(15):
try:
async with aiohttp.ClientSession() as client:
async with client.get(url, params=params,
headers=headers) as response:
assert response.status == 200
LOGGER.info('Task url: {}'.format(response.url))
try:
text = await response.text()
except:
text = await response.read()
return text
except Exception as e:
LOGGER.exception(e)
return None
async def target_fetch(url, headers, timeout=15):
"""
:param url: target url
:return: text
"""
with async_timeout.timeout(timeout):
try:
async with aiohttp.ClientSession() as client:
async with client.get(url, headers=headers) as response:
assert response.status == 200
LOGGER.info('Task url: {}'.format(response.url))
try:
text = await response.text()
except:
try:
text = await response.read()
except aiohttp.ServerDisconnectedError as e:
LOGGER.exception(e)
text = None
return text
except Exception as e:
LOGGER.exception(str(e))
return None
async def owllook_search(request):
start = time.time()
name = str(request.args.get('wd', '')).strip()
novels_keyword = name.split(' ')[0]
motor_db = motor_base.get_db()
if not name:
return redirect('/')
else:
# 记录搜索小说名
try:
await motor_db.search_records.update_one({'keyword': name}, {'$inc': {'count': 1}}, upsert=True)
except Exception as e:
LOGGER.exception(e)
# 通过搜索引擎获取检索结果
parse_result = None
if name.startswith('!baidu'):
novels_keyword = name.split('baidu')[1].strip()
novels_name = 'intitle:{name} 小说 阅读'.format(name=novels_keyword)
parse_result = await get_novels_info(class_name='baidu', novels_name=novels_name)
elif name.startswith('!360'):
novels_keyword = name.split('360')[1].strip()
novels_name = "{name} 小说 最新章节".format(name=novels_keyword)
parse_result = await get_novels_info(class_name='so', novels_name=novels_name)
elif name.startswith('!bing'):
novels_keyword = name.split('bing')[1].strip()
novels_name = "{name} 小说 阅读 最新章节".format(name=novels_keyword)
parse_result = await get_novels_info(class_name='bing', novels_name=novels_name)
elif name.startswith('!duck_go'):
novels_keyword = name.split('duck_go')[1].strip()
novels_name = '{name} 小说 阅读 最新章节'.format(name=novels_keyword)
parse_result = await get_novels_info(class_name='duck_go', novels_name=novels_name)
else:
for each_engine in ENGINE_PRIORITY:
# for bing
if each_engine == "bing":
novels_name = "{name} 小说 阅读 最新章节".format(name=name)
parse_result = await get_novels_info(class_name='bing', novels_name=novels_name)
if parse_result:
break
# for 360 so
if each_engine == "360":
novels_name = "{name} 小说 最新章节".format(name=name)
parse_result = await get_novels_info(class_name='so', novels_name=novels_name)
if parse_result:
break
# for baidu
if each_engine == "baidu":
novels_name = 'intitle:{name} 小说 阅读'.format(name=name)
parse_result = await get_novels_info(class_name='baidu', novels_name=novels_name)
if parse_result:
break
# for duckduckgo
if each_engine == "duck_go":
novels_name = '{name} 小说 阅读 最新章节'.format(name=name)
parse_result = await get_novels_info(class_name='duck_go', novels_name=novels_name)
if parse_result:
break
if parse_result:
# result_sorted = sorted(
# parse_result, reverse=True, key=lambda res: res['timestamp']) if ':baidu' not in name else parse_result
# 优先依靠是否解析进行排序 其次以更新时间进行排序
result_sorted = sorted(
parse_result,
reverse=True,
key=itemgetter('is_recommend', 'is_parse', 'timestamp'))
user = request['session'].get('user', None)
if user:
try:
time_current = get_time()
res = await motor_db.user_message.update_one({'user': user},
{'$set': {'last_update_time': time_current}},
upsert=True)
# 此处语法操作过多 下次看一遍mongo再改
if res:
is_ok = await motor_db.user_message.update_one(
{'user': user, 'search_records.keyword': {'$ne': novels_keyword}},
{'$push': {'search_records': {'keyword': novels_keyword, 'counts': 1}}},
)
if is_ok:
await motor_db.user_message.update_one(
{'user': user, 'search_records.keyword': novels_keyword},
{'$inc': {'search_records.$.counts': 1}}
)
except Exception as e:
LOGGER.exception(e)
return template(
'result.html',
is_login=1,
user=user,
name=novels_keyword,
time='%.2f' % (time.time() - start),
result=result_sorted,
count=len(parse_result))
else:
return template(
'result.html',
is_login=0,
name=novels_keyword,
time='%.2f' % (time.time() - start),
result=result_sorted,
count=len(parse_result))
else:
return html("No Result！请将小说名反馈给本站，谢谢！")
async def get_the_latest_chapter(chapter_url, timeout=15):
try:
with async_timeout.timeout(timeout):
url = parse_qs(urlparse(chapter_url).query).get('url', '')
novels_name = parse_qs(urlparse(chapter_url).query).get(
'novels_name', '')
data = None
if url and novels_name:
url = url[0]
novels_name = novels_name[0]
netloc = urlparse(url).netloc
if netloc in LATEST_RULES.keys():
headers = {'user-agent': await get_random_user_agent()}
try:
html = await target_fetch(url=url,
headers=headers,
timeout=timeout)
if html is None:
html = get_html_by_requests(url=url,
headers=headers,
timeout=timeout)
except TypeError:
html = get_html_by_requests(url=url,
headers=headers,
timeout=timeout)
except Exception as e:
LOGGER.exception(e)
return None
try:
soup = BeautifulSoup(html, 'html5lib')
except Exception as e:
LOGGER.exception(e)
return None
latest_chapter_name, latest_chapter_url = None, None
if LATEST_RULES[netloc].plan:
meta_value = LATEST_RULES[netloc].meta_value
latest_chapter_name = soup.select(
'meta[property="{0}"]'.format(
meta_value["latest_chapter_name"])
) or soup.select('meta[name="{0}"]'.format(
meta_value["latest_chapter_name"]))
latest_chapter_name = latest_chapter_name[0].get(
'content', None) if latest_chapter_name else None
latest_chapter_url = soup.select(
'meta[property="{0}"]'.format(
meta_value["latest_chapter_url"])
) or soup.select('meta[name="{0}"]'.format(
meta_value["latest_chapter_url"]))
latest_chapter_url = urljoin(
chapter_url, latest_chapter_url[0].get(
'content',
None)) if latest_chapter_url else None
else:
selector = LATEST_RULES[netloc].selector
content_url = selector.get('content_url')
if selector.get('id', None):
latest_chapter_soup = soup.find_all(
id=selector['id'])
elif selector.get('class', None):
latest_chapter_soup = soup.find_all(
class_=selector['class'])
else:
latest_chapter_soup = soup.select(
selector.get('tag'))
if latest_chapter_soup:
if content_url == '1':
# TODO
pass
elif content_url == '0':
# TODO
pass
else:
latest_chapter_url = content_url + latest_chapter_soup[
0].get('href', None)
latest_chapter_name = latest_chapter_soup[0].get(
'title', None)
if latest_chapter_name and latest_chapter_url:
time_current = get_time()
# print(latest_chapter_url)
data = {
"latest_chapter_name":
latest_chapter_name,
"latest_chapter_url":
latest_chapter_url,
"owllook_chapter_url":
chapter_url,
"owllook_content_url":
"/owllook_content?url={latest_chapter_url}&name={name}&chapter_url={chapter_url}&novels_name={novels_name}"
.format(
latest_chapter_url=latest_chapter_url,
name=latest_chapter_name,
chapter_url=url,
novels_name=novels_name,
),
}
# 存储最新章节
motor_db = MotorBase().get_db()
await motor_db.latest_chapter.update_one(
{
"novels_name": novels_name,
'owllook_chapter_url': chapter_url
}, {
'$set': {
'data': data,
"finished_at": time_current
}
},
upsert=True)
return data
except Exception as e:
LOGGER.exception(e)
return None
async def owllook_content(request):
"""
返回小说章节内容页
: content_url 这决定当前U页面url的生成方式
: url 章节内容页源url
: chapter_url 小说目录源url
: novels_name 小说名称
:return: 小说章节内容页
"""
url = request.args.get('url', None)
chapter_url = request.args.get('chapter_url', None)
novels_name = request.args.get('novels_name', None)
name = request.args.get('name', '')
is_ajax = request.args.get('is_ajax', '')
# 当小说内容url不在解析规则内 跳转到原本url
netloc = get_netloc(url)
if netloc not in RULES.keys():
return redirect(url)
user = request['session'].get('user', None)
# 拼接小说目录url
book_url = "/chapter?url={chapter_url}&novels_name={novels_name}".format(
chapter_url=chapter_url,
novels_name=novels_name)
motor_db = motor_base.get_db()
if url == chapter_url:
# 阅读到最后章节时候 在数据库中保存最新阅读章节
if user and is_ajax == "owl_cache":
owl_referer = request.headers.get('Referer', '').split('owllook_content')[1]
if owl_referer:
latest_read = "/owllook_content" + owl_referer
await motor_db.user_message.update_one(
{'user': user, 'books_url.book_url': book_url},
{'$set': {'books_url.$.last_read_url': latest_read}})
return redirect(book_url)
content_url = RULES[netloc].content_url
content_data = await cache_owllook_novels_content(url=url, netloc=netloc)
if content_data:
try:
content = content_data.get('content', '获取失败')
next_chapter = content_data.get('next_chapter', [])
title = content_data.get('title', '').replace(novels_name, '')
name = title if title else name
# 拼接小说书签url
bookmark_url = "{path}?url={url}&name={name}&chapter_url={chapter_url}&novels_name={novels_name}".format(
path=request.path,
url=url,
name=name,
chapter_url=chapter_url,
novels_name=novels_name
)
# 破坏广告链接
content = str(content).strip('[]Jjs,').replace('http', 'hs')
if user:
bookmark = await motor_db.user_message.find_one({'user': user, 'bookmarks.bookmark': bookmark_url})
book = await motor_db.user_message.find_one({'user': user, 'books_url.book_url': book_url})
bookmark = 1 if bookmark else 0
if book:
# 当书架中存在该书源
book = 1
# 保存最后一次阅读记录
if is_ajax == "owl_cache":
owl_referer = request.headers.get('Referer', bookmark_url).split('owllook_content')[1]
latest_read = "/owllook_content" + owl_referer
await motor_db.user_message.update_one(
{'user': user, 'books_url.book_url': book_url},
{'$set': {'books_url.$.last_read_url': latest_read}})
else:
book = 0
if is_ajax == "owl_cache":
owl_cache_dict = dict(
is_login=1,
user=user,
name=name,
url=url,
bookmark=bookmark,
book=book,
content_url=content_url,
chapter_url=chapter_url,
novels_name=novels_name,
next_chapter=next_chapter,
soup=content
)
return json(owl_cache_dict)
return template(
'content.html',
is_login=1,
user=user,
name=name,
url=url,
bookmark=bookmark,
book=book,
content_url=content_url,
chapter_url=chapter_url,
novels_name=novels_name,
next_chapter=next_chapter,
soup=content)
else:
if is_ajax == "owl_cache":
owl_cache_dict = dict(
is_login=0,
name=name,
url=url,
bookmark=0,
book=0,
content_url=content_url,
chapter_url=chapter_url,
novels_name=novels_name,
next_chapter=next_chapter,
soup=content
)
return json(owl_cache_dict)
return template(
'content.html',
is_login=0,
name=name,
url=url,
bookmark=0,
book=0,
content_url=content_url,
chapter_url=chapter_url,
novels_name=novels_name,
next_chapter=next_chapter,
soup=content)
except Exception as e:
LOGGER.exception(e)
return redirect(book_url)
else:
if user:
is_login = 1
user = user
return template('parse_error.html', url=url, is_login=is_login, user=user)
else:
is_login = 0
return template('parse_error.html', url=url, is_login=is_login)
async def author_notification(request):
"""
作者新书通知
:param request:
:return:
: -1 用户session失效 需要重新登录
: 2 无该作者信息
: 3 作者已经添加
: 4 超过添加的上限
: 0 操作失败
: 1 操作成功
"""
user = request['session'].get('user', None)
user_data = parse_qs(str(request.body, encoding='utf-8'))
if user:
try:
motor_db = motor_base.get_db()
all_authors = await motor_db.user_message.find_one(
{'user': user}, {
'author_latest': 1,
'_id': 0
})
count = len(all_authors.get('author_latest', []))
if count == CONFIG.WEBSITE.get("AUTHOR_LATEST_COUNT", 5):
return json({'status': 4})
author_name = user_data.get('author_name', None)[0]
data = []
author_cursor = motor_db.all_books.find({'author': author_name}, {
'name': 1,
'url': 1,
'_id': 0
})
async for document in author_cursor:
data.append(document)
if data:
time = get_time()
res = await motor_db.user_message.update_one(
{'user': user}, {'$set': {
'last_update_time': time
}},
upsert=True)
is_exist = await motor_db.user_message.find_one({
'user':
user,
'author_latest.author_name':
author_name
})
if is_exist:
return json({'status': 3})
if res:
await motor_db.user_message.update_one(
{
'user': user,
'author_latest.author_name': {
'$ne': author_name
}
}, {
'$push': {
'author_latest': {
'author_name': author_name,
'add_time': time
}
}
})
is_author_exist = await motor_db.author_message.find_one(
{'name': author_name})
if not is_author_exist:
author_data = {
"author_name": author_name,
"nums": len(data),
"updated_time": get_time(),
}
await motor_db.author_message.save(author_data)
LOGGER.info('作者添加成功')
return json({'status': 1})
else:
return json({'status': 2})
else:
return json({'status': 2})
except Exception as e:
LOGGER.exception(e)
return json({'status': 0})
else:
return json({'status': -1})