async def update_all_books():
try:
motor_db = MotorBase().db
# 获取所有书架链接游标
books_url_cursor = motor_db.user_message.find({}, {
'books_url.book_url': 1,
'_id': 0
})
book_urls = []
already_urls = set()
# url_tasks = [get_the_latest_chapter(each_url) for each_url in set(book_urls)]
# tasks = [asyncio.ensure_future(i) for i in url_tasks]
# return await asyncio.gather(*tasks)
async for document in books_url_cursor:
if document:
books_url = document['books_url']
# 一组书架链接列表数据
# book_urls += [book_url['book_url'] for book_url in books_url]
for book_url in books_url:
chapter_url = book_url['book_url']
if chapter_url not in already_urls:
try:
with async_timeout.timeout(20):
await get_the_latest_chapter(chapter_url)
except Exception as e:
LOGGER.exception(e)
already_urls.add(chapter_url)
except Exception as e:
LOGGER.exception(e)
return False
class ZHNovelInfoSpider(Spider):
start_urls = []
request_config = {'RETRIES': 3, 'DELAY': 2, 'TIMEOUT': 10}
motor_db = MotorBase(loop=loop).get_db()
async def parse(self, res):
item = await ZHNovelInfoItem.get_item(html=res.html)
item_data = {
'novel_name': item.novel_name,
'author': item.author,
'cover': item.cover,
'abstract': item.abstract,
'status': item.status,
'novels_type': item.novels_type,
'novel_chapter_url': item.novel_chapter_url,
'target_url': res.url,
'spider': 'zongheng',
'updated_at': time.strftime("%Y-%m-%d %X", time.localtime()),
}
print('获取 {} 小说信息成功'.format(item_data['novel_name']))
print(item_data)
await self.motor_db.all_novels_info.update_one(
{
'novel_name': item_data['novel_name'],
'spider': 'zongheng'
}, {'$set': item_data},
upsert=True)
async def owllook_add_bookmark(request):
"""
:param request:
:return:
: -1 用户session失效 需要重新登录
: 0 添加书签失败
: 1 添加书签成功
"""
user = request['session'].get('user', None)
data = parse_qs(str(request.body, encoding='utf-8'))
bookmarkurl = data.get('bookmarkurl', '')
if user and bookmarkurl:
url = unquote(bookmarkurl[0])
time = get_time()
try:
motor_db = MotorBase().db
res = await motor_db.user_message.update_one({'user': user}, {'$set': {'last_update_time': time}},
upsert=True)
if res:
await motor_db.user_message.update_one(
{'user': user, 'bookmarks.bookmark': {'$ne': url}},
{'$push': {'bookmarks': {'bookmark': url, 'add_time': time}}})
LOGGER.info('书签添加成功')
return json({'status': 1})
except Exception as e:
LOGGER.exception(e)
return json({'status': 0})
else:
return json({'status': -1})
async def get_tag():
motor_db = MotorBase().get_db()
novels = Novels()
# 获取所有书架链接游标
books_url_cursor = motor_db.user_message.find({}, {'books_url.book_url': 1, 'user': 1, '_id': 0})
async for document in books_url_cursor:
if document:
books_url = document.get('books_url', None)
if books_url:
all_user = {}
user = document['user']
all_user[user + '_novels'] = []
all_user[user + '_tag'] = []
all_user[user + '_author'] = []
for book_url in books_url:
chapter_url = book_url['book_url']
novels_name = parse_qs(urlparse(chapter_url).query).get('novels_name', '')[0]
all_user[user + '_novels'].append(novels_name)
novels_info = novels.search_name(novels_name)
if novels_info:
novels_type = novels_info['novels_type'].split('#')
author = novels_info['author']
all_user[user + '_author'].append(author)
all_user[user + '_tag'].extend(novels_type)
data = {
'user_novels': all_user[user + '_novels'],
'user_tag': all_user[user + '_tag'],
'user_author': all_user[user + '_author'],
}
await motor_db.user_tag.update_one(
{"user": user},
{'$set': {'data': data, "updated_at": get_time()}}, upsert=True)
pprint(data)
async def owllook_delete_bookmark(request):
"""
:param request:
:return:
: -1 用户session失效 需要重新登录
: 0 删除书签失败
: 1 删除书签成功
"""
user = request['session'].get('user', None)
data = parse_qs(str(request.body, encoding='utf-8'))
bookmarkurl = data.get('bookmarkurl', '')
if user and bookmarkurl:
bookmark = unquote(bookmarkurl[0])
try:
motor_db = MotorBase().db
await motor_db.user_message.update_one({'user': user},
{'$pull': {'bookmarks': {"bookmark": bookmark}}})
LOGGER.info('删除书签成功')
return json({'status': 1})
except Exception as e:
LOGGER.exception(e)
return json({'status': 0})
else:
return json({'status': -1})
async def similar_user(request):
user = request['session'].get('user', None)
if user:
motor_db = MotorBase().db
try:
similar_info = await motor_db.user_recommend.find_one(
{'user': user})
if similar_info:
similar_user = similar_info['similar_user']
user_tag = similar_info['user_tag']
updated_at = similar_info['updated_at']
return template('similar_user.html',
title='与' + user + '相似的书友',
is_login=1,
is_similar=1,
user=user,
similar_user=similar_user,
user_tag=user_tag,
updated_at=updated_at)
else:
return template('similar_user.html',
title='与' + user + '相似的书友',
is_login=1,
is_similar=1,
user=user)
except Exception as e:
LOGGER.error(e)
return redirect('/')
else:
return redirect('/')
async def owllook_add_book(request):
"""
:param request:
:return:
: -1 用户session失效 需要重新登录
: 0 添加书架失败
: 1 添加书架成功
"""
user = request['session'].get('user', None)
data = parse_qs(str(request.body, encoding='utf-8'))
novels_name = data.get('novels_name', '')
chapter_url = data.get('chapter_url', '')
last_read_url = data.get('last_read_url', '')
if user and novels_name and chapter_url:
url = "/chapter?url={chapter_url}&novels_name={novels_name}".format(chapter_url=chapter_url[0],
novels_name=novels_name[0])
time = get_time()
try:
motor_db = MotorBase().db
res = await motor_db.user_message.update_one({'user': user}, {'$set': {'last_update_time': time}},
upsert=True)
if res:
await motor_db.user_message.update_one(
{'user': user, 'books_url.book_url': {'$ne': url}},
{'$push': {
'books_url': {'book_url': url, 'add_time': time, 'last_read_url': unquote(last_read_url[0])}}})
LOGGER.info('书架添加成功')
return json({'status': 1})
except Exception as e:
LOGGER.exception(e)
return json({'status': 0})
else:
return json({'status': -1})
async def owllook_delete_book(request):
"""
:param request:
:return:
: -1 用户session失效 需要重新登录
: 0 删除书架失败
: 1 删除书架成功
"""
user = request['session'].get('user', None)
data = parse_qs(str(request.body, encoding='utf-8'))
if user:
if data.get('book_url', None):
book_url = data.get('book_url', None)[0]
else:
novels_name = data.get('novels_name', '')
chapter_url = data.get('chapter_url', '')
book_url = "/chapter?url={chapter_url}&novels_name={novels_name}".format(chapter_url=chapter_url[0],
novels_name=novels_name[0])
try:
motor_db = MotorBase().db
await motor_db.user_message.update_one({'user': user},
{'$pull': {'books_url': {"book_url": unquote(book_url)}}})
LOGGER.info('删除书架成功')
return json({'status': 1})
except Exception as e:
LOGGER.exception(e)
return json({'status': 0})
else:
return json({'status': -1})
async def books(request):
user = request['session'].get('user', None)
if user:
motor_db = MotorBase().db
try:
data = await motor_db.user_message.find_one({'user': user})
if data:
books_url = data.get('books_url', None)
if books_url:
result = []
for i in books_url:
item_result = {}
book_url = i.get('book_url', None)
last_read_url = i.get("last_read_url", "")
book_query = parse_qs(urlparse(book_url).query)
last_read_chapter_name = parse_qs(last_read_url).get(
'name', ['暂无'])[0]
item_result['novels_name'] = book_query.get(
'novels_name', '')[0] if book_query.get(
'novels_name', '') else ''
item_result['book_url'] = book_url
latest_data = await motor_db.latest_chapter.find_one(
{'owllook_chapter_url': book_url})
if latest_data:
item_result['latest_chapter_name'] = latest_data[
'data']['latest_chapter_name']
item_result['owllook_content_url'] = latest_data[
'data']['owllook_content_url']
else:
get_latest_data = await get_the_latest_chapter(
book_url) or {}
item_result[
'latest_chapter_name'] = get_latest_data.get(
'latest_chapter_name', '暂未获取，请反馈')
item_result[
'owllook_content_url'] = get_latest_data.get(
'owllook_content_url', '')
item_result['add_time'] = i.get('add_time', '')
item_result[
"last_read_url"] = last_read_url if last_read_url else book_url
item_result[
"last_read_chapter_name"] = last_read_chapter_name
result.append(item_result)
return template(
'admin_books.html',
title='{user}的书架 - owllook'.format(user=user),
is_login=1,
user=user,
is_bookmark=1,
result=result[::-1])
return template('admin_books.html',
title='{user}的书架 - owllook'.format(user=user),
is_login=1,
user=user,
is_bookmark=0)
except Exception as e:
LOGGER.error(e)
return redirect('/')
else:
return redirect('/')
async def get_user_tag():
motor_db = MotorBase().get_db()
user_tag_cursor = motor_db.user_tag.find({}, {
'data.user_tag': 1,
'user': 1,
'_id': 0
})
result = {}
async for document in user_tag_cursor:
if document['data']['user_tag']:
result[document['user'].replace(
'.', '·')] = document['data']['user_tag']
for key, value in result.items():
result_copy = deepcopy(result)
del result_copy[key]
cos = CosineSimilarity(value, result_copy)
vector = cos.create_vector()
resultDic = cos.calculate(vector)
pprint(resultDic)
# pprint(type(resultList[1]))
await motor_db.user_recommend.update_one({"user": key}, {
'$set': {
'similar_user': resultDic,
'user_tag': result[key],
"updated_at": get_time()
}
},
upsert=True)
async def update_all_books(loop, timeout=15):
try:
motor_db = MotorBase().get_db()
# 获取所有书架链接游标
books_url_cursor = motor_db.user_message.find({}, {
'books_url.book_url': 1,
'_id': 0
})
book_urls = []
already_urls = set()
async for document in books_url_cursor:
if document:
books_url = document['books_url']
for book_url in books_url:
chapter_url = book_url['book_url']
if chapter_url not in already_urls:
try:
await get_the_latest_chapter(
chapter_url, loop, timeout)
except Exception as e:
LOGGER.exception(e)
already_urls.add(chapter_url)
# 一组书架链接列表数据
# book_urls += [book_url['book_url'] for book_url in books_url]
# url_tasks = [get_the_latest_chapter(each_url, loop) for each_url in set(book_urls)]
# tasks = [asyncio.ensure_future(i) for i in url_tasks]
# try:
# await asyncio.gather(*tasks)
# except asyncio.TimeoutError as e:
# pass
except Exception as e:
LOGGER.exception(e)
return False
class ZHNovelsSpider(Spider):
start_urls = [
'http://book.zongheng.com/store/c0/c0/b9/u0/p1/v9/s9/t0/ALL.html'
]
request_config = {'RETRIES': 8, 'DELAY': 0, 'TIMEOUT': 3}
concurrency = 60
motor_db = MotorBase(loop=loop).get_db()
async def parse(self, res):
items_data = await ZHNovelsItem.get_items(html=res.html)
tasks = []
for item in items_data:
if item.novel_url:
res_dic = {
'novel_url': item.novel_url,
'novel_name': item.novel_name,
'novel_author': item.novel_author,
'novel_author_home_url': item.novel_author_home_url,
'novel_type': item.novel_type,
'novel_cover': item.novel_cover,
'novel_abstract': item.novel_abstract,
'novel_latest_chapter': item.novel_latest_chapter,
'spider': 'zongheng',
'updated_at': time.strftime("%Y-%m-%d %X",
time.localtime()),
}
tasks.append(asyncio.ensure_future(self.save(res_dic)))
# if self.all_novels_col.find_one(
# {"novel_name": item.novel_name, 'novel_author': item.novel_author}) is None:
# self.all_novels_col.insert_one(res_dic)
# # async_callback(self.save, res_dic=res_dic)
# print(item.novel_name + ' - 抓取成功')
good_nums = 0
if tasks:
done_list, pending_list = await asyncio.wait(tasks)
for task in done_list:
if task.result():
good_nums += 1
print(f"共{len(tasks)}本小说，抓取成功{good_nums}本")
async def save(self, res_dic):
# 存进数据库
res_dic = res_dic
try:
await self.motor_db.all_novels.update_one(
{
'novel_url': res_dic['novel_url'],
'novel_name': res_dic['novel_name']
}, {'$set': res_dic},
upsert=True)
print(res_dic['novel_name'] + ' - 抓取成功')
return True
except Exception as e:
self.logger.exception(e)
return False
async def cache_others_search_ranking(spider='qidian', novel_type='全部类别'):
motor_db = MotorBase().get_db()
item_data = await motor_db.novels_ranking.find_one(
{
'spider': spider,
'type': novel_type
}, {
'data': 1,
'_id': 0
})
return item_data
async def save(self, **kwargs):
# 存进数据库
res_dic = kwargs.get('res_dic')
try:
motor_db = MotorBase().get_db()
await motor_db.all_novels_info.update_one({
'novel_name': res_dic['novel_name'], 'spider': 'heiyan'},
{'$set': res_dic},
upsert=True)
except Exception as e:
self.logger.exception(e)
async def cache_owllook_search_ranking():
motor_db = MotorBase().get_db()
keyword_cursor = motor_db.search_records.find(
{'count': {'$gte': 50}},
{'keyword': 1, 'count': 1, '_id': 0}
).sort('count', -1).limit(35)
result = []
index = 1
async for document in keyword_cursor:
result.append({'keyword': document['keyword'], 'count': document['count'], 'index': index})
index += 1
return result
async def get_user_tag():
motor_db = MotorBase().get_db()
user_tag_cursor = motor_db.user_tag.find({}, {
'data': 1,
'user': 1,
'_id': 0
})
result = {}
user_book_dict = {}
async for document in user_tag_cursor:
if document['data']:
user_book_dict[document['user'].replace(
'.', '·')] = document['data']
result[document['user'].replace(
'.', '·')] = document['data']['user_tag']
for key, value in result.items():
if not value:
continue
print("\nUser:"******"User tags:", set(value))
print("User books:", user_book_dict[key]["user_novels"])
result_copy = deepcopy(result)
del result_copy[key]
cos = CosineSimilarity(value, result_copy)
vector = cos.create_vector()
resultDic = cos.calculate(vector)
print("相似用户:")
pprint(resultDic[:10])
# pprint(type(resultList[1]))
booksDic = collections.defaultdict(float)
for userBooks in resultDic:
for simuser, simrate in userBooks.items():
# print(simuser, simrate)
for book in user_book_dict[simuser]["user_novels"]:
booksDic[book] += simrate
# print(booksDic)
# 推荐20本书，注意这里的推荐并没有去除用户收藏的书籍
recommend = sorted(booksDic.items(), key=itemgetter(1),
reverse=True)[0:20]
print("书籍推荐：")
pprint(recommend)
recommend_novels = [book for book, simrate in recommend]
await motor_db.user_recommend.update_one({"user": key}, {
'$set': {
'similar_user': resultDic,
'user_tag': result[key],
'recommend_novels': recommend_novels,
"updated_at": get_time()
}
},
upsert=True)
async def cache_owllook_search_ranking():
motor_db = MotorBase().db
keyword_cursor = motor_db.search_records.find({
'count': {
'$gte': 50
}
}, {
'keyword': 1,
'_id': 0
}).sort('count', -1).limit(25)
result = []
async for document in keyword_cursor:
result.append(document['keyword'])
return result
class QidianNovelsSpider(Spider):
# start_urls = ['https://www.qidian.com/all?page=1']
request_config = {'RETRIES': 10, 'DELAY': 0, 'TIMEOUT': 3}
concurrency = 100
motor_db = MotorBase(loop=loop).get_db()
async def parse(self, res):
items_data = await QidianNovelsItem.get_items(html=res.html)
tasks = []
for item in items_data:
res_dic = {
'novel_url': item.novel_url,
'novel_name': item.novel_name,
'novel_author': item.novel_author,
'novel_author_home_url': item.novel_author_home_url,
'novel_type': item.novel_type,
'novel_cover': item.novel_cover,
'novel_abstract': item.novel_abstract,
'spider': 'qidian',
'updated_at': time.strftime("%Y-%m-%d %X", time.localtime()),
}
tasks.append(asyncio.ensure_future(self.save(res_dic)))
good_nums = 0
if tasks:
done_list, pending_list = await asyncio.wait(tasks)
for task in done_list:
if task.result():
good_nums += 1
print(f"共{len(tasks)}本小说，抓取成功{good_nums}本")
async def save(self, res_dic):
# 存进数据库
try:
await self.motor_db.all_novels.update_one(
{
'novel_url': res_dic['novel_url'],
'novel_name': res_dic['novel_name']
}, {'$set': res_dic},
upsert=True)
print(res_dic['novel_name'] + ' - 抓取成功')
return True
except Exception as e:
self.logger.exception(e)
return False
async def save(self, **kwargs):
# 存进数据库
res_dic = kwargs.get('res_dic')
try:
motor_db = MotorBase().db
await motor_db.novels_ranking.update_one(
{'target_url': res_dic['target_url']}, {
'$set': {
'data':
res_dic['data'],
'spider':
res_dic['spider'],
'finished_at':
time.strftime("%Y-%m-%d %X", time.localtime())
}
},
upsert=True)
except Exception as e:
self.logger.exception(e)
async def bookmarks(request):
user = request['session'].get('user', None)
if user:
motor_db = MotorBase().db
try:
data = await motor_db.user_message.find_one({'user': user})
if data:
# 获取所有书签
bookmarks = data.get('bookmarks', None)
if bookmarks:
result = []
for i in bookmarks:
item_result = {}
bookmark = i.get('bookmark', None)
query = parse_qs(urlparse(bookmark).query)
item_result['novels_name'] = query.get(
'novels_name', '')[0] if query.get(
'novels_name', '') else ''
item_result['chapter_name'] = query.get(
'name', '')[0] if query.get('name', '') else ''
item_result['chapter_url'] = query.get(
'chapter_url', '')[0] if query.get(
'chapter_url', '') else ''
item_result['bookmark'] = bookmark
item_result['add_time'] = i.get('add_time', '')
result.append(item_result)
return template(
'admin_bookmarks.html',
title='{user}的书签 - owllook'.format(user=user),
is_login=1,
user=user,
is_bookmark=1,
result=result[::-1])
return template('admin_bookmarks.html',
title='{user}的书签 - owllook'.format(user=user),
is_login=1,
user=user,
is_bookmark=0)
except Exception as e:
LOGGER.error(e)
return redirect('/')
else:
return redirect('/')
async def parse(self, res):
self.motor_db = MotorBase(loop=self.loop).get_db()
item = await HYNovelInfoItem.get_item(html=res.html)
item_data = {
'novel_name': item.novel_name,
'author': item.author,
'cover': item.cover,
'abstract': item.abstract,
'status': item.status,
'novels_type': item.novels_type,
'novel_chapter_url': item.novel_chapter_url,
'latest_chapter': item.latest_chapter,
'latest_chapter_time': item.latest_chapter_time,
'spider': 'heiyan',
'target_url': res.url,
'updated_at': time.strftime("%Y-%m-%d %X", time.localtime())
}
print('获取 {} 小说信息成功'.format(item_data['novel_name']))
await self.save(res_dic=item_data)
async def owllook_register(request):
"""
用户注册 不允许重名
:param request:
:return:
: -1 用户名已存在
: 0 用户名或密码不能为空
: 1 注册成功
"""
register_data = parse_qs(str(request.body, encoding='utf-8'))
user = register_data.get('user', [None])[0]
pwd = register_data.get('pwd', [None])[0]
email = register_data.get('email', [None])[0]
answer = register_data.get('answer', [None])[0]
reg_index = request.cookies['reg_index']
if user and pwd and email and answer and reg_index:
motor_db = MotorBase().db
is_exist = await motor_db.user.find_one({'user': user})
if not is_exist:
# 验证问题答案是否准确
real_answer = get_real_answer(str(reg_index))
if real_answer and real_answer == answer:
pass_first = hashlib.md5((WEBSITE["TOKEN"] + pwd).encode("utf-8")).hexdigest()
password = hashlib.md5(pass_first.encode("utf-8")).hexdigest()
time = get_time()
data = {
"user": user,
"password": password,
"email": email,
"register_time": time,
}
await motor_db.user.save(data)
return json({'status': 1})
else:
return json({'status': -2})
else:
return json({'status': -1})
else:
return json({'status': 0})
async def owllook_login(request):
"""
用户登录
:param request:
:return:
: -1 用户名或密码不能为空
: 0 用户名或密码错误
: 1 登陆成功
"""
login_data = parse_qs(str(request.body, encoding='utf-8'))
user = login_data.get('user', [None])[0]
pwd = login_data.get('pwd', [None])[0]
if user and pwd:
motor_db = MotorBase().db
data = await motor_db.user.find_one({'user': user})
if data:
pass_first = hashlib.md5((WEBSITE["TOKEN"] + pwd).encode("utf-8")).hexdigest()
password = hashlib.md5(pass_first.encode("utf-8")).hexdigest()
if password == data.get('password'):
response = json({'status': 1})
# 将session_id存于cokies
date = datetime.datetime.now()
response.cookies['owl_sid'] = request['session'].sid
response.cookies['owl_sid']['expires'] = date + datetime.timedelta(days=30)
response.cookies['owl_sid']['httponly'] = True
# 此处设置存于服务器session的user值
request['session']['user'] = user
# response.cookies['user'] = user
# response.cookies['user']['expires'] = date + datetime.timedelta(days=30)
# response.cookies['user']['httponly'] = True
# response = json({'status': 1})
# response.cookies['user'] = user
return response
else:
return json({'status': -2})
return json({'status': -1})
else:
return json({'status': 0})
async def update_all_books():
try:
motor_db = MotorBase().db
# 获取所有书架链接游标
books_url_cursor = motor_db.user_message.find({}, {
'books_url.book_url': 1,
'_id': 0
})
# 已更新url集合
already_urls = set()
async for document in books_url_cursor:
if document:
books_url = document['books_url']
# 一组书架链接列表数据
for book_url in books_url:
chapter_url = book_url['book_url']
if chapter_url not in already_urls:
await get_the_latest_chapter(chapter_url)
already_urls.add(chapter_url)
return True
except Exception as e:
LOGGER.exception(e)
return False
def setup_db(operate_bp, loop):
global motor_base
motor_base = MotorBase()
async def get_the_latest_chapter(chapter_url, loop=None, timeout=15):
try:
with async_timeout.timeout(timeout):
url = parse_qs(urlparse(chapter_url).query).get('url', '')
novels_name = parse_qs(urlparse(chapter_url).query).get(
'novels_name', '')
data = None
if url and novels_name:
url = url[0]
novels_name = novels_name[0]
netloc = urlparse(url).netloc
if netloc in LATEST_RULES.keys():
async with aiohttp.ClientSession(loop=loop) as client:
headers = {'user-agent': await get_random_user_agent()}
try:
html = await target_fetch(client=client,
url=url,
headers=headers,
timeout=timeout)
if html is None:
html = get_html_by_requests(url=url,
headers=headers,
timeout=timeout)
except TypeError:
html = get_html_by_requests(url=url,
headers=headers,
timeout=timeout)
except Exception as e:
LOGGER.exception(e)
return None
try:
soup = BeautifulSoup(html, 'html5lib')
except Exception as e:
LOGGER.exception(e)
return None
latest_chapter_name, latest_chapter_url = None, None
if LATEST_RULES[netloc].plan:
meta_value = LATEST_RULES[netloc].meta_value
latest_chapter_name = soup.select(
'meta[property="{0}"]'.format(
meta_value["latest_chapter_name"]))
latest_chapter_name = latest_chapter_name[0].get(
'content',
None) if latest_chapter_name else None
latest_chapter_url = soup.select(
'meta[property="{0}"]'.format(
meta_value["latest_chapter_url"]))
latest_chapter_url = urljoin(
url, latest_chapter_url[0].get(
'content',
None)) if latest_chapter_url else None
else:
selector = LATEST_RULES[netloc].selector
content_url = selector.get('content_url')
if selector.get('id', None):
latest_chapter_soup = soup.find_all(
id=selector['id'])
elif selector.get('class', None):
latest_chapter_soup = soup.find_all(
class_=selector['class'])
else:
latest_chapter_soup = soup.select(
selector.get('tag'))
if latest_chapter_soup:
if content_url == '1':
# TODO
pass
elif content_url == '0':
# TODO
pass
else:
latest_chapter_url = content_url + latest_chapter_soup[
0].get('href', None)
latest_chapter_name = latest_chapter_soup[
0].get('title', None)
if latest_chapter_name and latest_chapter_url:
time_current = get_time()
# print(latest_chapter_url)
data = {
"latest_chapter_name":
latest_chapter_name,
"latest_chapter_url":
latest_chapter_url,
"owllook_chapter_url":
chapter_url,
"owllook_content_url":
"/owllook_content?url={latest_chapter_url}&name={name}&chapter_url={chapter_url}&novels_name={novels_name}"
.format(
latest_chapter_url=latest_chapter_url,
name=latest_chapter_name,
chapter_url=url,
novels_name=novels_name,
),
}
# 存储最新章节
motor_db = MotorBase().get_db()
await motor_db.latest_chapter.update_one(
{
"novels_name": novels_name,
'owllook_chapter_url': chapter_url
}, {
'$set': {
'data': data,
"finished_at": time_current
}
},
upsert=True)
return data
except Exception as e:
LOGGER.exception(e)
return None
async def owllook_search(request):
start = time.time()
name = request.args.get('wd', '').strip()
motor_db = MotorBase().db
if not name:
return redirect('/')
else:
# 记录搜索小说名
try:
await motor_db.search_records.update_one({'keyword': name}, {'$inc': {'count': 1}}, upsert=True)
except Exception as e:
LOGGER.exception(e)
# 通过搜索引擎获取检索结果
parse_result = None
for each_engine in ENGINE_PRIORITY:
# for 360 so
if each_engine == "360":
novels_name = "{name} 小说 最新章节".format(name=name)
parse_result = await cache_owllook_so_novels_result(novels_name)
if parse_result:
break
if each_engine == "baidu":
# for baidu
novels_name = 'intitle:{name} 小说 阅读'.format(name=name) if ':baidu' not in name else name.split('baidu')[1]
parse_result = await cache_owllook_baidu_novels_result(novels_name)
if parse_result:
break
if parse_result:
# result_sorted = sorted(
# parse_result, reverse=True, key=lambda res: res['timestamp']) if ':baidu' not in name else parse_result
# 优先依靠是否解析进行排序 其次以更新时间进行排序
result_sorted = sorted(
parse_result, reverse=True,
key=itemgetter('is_parse', 'timestamp')) if ':baidu' not in name else parse_result
user = request['session'].get('user', None)
if user:
try:
time_current = get_time()
res = await motor_db.user_message.update_one({'user': user},
{'$set': {'last_update_time': time_current}},
upsert=True)
# 此处语法操作过多 下次看一遍mongo再改
if res:
is_ok = await motor_db.user_message.update_one(
{'user': user, 'search_records.keyword': {'$ne': name}},
{'$push': {'search_records': {'keyword': name, 'counts': 1}}},
)
if is_ok:
await motor_db.user_message.update_one(
{'user': user, 'search_records.keyword': name},
{'$inc': {'search_records.$.counts': 1}}
)
except Exception as e:
LOGGER.exception(e)
return template(
'result.html',
is_login=1,
user=user,
name=name,
time='%.2f' % (time.time() - start),
result=result_sorted,
count=len(parse_result))
else:
return template(
'result.html',
is_login=0,
name=name,
time='%.2f' % (time.time() - start),
result=result_sorted,
count=len(parse_result))
else:
return html("No Result！请将小说名反馈给本站，谢谢！")
def setup_db(novels_bp, loop):
global motor_base
motor_base = MotorBase()
def setup_db(admin_bp, loop):
global motor_base
motor_base = MotorBase()
def setup_db(rank_bp, loop):
global motor_base
motor_base = MotorBase()