async def fetch(client, url, name, is_web):
with async_timeout.timeout(15):
try:
headers = {'user-agent': get_random_user_agent()}
if is_web:
params = {
'wd': name,
'ie': 'utf-8',
'rn': CONFIG.BAIDU_RN,
'vf_bl': 1
}
else:
params = {'word': name}
async with client.get(url, params=params,
headers=headers) as response:
assert response.status == 200
LOGGER.info('Task url: {}'.format(response.url))
try:
text = await response.text()
except:
text = await response.read()
return text
except Exception as e:
LOGGER.exception(e)
return None
async def change_email(request):
"""
修改用户邮箱
:param request:
:return:
: -1 用户session失效 需要重新登录
: 0 修改邮箱失败
: 1 添加邮箱成功
"""
user = request['session'].get('user', None)
data = parse_qs(str(request.body, encoding='utf-8'))
if user:
try:
email = data.get('email', None)[0]
motor_db = motor_base.get_db()
await motor_db.user.update_one({'user': user},
{'$set': {
'email': email
}})
LOGGER.info('修改邮箱成功')
return json({'status': 1})
except Exception as e:
LOGGER.exception(e)
return json({'status': 0})
else:
return json({'status': -1})
async def owllook_delete_bookmark(request):
"""
删除书签
:param request:
:return:
: -1 用户session失效 需要重新登录
: 0 删除书签失败
: 1 删除书签成功
"""
user = request['session'].get('user', None)
data = parse_qs(str(request.body, encoding='utf-8'))
bookmarkurl = data.get('bookmarkurl', '')
if user and bookmarkurl:
bookmark = unquote(bookmarkurl[0])
try:
motor_db = motor_base.get_db()
await motor_db.user_message.update_one(
{'user': user},
{'$pull': {
'bookmarks': {
"bookmark": bookmark
}
}})
LOGGER.info('删除书签成功')
return json({'status': 1})
except Exception as e:
LOGGER.exception(e)
return json({'status': 0})
else:
return json({'status': -1})
async def owllook_delete_book(request):
"""
删除书架
:param request:
:return:
: -1 用户session失效 需要重新登录
: 0 删除书架失败
: 1 删除书架成功
"""
user = request['session'].get('user', None)
data = parse_qs(str(request.body, encoding='utf-8'))
if user:
if data.get('book_url', None):
book_url = data.get('book_url', None)[0]
else:
novels_name = data.get('novels_name', '')
chapter_url = data.get('chapter_url', '')
book_url = "/chapter?url={chapter_url}&novels_name={novels_name}".format(
chapter_url=chapter_url[0], novels_name=novels_name[0])
try:
motor_db = motor_base.get_db()
await motor_db.user_message.update_one(
{'user': user},
{'$pull': {
'books_url': {
"book_url": unquote(book_url)
}
}})
LOGGER.info('删除书架成功')
return json({'status': 1})
except Exception as e:
LOGGER.exception(e)
return json({'status': 0})
else:
return json({'status': -1})
async def fetch(client, url, novels_name):
with async_timeout.timeout(20):
try:
headers = {
'User-Agent': await get_random_user_agent(),
'Referer': "http://www.so.com/haosou.html?src=home"
}
params = {
'ie': 'utf-8',
'src': 'noscript_home',
'shb': 1,
'q': novels_name,
}
async with client.get(url, params=params,
headers=headers) as response:
assert response.status == 200
LOGGER.info('Task url: {}'.format(response.url))
try:
text = await response.text()
except:
text = await response.read()
return text
except Exception as e:
LOGGER.exception(e)
return None
async def fetch(client, url):
with async_timeout.timeout(10):
try:
headers = {'user-agent': get_random_user_agent()}
async with client.get(url, headers=headers) as response:
assert response.status == 200
LOGGER.info('Task url: {}'.format(response.url))
try:
text = await response.text()
except:
text = await response.read()
return text
except Exception as e:
LOGGER.exception(e)
return None
async def owllook_add_book(request):
"""
:param request:
:return:
: -1 用户session失效 需要重新登录
: 0 添加书架失败
: 1 添加书架成功
"""
user = request['session'].get('user', None)
data = parse_qs(str(request.body, encoding='utf-8'))
novels_name = data.get('novels_name', '')
chapter_url = data.get('chapter_url', '')
last_read_url = data.get('last_read_url', '')
if user and novels_name and chapter_url:
url = "/chapter?url={chapter_url}&novels_name={novels_name}".format(
chapter_url=chapter_url[0], novels_name=novels_name[0])
time = get_time()
try:
motor_db = motor_base.db
res = await motor_db.user_message.update_one(
{'user': user}, {'$set': {
'last_update_time': time
}},
upsert=True)
if res:
await motor_db.user_message.update_one(
{
'user': user,
'books_url.book_url': {
'$ne': url
}
}, {
'$push': {
'books_url': {
'book_url': url,
'add_time': time,
'last_read_url': unquote(last_read_url[0])
}
}
})
LOGGER.info('书架添加成功')
return json({'status': 1})
except Exception as e:
LOGGER.exception(e)
return json({'status': 0})
else:
return json({'status': -1})
def init_cache(sanic, loop):
LOGGER.info("Starting aiocache")
aiocache.settings.set_defaults(
class_="aiocache.RedisCache",
endpoint=REDIS_DICT.get('REDIS_ENDPOINT', None),
port=REDIS_DICT.get('REDIS_PORT', None),
db=REDIS_DICT.get('CACHE_DB', None),
password=REDIS_DICT.get('PASSWORD', None),
loop=loop,
)
LOGGER.info("Starting redis pool")
redis = RedisSession()
# redis instance for app
app.get_redis_pool = redis.get_redis_pool
# pass the getter method for the connection pool into the session
app.session_interface = RedisSessionInterface(app.get_redis_pool, cookie_name="owl_sid", expiry=30 * 24 * 60 * 60)
async def owllook_add_bookmark(request):
"""
:param request:
:return:
: -1 用户session失效 需要重新登录
: 0 添加书签失败
: 1 添加书签成功
"""
user = request['session'].get('user', None)
data = parse_qs(str(request.body, encoding='utf-8'))
bookmarkurl = data.get('bookmarkurl', '')
if user and bookmarkurl:
url = unquote(bookmarkurl[0])
time = get_time()
try:
motor_db = motor_base.db
res = await motor_db.user_message.update_one(
{'user': user}, {'$set': {
'last_update_time': time
}},
upsert=True)
if res:
await motor_db.user_message.update_one(
{
'user': user,
'bookmarks.bookmark': {
'$ne': url
}
}, {
'$push': {
'bookmarks': {
'bookmark': url,
'add_time': time
}
}
})
LOGGER.info('书签添加成功')
return json({'status': 1})
except Exception as e:
LOGGER.exception(e)
return json({'status': 0})
else:
return json({'status': -1})
async def add_session_to_request(request):
# before each request initialize a session
# using the client's request
host = request.headers.get('host', None)
user_agent = request.headers.get('user-agent', None)
if user_agent:
user_ip = request.headers.get('X-Forwarded-For')
LOGGER.info('user ip is: {}'.format(user_ip))
if user_ip in CONFIG.FORBIDDEN:
return html("<h3>网站正在维护...</h3>")
if CONFIG.VAL_HOST == 'true':
if not host or host not in CONFIG.HOST:
return redirect('http://www.owllook.net')
if CONFIG.WEBSITE['IS_RUNNING']:
await app.session_interface.open(request)
else:
return html("<h3>网站正在维护...</h3>")
else:
return html("<h3>网站正在维护...</h3>")
async def change_pass(request):
"""
修改用户密码
:param request:
:return:
: -1 用户session失效 需要重新登录
: 0 修改密码失败
: 1 添加密码成功
: -2 原始密码错误
"""
user = request['session'].get('user', None)
data = parse_qs(str(request.body, encoding='utf-8'))
if user:
try:
new_pass = data.get('new_pass', None)[0]
old_pass = data.get('old_pass', None)[0]
motor_db = motor_base.get_db()
user_data = await motor_db.user.find_one({'user': user})
if user_data:
pass_first = hashlib.md5(
(CONFIG.WEBSITE["TOKEN"] +
old_pass).encode("utf-8")).hexdigest()
pass_second = hashlib.md5(
(CONFIG.WEBSITE["TOKEN"] +
new_pass).encode("utf-8")).hexdigest()
new_password = hashlib.md5(
pass_second.encode("utf-8")).hexdigest()
password = hashlib.md5(pass_first.encode("utf-8")).hexdigest()
if password == user_data.get('password'):
await motor_db.user.update_one(
{'user': user}, {'$set': {
'password': new_password
}})
LOGGER.info('修改密码成功')
return json({'status': 1})
else:
return json({'status': -2})
except Exception as e:
LOGGER.exception(e)
return json({'status': 0})
else:
return json({'status': -1})
async def fetch(client, url, novels_name):
with async_timeout.timeout(20):
try:
headers = {
'user-agent': get_random_user_agent(),
'referer': "https://www.bing.com/"
}
params = {'q': novels_name, 'ensearch': 0}
async with client.get(url, params=params,
headers=headers) as response:
assert response.status == 200
LOGGER.info('Task url: {}'.format(response.url))
try:
text = await response.text()
except:
text = await response.read()
return text
except Exception as e:
LOGGER.exception(e)
return None
async def get_real_url(client, url):
with async_timeout.timeout(10):
try:
headers = {'user-agent': get_random_user_agent()}
async with client.head(url, headers=headers, allow_redirects=True) as response:
assert response.status == 200
LOGGER.info('Parse url: {}'.format(response.url))
# text = ""
# try:
# text = await response.text()
# except:
# text = await response.read()
# if text:
# print(text)
# text = re.findall(r'replace\(\"(.*?)\"\)', str(text))
# text = text[0] if text[0] else ""
url = response.url if response.url else None
return url
except Exception as e:
LOGGER.exception(e)
return None
async def fetch_url(self, client, url, params, headers):
"""
公共抓取函数
:param client:
:param url:
:param params:
:return:
"""
with async_timeout.timeout(15):
try:
async with client.get(url, params=params,
headers=headers) as response:
assert response.status == 200
LOGGER.info('Task url: {}'.format(response.url))
try:
text = await response.text()
except:
text = await response.read()
return text
except Exception as e:
LOGGER.exception(e)
return None
async def target_fetch(url, headers, timeout=15):
"""
:param url: target url
:return: text
"""
with async_timeout.timeout(timeout):
try:
async with aiohttp.ClientSession() as client:
async with client.get(url, headers=headers) as response:
assert response.status == 200
LOGGER.info('Task url: {}'.format(response.url))
try:
text = await response.text()
except:
try:
text = await response.read()
except aiohttp.ServerDisconnectedError as e:
LOGGER.exception(e)
text = None
return text
except Exception as e:
LOGGER.exception(str(e))
return None
async def target_fetch(client, url):
"""
:param client: aiohttp client
:param url: target url
:return: text
"""
with async_timeout.timeout(30):
try:
headers = {'user-agent': get_random_user_agent()}
async with client.get(url, headers=headers) as response:
assert response.status == 200
LOGGER.info('Task url: {}'.format(response.url))
try:
text = await response.text()
except:
try:
text = await response.read()
except aiohttp.ServerDisconnectedError as e:
LOGGER.exception(e)
text = None
return text
except Exception as e:
LOGGER.exception(e)
return None
async def index(request):
motor_db = MotorBase().db
ranking_cursor = motor_db.novels_ranking.find({})
async for document in ranking_cursor:
LOGGER.info(document)
return response.text('Ranking test')
}, {
'class': "aiocache.plugins.TimingPlugin"
}]
}
})
# aiocache.settings.set_defaults(
# class_="aiocache.RedisCache",
# endpoint=REDIS_DICT.get('REDIS_ENDPOINT', 'localhost'),
# port=REDIS_DICT.get('REDIS_PORT', 6379),
# db=REDIS_DICT.get('CACHE_DB', 0),
# password=REDIS_DICT.get('REDIS_PASSWORD', None),
# loop=loop,
# )
LOGGER.info("Starting redis pool")
redis_session = RedisSession()
# redis instance for app
app.get_redis_pool = redis_session.get_redis_pool
# pass the getter method for the connection pool into the session
app.session_interface = RedisSessionInterface(app.get_redis_pool,
cookie_name="owl_sid",
expiry=30 * 24 * 60 * 60)
@app.middleware('request')
async def add_session_to_request(request):
# before each request initialize a session
# using the client's request
host = request.headers.get('host', None)
user_agent = request.headers.get('user-agent', None)
async def author_notification(request):
"""
作者新书通知
:param request:
:return:
: -1 用户session失效 需要重新登录
: 2 无该作者信息
: 3 作者已经添加
: 4 超过添加的上限
: 0 操作失败
: 1 操作成功
"""
user = request['session'].get('user', None)
user_data = parse_qs(str(request.body, encoding='utf-8'))
if user:
try:
motor_db = motor_base.get_db()
all_authors = await motor_db.user_message.find_one(
{'user': user}, {
'author_latest': 1,
'_id': 0
})
count = len(all_authors.get('author_latest', []))
if count == CONFIG.WEBSITE.get("AUTHOR_LATEST_COUNT", 5):
return json({'status': 4})
author_name = user_data.get('author_name', None)[0]
data = []
author_cursor = motor_db.all_books.find({'author': author_name}, {
'name': 1,
'url': 1,
'_id': 0
})
async for document in author_cursor:
data.append(document)
if data:
time = get_time()
res = await motor_db.user_message.update_one(
{'user': user}, {'$set': {
'last_update_time': time
}},
upsert=True)
is_exist = await motor_db.user_message.find_one({
'user':
user,
'author_latest.author_name':
author_name
})
if is_exist:
return json({'status': 3})
if res:
await motor_db.user_message.update_one(
{
'user': user,
'author_latest.author_name': {
'$ne': author_name
}
}, {
'$push': {
'author_latest': {
'author_name': author_name,
'add_time': time
}
}
})
is_author_exist = await motor_db.author_message.find_one(
{'name': author_name})
if not is_author_exist:
author_data = {
"author_name": author_name,
"nums": len(data),
"updated_time": get_time(),
}
await motor_db.author_message.save(author_data)
LOGGER.info('作者添加成功')
return json({'status': 1})
else:
return json({'status': 2})
else:
return json({'status': 2})
except Exception as e:
LOGGER.exception(e)
return json({'status': 0})
else:
return json({'status': -1})
