async def fetch(client, url, name, is_web): with async_timeout.timeout(15): try: headers = {'user-agent': get_random_user_agent()} if is_web: params = { 'wd': name, 'ie': 'utf-8', 'rn': CONFIG.BAIDU_RN, 'vf_bl': 1 } else: params = {'word': name} async with client.get(url, params=params, headers=headers) as response: assert response.status == 200 LOGGER.info('Task url: {}'.format(response.url)) try: text = await response.text() except: text = await response.read() return text except Exception as e: LOGGER.exception(e) return None
async def change_email(request): """ 修改用户邮箱 :param request: :return: : -1 用户session失效 需要重新登录 : 0 修改邮箱失败 : 1 添加邮箱成功 """ user = request['session'].get('user', None) data = parse_qs(str(request.body, encoding='utf-8')) if user: try: email = data.get('email', None)[0] motor_db = motor_base.get_db() await motor_db.user.update_one({'user': user}, {'$set': { 'email': email }}) LOGGER.info('修改邮箱成功') return json({'status': 1}) except Exception as e: LOGGER.exception(e) return json({'status': 0}) else: return json({'status': -1})
async def owllook_delete_bookmark(request): """ 删除书签 :param request: :return: : -1 用户session失效 需要重新登录 : 0 删除书签失败 : 1 删除书签成功 """ user = request['session'].get('user', None) data = parse_qs(str(request.body, encoding='utf-8')) bookmarkurl = data.get('bookmarkurl', '') if user and bookmarkurl: bookmark = unquote(bookmarkurl[0]) try: motor_db = motor_base.get_db() await motor_db.user_message.update_one( {'user': user}, {'$pull': { 'bookmarks': { "bookmark": bookmark } }}) LOGGER.info('删除书签成功') return json({'status': 1}) except Exception as e: LOGGER.exception(e) return json({'status': 0}) else: return json({'status': -1})
async def owllook_delete_book(request): """ 删除书架 :param request: :return: : -1 用户session失效 需要重新登录 : 0 删除书架失败 : 1 删除书架成功 """ user = request['session'].get('user', None) data = parse_qs(str(request.body, encoding='utf-8')) if user: if data.get('book_url', None): book_url = data.get('book_url', None)[0] else: novels_name = data.get('novels_name', '') chapter_url = data.get('chapter_url', '') book_url = "/chapter?url={chapter_url}&novels_name={novels_name}".format( chapter_url=chapter_url[0], novels_name=novels_name[0]) try: motor_db = motor_base.get_db() await motor_db.user_message.update_one( {'user': user}, {'$pull': { 'books_url': { "book_url": unquote(book_url) } }}) LOGGER.info('删除书架成功') return json({'status': 1}) except Exception as e: LOGGER.exception(e) return json({'status': 0}) else: return json({'status': -1})
async def fetch(client, url, novels_name): with async_timeout.timeout(20): try: headers = { 'User-Agent': await get_random_user_agent(), 'Referer': "http://www.so.com/haosou.html?src=home" } params = { 'ie': 'utf-8', 'src': 'noscript_home', 'shb': 1, 'q': novels_name, } async with client.get(url, params=params, headers=headers) as response: assert response.status == 200 LOGGER.info('Task url: {}'.format(response.url)) try: text = await response.text() except: text = await response.read() return text except Exception as e: LOGGER.exception(e) return None
async def fetch(client, url): with async_timeout.timeout(10): try: headers = {'user-agent': get_random_user_agent()} async with client.get(url, headers=headers) as response: assert response.status == 200 LOGGER.info('Task url: {}'.format(response.url)) try: text = await response.text() except: text = await response.read() return text except Exception as e: LOGGER.exception(e) return None
async def owllook_add_book(request): """ :param request: :return: : -1 用户session失效 需要重新登录 : 0 添加书架失败 : 1 添加书架成功 """ user = request['session'].get('user', None) data = parse_qs(str(request.body, encoding='utf-8')) novels_name = data.get('novels_name', '') chapter_url = data.get('chapter_url', '') last_read_url = data.get('last_read_url', '') if user and novels_name and chapter_url: url = "/chapter?url={chapter_url}&novels_name={novels_name}".format( chapter_url=chapter_url[0], novels_name=novels_name[0]) time = get_time() try: motor_db = motor_base.db res = await motor_db.user_message.update_one( {'user': user}, {'$set': { 'last_update_time': time }}, upsert=True) if res: await motor_db.user_message.update_one( { 'user': user, 'books_url.book_url': { '$ne': url } }, { '$push': { 'books_url': { 'book_url': url, 'add_time': time, 'last_read_url': unquote(last_read_url[0]) } } }) LOGGER.info('书架添加成功') return json({'status': 1}) except Exception as e: LOGGER.exception(e) return json({'status': 0}) else: return json({'status': -1})
def init_cache(sanic, loop): LOGGER.info("Starting aiocache") aiocache.settings.set_defaults( class_="aiocache.RedisCache", endpoint=REDIS_DICT.get('REDIS_ENDPOINT', None), port=REDIS_DICT.get('REDIS_PORT', None), db=REDIS_DICT.get('CACHE_DB', None), password=REDIS_DICT.get('PASSWORD', None), loop=loop, ) LOGGER.info("Starting redis pool") redis = RedisSession() # redis instance for app app.get_redis_pool = redis.get_redis_pool # pass the getter method for the connection pool into the session app.session_interface = RedisSessionInterface(app.get_redis_pool, cookie_name="owl_sid", expiry=30 * 24 * 60 * 60)
async def owllook_add_bookmark(request): """ :param request: :return: : -1 用户session失效 需要重新登录 : 0 添加书签失败 : 1 添加书签成功 """ user = request['session'].get('user', None) data = parse_qs(str(request.body, encoding='utf-8')) bookmarkurl = data.get('bookmarkurl', '') if user and bookmarkurl: url = unquote(bookmarkurl[0]) time = get_time() try: motor_db = motor_base.db res = await motor_db.user_message.update_one( {'user': user}, {'$set': { 'last_update_time': time }}, upsert=True) if res: await motor_db.user_message.update_one( { 'user': user, 'bookmarks.bookmark': { '$ne': url } }, { '$push': { 'bookmarks': { 'bookmark': url, 'add_time': time } } }) LOGGER.info('书签添加成功') return json({'status': 1}) except Exception as e: LOGGER.exception(e) return json({'status': 0}) else: return json({'status': -1})
async def add_session_to_request(request): # before each request initialize a session # using the client's request host = request.headers.get('host', None) user_agent = request.headers.get('user-agent', None) if user_agent: user_ip = request.headers.get('X-Forwarded-For') LOGGER.info('user ip is: {}'.format(user_ip)) if user_ip in CONFIG.FORBIDDEN: return html("<h3>网站正在维护...</h3>") if CONFIG.VAL_HOST == 'true': if not host or host not in CONFIG.HOST: return redirect('http://www.owllook.net') if CONFIG.WEBSITE['IS_RUNNING']: await app.session_interface.open(request) else: return html("<h3>网站正在维护...</h3>") else: return html("<h3>网站正在维护...</h3>")
async def change_pass(request): """ 修改用户密码 :param request: :return: : -1 用户session失效 需要重新登录 : 0 修改密码失败 : 1 添加密码成功 : -2 原始密码错误 """ user = request['session'].get('user', None) data = parse_qs(str(request.body, encoding='utf-8')) if user: try: new_pass = data.get('new_pass', None)[0] old_pass = data.get('old_pass', None)[0] motor_db = motor_base.get_db() user_data = await motor_db.user.find_one({'user': user}) if user_data: pass_first = hashlib.md5( (CONFIG.WEBSITE["TOKEN"] + old_pass).encode("utf-8")).hexdigest() pass_second = hashlib.md5( (CONFIG.WEBSITE["TOKEN"] + new_pass).encode("utf-8")).hexdigest() new_password = hashlib.md5( pass_second.encode("utf-8")).hexdigest() password = hashlib.md5(pass_first.encode("utf-8")).hexdigest() if password == user_data.get('password'): await motor_db.user.update_one( {'user': user}, {'$set': { 'password': new_password }}) LOGGER.info('修改密码成功') return json({'status': 1}) else: return json({'status': -2}) except Exception as e: LOGGER.exception(e) return json({'status': 0}) else: return json({'status': -1})
async def fetch(client, url, novels_name): with async_timeout.timeout(20): try: headers = { 'user-agent': get_random_user_agent(), 'referer': "https://www.bing.com/" } params = {'q': novels_name, 'ensearch': 0} async with client.get(url, params=params, headers=headers) as response: assert response.status == 200 LOGGER.info('Task url: {}'.format(response.url)) try: text = await response.text() except: text = await response.read() return text except Exception as e: LOGGER.exception(e) return None
async def get_real_url(client, url): with async_timeout.timeout(10): try: headers = {'user-agent': get_random_user_agent()} async with client.head(url, headers=headers, allow_redirects=True) as response: assert response.status == 200 LOGGER.info('Parse url: {}'.format(response.url)) # text = "" # try: # text = await response.text() # except: # text = await response.read() # if text: # print(text) # text = re.findall(r'replace\(\"(.*?)\"\)', str(text)) # text = text[0] if text[0] else "" url = response.url if response.url else None return url except Exception as e: LOGGER.exception(e) return None
async def fetch_url(self, client, url, params, headers): """ 公共抓取函数 :param client: :param url: :param params: :return: """ with async_timeout.timeout(15): try: async with client.get(url, params=params, headers=headers) as response: assert response.status == 200 LOGGER.info('Task url: {}'.format(response.url)) try: text = await response.text() except: text = await response.read() return text except Exception as e: LOGGER.exception(e) return None
async def target_fetch(url, headers, timeout=15): """ :param url: target url :return: text """ with async_timeout.timeout(timeout): try: async with aiohttp.ClientSession() as client: async with client.get(url, headers=headers) as response: assert response.status == 200 LOGGER.info('Task url: {}'.format(response.url)) try: text = await response.text() except: try: text = await response.read() except aiohttp.ServerDisconnectedError as e: LOGGER.exception(e) text = None return text except Exception as e: LOGGER.exception(str(e)) return None
async def target_fetch(client, url): """ :param client: aiohttp client :param url: target url :return: text """ with async_timeout.timeout(30): try: headers = {'user-agent': get_random_user_agent()} async with client.get(url, headers=headers) as response: assert response.status == 200 LOGGER.info('Task url: {}'.format(response.url)) try: text = await response.text() except: try: text = await response.read() except aiohttp.ServerDisconnectedError as e: LOGGER.exception(e) text = None return text except Exception as e: LOGGER.exception(e) return None
async def index(request): motor_db = MotorBase().db ranking_cursor = motor_db.novels_ranking.find({}) async for document in ranking_cursor: LOGGER.info(document) return response.text('Ranking test')
}, { 'class': "aiocache.plugins.TimingPlugin" }] } }) # aiocache.settings.set_defaults( # class_="aiocache.RedisCache", # endpoint=REDIS_DICT.get('REDIS_ENDPOINT', 'localhost'), # port=REDIS_DICT.get('REDIS_PORT', 6379), # db=REDIS_DICT.get('CACHE_DB', 0), # password=REDIS_DICT.get('REDIS_PASSWORD', None), # loop=loop, # ) LOGGER.info("Starting redis pool") redis_session = RedisSession() # redis instance for app app.get_redis_pool = redis_session.get_redis_pool # pass the getter method for the connection pool into the session app.session_interface = RedisSessionInterface(app.get_redis_pool, cookie_name="owl_sid", expiry=30 * 24 * 60 * 60) @app.middleware('request') async def add_session_to_request(request): # before each request initialize a session # using the client's request host = request.headers.get('host', None) user_agent = request.headers.get('user-agent', None)
async def author_notification(request): """ 作者新书通知 :param request: :return: : -1 用户session失效 需要重新登录 : 2 无该作者信息 : 3 作者已经添加 : 4 超过添加的上限 : 0 操作失败 : 1 操作成功 """ user = request['session'].get('user', None) user_data = parse_qs(str(request.body, encoding='utf-8')) if user: try: motor_db = motor_base.get_db() all_authors = await motor_db.user_message.find_one( {'user': user}, { 'author_latest': 1, '_id': 0 }) count = len(all_authors.get('author_latest', [])) if count == CONFIG.WEBSITE.get("AUTHOR_LATEST_COUNT", 5): return json({'status': 4}) author_name = user_data.get('author_name', None)[0] data = [] author_cursor = motor_db.all_books.find({'author': author_name}, { 'name': 1, 'url': 1, '_id': 0 }) async for document in author_cursor: data.append(document) if data: time = get_time() res = await motor_db.user_message.update_one( {'user': user}, {'$set': { 'last_update_time': time }}, upsert=True) is_exist = await motor_db.user_message.find_one({ 'user': user, 'author_latest.author_name': author_name }) if is_exist: return json({'status': 3}) if res: await motor_db.user_message.update_one( { 'user': user, 'author_latest.author_name': { '$ne': author_name } }, { '$push': { 'author_latest': { 'author_name': author_name, 'add_time': time } } }) is_author_exist = await motor_db.author_message.find_one( {'name': author_name}) if not is_author_exist: author_data = { "author_name": author_name, "nums": len(data), "updated_time": get_time(), } await motor_db.author_message.save(author_data) LOGGER.info('作者添加成功') return json({'status': 1}) else: return json({'status': 2}) else: return json({'status': 2}) except Exception as e: LOGGER.exception(e) return json({'status': 0}) else: return json({'status': -1})