from rssant_config import CONFIG
from rssant_common.helper import get_referer_of_url, aiohttp_client_session
from rssant_feedlib.reader import DEFAULT_USER_AGENT, PrivateAddressError
from rssant_feedlib.async_reader import AsyncFeedReader
from rssant_feedlib.blacklist import compile_url_blacklist
LOG = logging.getLogger(__name__)
# 这些图片Referer错误会返回200，无法有效处理失败情况
# 需要直接设置为正确的Referer
REFERER_FORCE_URL_LIST = """
qpic.cn
qlogo.cn
qq.com
"""
is_referer_force_url = compile_url_blacklist(REFERER_FORCE_URL_LIST)
PROXY_REQUEST_HEADERS = [
'Accept',
'Accept-Encoding',
'ETag',
'If-Modified-Since',
'Cache-Control',
]
PROXY_RESPONSE_HEADERS = [
'Transfer-Encoding',
'Cache-Control',
'ETag',
'Last-Modified',
'Expires',
from rssant_common.validator import compiler
from rssant_common.dns_service import DNS_SERVICE
from rssant_config import CONFIG
LOG = logging.getLogger(__name__)
_MAX_STORY_HTML_LENGTH = 5 * 1000 * 1024
_MAX_STORY_CONTENT_LENGTH = 1000 * 1024
_MAX_STORY_SUMMARY_LENGTH = 300
REFERER_DENY_LIST = """
qpic.cn
qlogo.cn
qq.com
"""
is_referer_deny_url = compile_url_blacklist(REFERER_DENY_LIST)
StorySchema = T.dict(
unique_id=T.str,
title=T.str,
content_hash_base64=T.str,
author=T.str.optional,
link=T.url.optional,
image_url=T.url.optional,
iframe_url=T.url.optional,
audio_url=T.url.optional,
has_mathjax=T.bool.optional,
dt_published=T.datetime.optional,
dt_updated=T.datetime.optional,
summary=T.str.optional,
content=T.str.optional,