Python compile_url_blacklist示例

示例#1

from rssant_config import CONFIG
from rssant_common.helper import get_referer_of_url, aiohttp_client_session
from rssant_feedlib.reader import DEFAULT_USER_AGENT, PrivateAddressError
from rssant_feedlib.async_reader import AsyncFeedReader
from rssant_feedlib.blacklist import compile_url_blacklist

LOG = logging.getLogger(__name__)

# 这些图片Referer错误会返回200，无法有效处理失败情况
# 需要直接设置为正确的Referer
REFERER_FORCE_URL_LIST = """
qpic.cn
qlogo.cn
qq.com
"""
is_referer_force_url = compile_url_blacklist(REFERER_FORCE_URL_LIST)

PROXY_REQUEST_HEADERS = [
    'Accept',
    'Accept-Encoding',
    'ETag',
    'If-Modified-Since',
    'Cache-Control',
]

PROXY_RESPONSE_HEADERS = [
    'Transfer-Encoding',
    'Cache-Control',
    'ETag',
    'Last-Modified',
    'Expires',

示例#2

from rssant_common.validator import compiler
from rssant_common.dns_service import DNS_SERVICE
from rssant_config import CONFIG

LOG = logging.getLogger(__name__)

_MAX_STORY_HTML_LENGTH = 5 * 1000 * 1024
_MAX_STORY_CONTENT_LENGTH = 1000 * 1024
_MAX_STORY_SUMMARY_LENGTH = 300

REFERER_DENY_LIST = """
qpic.cn
qlogo.cn
qq.com
"""
is_referer_deny_url = compile_url_blacklist(REFERER_DENY_LIST)

StorySchema = T.dict(
    unique_id=T.str,
    title=T.str,
    content_hash_base64=T.str,
    author=T.str.optional,
    link=T.url.optional,
    image_url=T.url.optional,
    iframe_url=T.url.optional,
    audio_url=T.url.optional,
    has_mathjax=T.bool.optional,
    dt_published=T.datetime.optional,
    dt_updated=T.datetime.optional,
    summary=T.str.optional,
    content=T.str.optional,