def __init__(
self,
session=None,
user_agent=DEFAULT_USER_AGENT,
request_timeout=30,
max_content_length=10 * 1024 * 1024,
allow_non_webpage=False,
rss_proxy_url=None,
rss_proxy_token=None,
dns_service: DNSService = DNS_SERVICE,
):
if session is None:
session = requests.session()
if dns_service:
session.mount('http://', dns_service.requests_http_adapter())
session.mount('https://', dns_service.requests_http_adapter())
self._close_session = True
else:
self._close_session = False
self.session = session
self.user_agent = user_agent
self.request_timeout = request_timeout
self.max_content_length = max_content_length
self.allow_non_webpage = allow_non_webpage
self.rss_proxy_url = rss_proxy_url
self.rss_proxy_token = rss_proxy_token
self.dns_service = dns_service
self._cacert = cacert.where()
def test_read_incomplete_response_gzip(reader_class: Type[FeedReader],
httpserver: HTTPServer):
dns_service = DNSService.create(allow_private_address=True)
with SocketHttpServer.incomplete_gzip() as server:
with reader_class(dns_service=dns_service) as reader:
response = reader.read(server.url)
assert response.status == FeedResponseStatus.CHUNKED_ENCODING_ERROR
def test_read_private_addres(reader_class: Type[FeedReader],
httpserver: HTTPServer):
httpserver.expect_request("/private-address").respond_with_json(0)
url = httpserver.url_for("/private-address")
dns_service = DNSService.create(allow_private_address=False)
with reader_class(dns_service=dns_service) as reader:
response = reader.read(url)
assert response.status == FeedResponseStatus.PRIVATE_ADDRESS_ERROR
assert not response.content
def test_resolve_private_address(resolve, url, expect):
dns_service = DNSService.create(allow_private_address=False)
host = urlparse(url).hostname
try:
resolve(dns_service, host)
except PrivateAddressError:
is_private = True
else:
is_private = False
assert is_private == expect
def test_read_non_webpage(reader_class: Type[FeedReader],
httpserver: HTTPServer, mime_type: str):
options = dict(dns_service=DNSService.create(allow_private_address=True))
local_resp = WerkzeugResponse(b'xxxxxxxx', mimetype=mime_type)
httpserver.expect_request("/non-webpage").respond_with_response(local_resp)
url = httpserver.url_for("/non-webpage")
with reader_class(**options) as reader:
response = reader.read(url)
assert response.status == FeedResponseStatus.CONTENT_TYPE_NOT_SUPPORT_ERROR
assert not response.content
def test_read_status(reader_class: Type[FeedReader], httpserver: HTTPServer,
status: int):
dns_service = DNSService.create(allow_private_address=True)
options = dict(allow_non_webpage=True, dns_service=dns_service)
local_resp = WerkzeugResponse(str(status), status=status)
httpserver.expect_request("/status").respond_with_response(local_resp)
url = httpserver.url_for("/status")
with reader_class(**options) as reader:
response = reader.read(url)
assert response.status == status
assert response.content == str(status).encode()
def _setup_rss_proxy(httpserver: HTTPServer):
httpserver.expect_request("/rss-proxy", method='POST')\
.respond_with_handler(rss_proxy_handler)
httpserver.expect_request("/not-proxy").respond_with_data('ERROR', status=500)
httpserver.expect_request("/direct/200").respond_with_data('DIRECT', status=200)
proxy_url = httpserver.url_for('/rss-proxy')
options = dict(
dns_service=DNSService.create(allow_private_address=True),
rss_proxy_url=proxy_url,
rss_proxy_token=_RSS_PROXY_TOKEN,
)
return options
def _create_finder(start_url, **kwargs):
messages = []
def message_handler(msg):
messages.append(msg)
finder = FeedFinder(
start_url,
message_handler=message_handler,
dns_service=DNSService.create(allow_private_address=True),
**kwargs,
)
return finder, messages
def test_read_testdata(reader_class: Type[FeedReader], httpserver: HTTPServer,
filepath: str):
filepath = _data_dir / filepath
content = filepath.read_bytes()
urls = []
for i, x in enumerate(_collect_header_cases()):
local_resp = WerkzeugResponse(content, content_type=x)
httpserver.expect_request(f"/testdata/{i}").respond_with_response(
local_resp)
urls.append(httpserver.url_for(f"/testdata/{i}"))
options = dict(dns_service=DNSService.create(allow_private_address=True))
with reader_class(**options) as reader:
for url in urls:
response = reader.read(url)
assert response.ok
assert response.content == content
assert response.encoding
assert response.feed_type
def _sync_resolve_host(dns_service: DNSService, host: str):
return dns_service.resolve_urllib3(host)
def _requests_session():
dns_service = DNSService.create(allow_private_address=False)
session = requests.session()
session.mount('http://', dns_service.requests_http_adapter())
session.mount('https://', dns_service.requests_http_adapter())
return session
