def __init__(
self,
start_url,
message_handler=None,
max_trys=10,
reader=None,
rss_proxy_url=None,
rss_proxy_token=None,
dns_service=DNS_SERVICE,
):
start_url = coerce_url(start_url)
try:
start_url = validate_url(start_url)
except Invalid:
raise ValueError(f"invalid start_url {start_url!r}")
self._set_start_url(start_url)
self.message_handler = message_handler
self.max_trys = max_trys
if reader is None:
reader = FeedReader(
dns_service=dns_service,
rss_proxy_url=rss_proxy_url,
rss_proxy_token=rss_proxy_token,
)
self._close_reader = True
else:
self._close_reader = False
self.reader = reader
self._links = {start_url: ScoredLink(start_url, 1.0)}
self._visited = set()
self._guessed = False
def __init__(
self,
start_url,
message_handler=None,
max_trys=10,
reader=None,
allow_private_address=False,
rss_proxy_url=None,
rss_proxy_token=None,
):
start_url = coerce_url(start_url)
self._set_start_url(start_url)
self.message_handler = message_handler
self.max_trys = max_trys
if reader is None:
reader = FeedReader(
allow_private_address=allow_private_address,
rss_proxy_url=rss_proxy_url,
rss_proxy_token=rss_proxy_token,
)
self._close_reader = True
else:
self._close_reader = False
self.reader = reader
self._links = {start_url: ScoredLink(start_url, 1.0)}
self._visited = set()
self._guessed = False
def _import_one_line_text(text):
text = text.strip()
parts = text.split(maxsplit=2)
if len(parts) != 1:
return None
url = coerce_url(parts[0])
try:
_validate_url(url)
except Invalid:
return None
return url
def __init__(self,
start_url,
message_handler=None,
max_trys=10,
reader=None,
validate=True):
start_url = coerce_url(start_url)
self._set_start_url(start_url)
self.message_handler = message_handler
self.max_trys = max_trys
if reader is None:
reader = FeedReader()
self._close_reader = True
else:
self._close_reader = False
self.reader = reader
self.validate = validate
self._links = {start_url: ScoredLink(start_url, 1.0)}
self._visited = set()
