def __init__(
self,
url: URLTypes,
allow_relative: bool = False,
params: QueryParamTypes = None,
) -> None:
if isinstance(url, str):
self._uri_reference = rfc3986.api.iri_reference(url).encode()
else:
self._uri_reference = url._uri_reference
# Normalize scheme and domain name.
if self.is_absolute_url:
self._uri_reference = self._uri_reference.normalize()
# Add any query parameters.
if params:
query_string = str(QueryParams(params))
self._uri_reference = self._uri_reference.copy_with(query=query_string)
# Enforce absolute URLs by default.
if not allow_relative:
if not self.scheme:
raise InvalidURL("No scheme included in URL.")
if not self.host:
raise InvalidURL("No host included in URL.")
# If the URL is HTTP but the host is on the HSTS preload list switch to HTTPS.
if (
self.scheme == "http"
and self.host
and hstspreload.in_hsts_preload(self.host)
):
self._uri_reference = self._uri_reference.copy_with(scheme="https")
def merge_url(self, url: URLTypes) -> URL:
"""
Merge a URL argument together with any 'base_url' on the client,
to create the URL used for the outgoing request.
"""
url = self.base_url.join(relative_url=url)
if url.scheme == "http" and hstspreload.in_hsts_preload(url.host):
url = url.copy_with(scheme="https")
return url
def _merge_url(self, url: URLTypes) -> URL:
"""
Merge a URL argument together with any 'base_url' on the client,
to create the URL used for the outgoing request.
"""
url = self.base_url.join(relative_url=url)
if (
url.scheme == "http"
and hstspreload.in_hsts_preload(url.host)
and len(url.host.split(".")) > 1
):
port = None if url.port == 80 else url.port
url = url.copy_with(scheme="https", port=port)
return url
def check_http_to_https(self, wikicode, extlink, url):
if url.scheme != "http":
return
# check HSTS preload list first
# (Chromium's static list of sites supporting HTTP Strict Transport Security)
if in_hsts_preload(url.netloc.lower()):
new_url = str(extlink.url).replace("http://", "https://", 1)
# check HTTPS Everywhere rules next
elif self.https_everywhere_rules.matchingRulesets(url.netloc.lower()):
match = self.https_everywhere_rules.transformUrl(url)
new_url = match.url
# check the Smarter Encryption list
elif url.netloc.lower() in self.selist:
new_url = str(extlink.url).replace("http://", "https://", 1)
else:
return
# there is no reason to update broken links
if self.check_url(new_url):
extlink.url = new_url
else:
logger.warning("broken URL not updated to https: {}".format(url))
def merge_url(self, url: URLTypes) -> URL:
url = self.base_url.join(relative_url=url)
if url.scheme == "http" and hstspreload.in_hsts_preload(url.host):
url = url.copy_with(scheme="https")
return url
def test_in_hsts_preload(host, expected):
assert hstspreload.in_hsts_preload(host) is expected
def test_data_types(host, expected):
assert hstspreload.in_hsts_preload(host) is expected
def is_hsts(self):
return hstspreload.in_hsts_preload(self.host)
