def __init__( self, url: URLTypes, allow_relative: bool = False, params: QueryParamTypes = None, ) -> None: if isinstance(url, str): self._uri_reference = rfc3986.api.iri_reference(url).encode() else: self._uri_reference = url._uri_reference # Normalize scheme and domain name. if self.is_absolute_url: self._uri_reference = self._uri_reference.normalize() # Add any query parameters. if params: query_string = str(QueryParams(params)) self._uri_reference = self._uri_reference.copy_with(query=query_string) # Enforce absolute URLs by default. if not allow_relative: if not self.scheme: raise InvalidURL("No scheme included in URL.") if not self.host: raise InvalidURL("No host included in URL.") # If the URL is HTTP but the host is on the HSTS preload list switch to HTTPS. if ( self.scheme == "http" and self.host and hstspreload.in_hsts_preload(self.host) ): self._uri_reference = self._uri_reference.copy_with(scheme="https")
def merge_url(self, url: URLTypes) -> URL: """ Merge a URL argument together with any 'base_url' on the client, to create the URL used for the outgoing request. """ url = self.base_url.join(relative_url=url) if url.scheme == "http" and hstspreload.in_hsts_preload(url.host): url = url.copy_with(scheme="https") return url
def _merge_url(self, url: URLTypes) -> URL: """ Merge a URL argument together with any 'base_url' on the client, to create the URL used for the outgoing request. """ url = self.base_url.join(relative_url=url) if ( url.scheme == "http" and hstspreload.in_hsts_preload(url.host) and len(url.host.split(".")) > 1 ): port = None if url.port == 80 else url.port url = url.copy_with(scheme="https", port=port) return url
def check_http_to_https(self, wikicode, extlink, url): if url.scheme != "http": return # check HSTS preload list first # (Chromium's static list of sites supporting HTTP Strict Transport Security) if in_hsts_preload(url.netloc.lower()): new_url = str(extlink.url).replace("http://", "https://", 1) # check HTTPS Everywhere rules next elif self.https_everywhere_rules.matchingRulesets(url.netloc.lower()): match = self.https_everywhere_rules.transformUrl(url) new_url = match.url # check the Smarter Encryption list elif url.netloc.lower() in self.selist: new_url = str(extlink.url).replace("http://", "https://", 1) else: return # there is no reason to update broken links if self.check_url(new_url): extlink.url = new_url else: logger.warning("broken URL not updated to https: {}".format(url))
def merge_url(self, url: URLTypes) -> URL: url = self.base_url.join(relative_url=url) if url.scheme == "http" and hstspreload.in_hsts_preload(url.host): url = url.copy_with(scheme="https") return url
def test_in_hsts_preload(host, expected): assert hstspreload.in_hsts_preload(host) is expected
def test_data_types(host, expected): assert hstspreload.in_hsts_preload(host) is expected
def is_hsts(self): return hstspreload.in_hsts_preload(self.host)