def test_is_valid_host():
assert not check.is_valid_host(b"")
assert check.is_valid_host(b"one.two")
assert not check.is_valid_host(b"one" * 255)
assert check.is_valid_host(b"one.two.")
# Allow underscore
assert check.is_valid_host(b"one_two")
def parse_authority(authority: AnyStr, check: bool) -> Tuple[str, Optional[int]]:
"""Extract the host and port from host header/authority information
Raises:
ValueError, if check is True and the authority information is malformed.
"""
try:
if isinstance(authority, bytes):
authority_str = authority.decode("idna")
else:
authority_str = authority
m = _authority_re.match(authority_str)
if not m:
raise ValueError
host = m.group("host")
if host.startswith("[") and host.endswith("]"):
host = host[1:-1]
if not is_valid_host(host):
raise ValueError
if m.group("port"):
port = int(m.group("port"))
if not is_valid_port(port):
raise ValueError
return host, port
else:
return host, None
except ValueError:
if check:
raise
else:
return always_str(authority, "utf-8", "surrogateescape"), None
def from_file(cls, f):
ver, msg, rsv, atyp = struct.unpack("!BBBB", f.safe_read(4))
if rsv != 0x00:
raise SocksError(REP.GENERAL_SOCKS_SERVER_FAILURE,
"Socks Request: Invalid reserved byte: %s" % rsv)
if atyp == ATYP.IPV4_ADDRESS:
# We use tnoa here as ntop is not commonly available on Windows.
host = ipaddress.IPv4Address(f.safe_read(4)).compressed
use_ipv6 = False
elif atyp == ATYP.IPV6_ADDRESS:
host = ipaddress.IPv6Address(f.safe_read(16)).compressed
use_ipv6 = True
elif atyp == ATYP.DOMAINNAME:
length, = struct.unpack("!B", f.safe_read(1))
host = f.safe_read(length)
if not check.is_valid_host(host):
raise SocksError(REP.GENERAL_SOCKS_SERVER_FAILURE,
"Invalid hostname: %s" % host)
host = host.decode("idna")
use_ipv6 = False
else:
raise SocksError(REP.ADDRESS_TYPE_NOT_SUPPORTED,
"Socks Request: Unknown ATYP: %s" % atyp)
port, = struct.unpack("!H", f.safe_read(2))
addr = tcp.Address((host, port), use_ipv6=use_ipv6)
return cls(ver, msg, atyp, addr)
def from_file(cls, f):
ver, msg, rsv, atyp = struct.unpack("!BBBB", f.safe_read(4))
if rsv != 0x00:
raise SocksError(
REP.GENERAL_SOCKS_SERVER_FAILURE,
"Socks Request: Invalid reserved byte: %s" % rsv
)
if atyp == ATYP.IPV4_ADDRESS:
# We use tnoa here as ntop is not commonly available on Windows.
host = ipaddress.IPv4Address(f.safe_read(4)).compressed
elif atyp == ATYP.IPV6_ADDRESS:
host = ipaddress.IPv6Address(f.safe_read(16)).compressed
elif atyp == ATYP.DOMAINNAME:
length, = struct.unpack("!B", f.safe_read(1))
host = f.safe_read(length)
if not check.is_valid_host(host):
raise SocksError(REP.GENERAL_SOCKS_SERVER_FAILURE, "Invalid hostname: %s" % host)
host = host.decode("idna")
else:
raise SocksError(REP.ADDRESS_TYPE_NOT_SUPPORTED,
"Socks Request: Unknown ATYP: %s" % atyp)
port, = struct.unpack("!H", f.safe_read(2))
addr = (host, port)
return cls(ver, msg, atyp, addr)
def parse(server_spec: str) -> ServerSpec:
"""
Parses a server mode specification, e.g.:
- http://example.com/
- example.org
- example.com:443
Raises:
ValueError, if the server specification is invalid.
"""
m = server_spec_re.match(server_spec)
if not m:
raise ValueError(f"Invalid server specification: {server_spec}")
# defaulting to https/port 443 may annoy some folks, but it's secure-by-default.
scheme = m.group("scheme") or "https"
if scheme not in ("http", "https"):
raise ValueError(f"Invalid server scheme: {scheme}")
host = m.group("host")
# IPv6 brackets
if host.startswith("[") and host.endswith("]"):
host = host[1:-1]
if not check.is_valid_host(host.encode("idna")):
raise ValueError(f"Invalid hostname: {host}")
if m.group("port"):
port = int(m.group("port"))
else:
port = {"http": 80, "https": 443}[scheme]
if not check.is_valid_port(port):
raise ValueError(f"Invalid port: {port}")
return ServerSpec(scheme, (host, port))
def sni(self):
for extension in self._client_hello.extensions:
is_valid_sni_extension = (extension.type == 0x00
and len(extension.server_names) == 1
and extension.server_names[0].type == 0
and check.is_valid_host(
extension.server_names[0].name))
if is_valid_sni_extension:
return extension.server_names[0].name.decode("idna")
def parse(url):
"""
URL-parsing function that checks that
- port is an integer 0-65535
- host is a valid IDNA-encoded hostname with no null-bytes
- path is valid ASCII
Args:
A URL (as bytes or as unicode)
Returns:
A (scheme, host, port, path) tuple
Raises:
ValueError, if the URL is not properly formatted.
"""
# FIXME: We shouldn't rely on urllib here.
# Size of Ascii character after encoding is 1 byte which is same as its size
# But non-Ascii character's size after encoding will be more than its size
def ascii_check(l):
if len(l) == len(str(l).encode()):
return True
return False
if isinstance(url, bytes):
url = url.decode()
if not ascii_check(url):
url = urllib.parse.urlsplit(url)
url = list(url)
url[3] = urllib.parse.quote(url[3])
url = urllib.parse.urlunsplit(url)
parsed = urllib.parse.urlparse(url)
if not parsed.hostname:
raise ValueError("No hostname given")
else:
host = parsed.hostname.encode("idna")
if isinstance(parsed, urllib.parse.ParseResult):
parsed = parsed.encode("ascii")
port = parsed.port
if not port:
port = 443 if parsed.scheme == b"https" else 80
full_path = urllib.parse.urlunparse(
(b"", b"", parsed.path, parsed.params, parsed.query, parsed.fragment)
)
if not full_path.startswith(b"/"):
full_path = b"/" + full_path
if not check.is_valid_host(host):
raise ValueError("Invalid Host")
return parsed.scheme, host, port, full_path
def sni(self):
for extension in self._client_hello.extensions:
is_valid_sni_extension = (
extension.type == 0x00 and
len(extension.server_names) == 1 and
extension.server_names[0].type == 0 and
check.is_valid_host(extension.server_names[0].name)
)
if is_valid_sni_extension:
return extension.server_names[0].name.decode("idna")
def sni(self) -> typing.Optional[bytes]:
if self._client_hello.extensions:
for extension in self._client_hello.extensions.extensions:
is_valid_sni_extension = (
extension.type == 0x00 and
len(extension.body.server_names) == 1 and
extension.body.server_names[0].name_type == 0 and
check.is_valid_host(extension.body.server_names[0].host_name)
)
if is_valid_sni_extension:
return extension.body.server_names[0].host_name
return None
def parse(url):
"""
URL-parsing function that checks that
- port is an integer 0-65535
- host is a valid IDNA-encoded hostname with no null-bytes
- path is valid ASCII
Args:
A URL (as bytes or as unicode)
Returns:
A (scheme, host, port, path) tuple
Raises:
ValueError, if the URL is not properly formatted.
"""
parsed = urllib.parse.urlparse(url)
if not parsed.hostname:
raise ValueError("No hostname given")
if isinstance(url, bytes):
host = parsed.hostname
# this should not raise a ValueError,
# but we try to be very forgiving here and accept just everything.
# decode_parse_result(parsed, "ascii")
else:
host = parsed.hostname.encode("idna")
parsed = encode_parse_result(parsed, "ascii")
port = parsed.port
if not port:
port = 443 if parsed.scheme == b"https" else 80
full_path = urllib.parse.urlunparse(
(b"", b"", parsed.path, parsed.params, parsed.query, parsed.fragment)
)
if not full_path.startswith(b"/"):
full_path = b"/" + full_path
if not check.is_valid_host(host):
raise ValueError("Invalid Host")
if not check.is_valid_port(port):
raise ValueError("Invalid Port")
return parsed.scheme, host, port, full_path
def parse(url):
"""
URL-parsing function that checks that
- port is an integer 0-65535
- host is a valid IDNA-encoded hostname with no null-bytes
- path is valid ASCII
Args:
A URL (as bytes or as unicode)
Returns:
A (scheme, host, port, path) tuple
Raises:
ValueError, if the URL is not properly formatted.
"""
parsed = urllib.parse.urlparse(url)
if not parsed.hostname:
raise ValueError("No hostname given")
if isinstance(url, bytes):
host = parsed.hostname
# this should not raise a ValueError,
# but we try to be very forgiving here and accept just everything.
# decode_parse_result(parsed, "ascii")
else:
host = parsed.hostname.encode("idna")
parsed = encode_parse_result(parsed, "ascii")
port = parsed.port
if not port:
port = 443 if parsed.scheme == b"https" else 80
full_path = urllib.parse.urlunparse(
(b"", b"", parsed.path, parsed.params, parsed.query, parsed.fragment)
)
if not full_path.startswith(b"/"):
full_path = b"/" + full_path
if not check.is_valid_host(host):
raise ValueError("Invalid Host")
if not check.is_valid_port(port):
raise ValueError("Invalid Port")
return parsed.scheme, host, port, full_path
def sni(self) -> Optional[str]:
"""
The [Server Name Indication](https://en.wikipedia.org/wiki/Server_Name_Indication),
which indicates which hostname the client wants to connect to.
"""
if self._client_hello.extensions:
for extension in self._client_hello.extensions.extensions:
is_valid_sni_extension = (
extension.type == 0x00
and len(extension.body.server_names) == 1
and extension.body.server_names[0].name_type == 0
and check.is_valid_host(
extension.body.server_names[0].host_name))
if is_valid_sni_extension:
return extension.body.server_names[0].host_name.decode(
"ascii")
return None
def _parse_authority_form(hostport):
"""
Returns (host, port) if hostport is a valid authority-form host specification.
http://tools.ietf.org/html/draft-luotonen-web-proxy-tunneling-01 section 3.1
Raises:
ValueError, if the input is malformed
"""
try:
host, port = hostport.split(b":")
port = int(port)
if not check.is_valid_host(host) or not check.is_valid_port(port):
raise ValueError()
except ValueError:
raise exceptions.HttpSyntaxException("Invalid host specification: {}".format(hostport))
return host, port
def _parse_authority_form(hostport):
"""
Returns (host, port) if hostport is a valid authority-form host specification.
http://tools.ietf.org/html/draft-luotonen-web-proxy-tunneling-01 section 3.1
Raises:
ValueError, if the input is malformed
"""
try:
host, port = hostport.split(b":")
port = int(port)
if not check.is_valid_host(host) or not check.is_valid_port(port):
raise ValueError()
except ValueError:
raise exceptions.HttpSyntaxException(
"Invalid host specification: {}".format(hostport))
return host, port
def parse(server_spec: str) -> ServerSpec:
"""
Parses a server mode specification, e.g.:
- http://example.com/
- example.org
- example.com:443
Raises:
ValueError, if the server specification is invalid.
"""
m = server_spec_re.match(server_spec)
if not m:
raise ValueError(f"Invalid server specification: {server_spec}")
if m.group("scheme"):
scheme = m.group("scheme")
else:
scheme = "https" if m.group("port") in ("443", None) else "http"
if scheme not in ("http", "https"):
raise ValueError(f"Invalid server scheme: {scheme}")
host = m.group("host")
# IPv6 brackets
if host.startswith("[") and host.endswith("]"):
host = host[1:-1]
if not check.is_valid_host(host):
raise ValueError(f"Invalid hostname: {host}")
if m.group("port"):
port = int(m.group("port"))
else:
port = {
"http": 80,
"https": 443
}[scheme]
if not check.is_valid_port(port):
raise ValueError(f"Invalid port: {port}")
return ServerSpec(scheme, (host, port)) # type: ignore
def parse(server_spec: str) -> ServerSpec:
"""
Parses a server mode specification, e.g.:
- http://example.com/
- example.org
- example.com:443
Raises:
ValueError, if the server specification is invalid.
"""
m = server_spec_re.match(server_spec)
if not m:
raise ValueError("Invalid server specification: {}".format(server_spec))
# defaulting to https/port 443 may annoy some folks, but it's secure-by-default.
scheme = m.group("scheme") or "https"
if scheme not in ("http", "https"):
raise ValueError("Invalid server scheme: {}".format(scheme))
host = m.group("host")
# IPv6 brackets
if host.startswith("[") and host.endswith("]"):
host = host[1:-1]
if not check.is_valid_host(host.encode("idna")):
raise ValueError("Invalid hostname: {}".format(host))
if m.group("port"):
port = int(m.group("port"))
else:
port = {
"http": 80,
"https": 443
}[scheme]
if not check.is_valid_port(port):
raise ValueError("Invalid port: {}".format(port))
return ServerSpec(scheme, (host, port))
def _load_webpack_config() -> WebpackConfig:
log.debug(f'Getting config from webpack config using {GET_CONFIG_SCRIPT_PATH}.')
proc = run(['node', GET_CONFIG_SCRIPT_PATH], stdout=PIPE)
log.debug(f'Output: {proc.stdout}.')
config = loads(proc.stdout)
dev_server_host = config.get('dev_server_host', None)
assert isinstance(dev_server_host, str), f'dev_server_host must be a string, got {dev_server_host}.'
assert is_valid_host(dev_server_host), f'dev_server_host must be a valid IP or hostname, got {dev_server_host}.'
if dev_server_host == '0.0.0.0':
dev_server_host = '127.0.0.1'
log.debug(f'Local dev server will use host {dev_server_host}.')
dev_server_port = config.get('dev_server_port', None)
if isinstance(dev_server_port, str):
try:
dev_server_port = int(dev_server_port, 10)
except ValueError:
pass
assert isinstance(dev_server_port, int), f'dev_server_port must be a number, got {dev_server_port}.'
assert is_valid_port(dev_server_port), f'dev_server_port must be a valid port, got {dev_server_port}.'
log.debug(f'Local dev server will use port {dev_server_port}.')
build_dir = config.get('build_dir', None)
assert isinstance(build_dir, str), f'build_dir must be a string, got {build_dir}.'
build_dir = Path(build_dir)
log.debug(f'Local builds will be in {build_dir}.')
upstream_url = config.get('upstream_url', None)
assert isinstance(upstream_url, str), f'upstream_url must be a string, got {upstream_url}.'
try:
parse_url(upstream_url)
except ValueError as e:
assert False, f'upstream_url must be a valid URL, got {upstream_url} ({e}).'
log.debug(f'Upstream builds will be at {upstream_url}.')
return WebpackConfig(dev_server_host, dev_server_port, build_dir, upstream_url)
def test_is_valid_host():
assert not check.is_valid_host(b"")
assert not check.is_valid_host(b"xn--ke.ws")
assert check.is_valid_host(b"one.two")
assert not check.is_valid_host(b"one" * 255)
assert check.is_valid_host(b"one.two.")
# Allow underscore
assert check.is_valid_host(b"one_two")
assert check.is_valid_host(b"::1")
# IP Address Validations
assert check.is_valid_host(b'127.0.0.1')
assert check.is_valid_host(b'2001:0db8:85a3:0000:0000:8a2e:0370:7334')
assert check.is_valid_host(b'2001:db8:85a3:0:0:8a2e:370:7334')
assert check.is_valid_host(b'2001:db8:85a3::8a2e:370:7334')
assert not check.is_valid_host(b'2001:db8::85a3::7334')
assert check.is_valid_host(b'2001-db8-85a3-8d3-1319-8a2e-370-7348.ipv6-literal.net')
# TLD must be between 2 and 63 chars
assert check.is_valid_host(b'example.tl')
assert check.is_valid_host(b'example.tld')
assert check.is_valid_host(b'example.' + b"x" * 63)
assert not check.is_valid_host(b'example.' + b"x" * 64)
# misc characters test
assert not check.is_valid_host(b'ex@mple')
assert not check.is_valid_host(b'*****@*****.**')
assert not check.is_valid_host(b'example..com')
assert not check.is_valid_host(b'.example.com')
assert not check.is_valid_host(b'@.example.com')
assert not check.is_valid_host(b'!.example.com')
# Every label must be between 1 and 63 chars
assert not check.is_valid_host(b'.tld')
assert check.is_valid_host(b'x' * 1 + b'.tld')
assert check.is_valid_host(b'x' * 30 + b'.tld')
assert not check.is_valid_host(b'x' * 64 + b'.tld')
assert check.is_valid_host(b'x' * 1 + b'.example.tld')
assert check.is_valid_host(b'x' * 30 + b'.example.tld')
assert not check.is_valid_host(b'x' * 64 + b'.example.tld')
# Misc Underscore Test Cases
assert check.is_valid_host(b'_example')
assert check.is_valid_host(b'_example_')
assert check.is_valid_host(b'example_')
assert check.is_valid_host(b'_a.example.tld')
assert check.is_valid_host(b'a_.example.tld')
assert check.is_valid_host(b'_a_.example.tld')
# Misc Dash/Hyphen/Minus Test Cases
assert check.is_valid_host(b'-example')
assert check.is_valid_host(b'-example_')
assert check.is_valid_host(b'example-')
assert check.is_valid_host(b'-a.example.tld')
assert check.is_valid_host(b'a-.example.tld')
assert check.is_valid_host(b'-a-.example.tld')
# Misc Combo Test Cases
assert check.is_valid_host(b'api-.example.com')
assert check.is_valid_host(b'__a.example-site.com')
assert check.is_valid_host(b'_-a.example-site.com')
assert check.is_valid_host(b'_a_.example-site.com')
assert check.is_valid_host(b'-a-.example-site.com')
assert check.is_valid_host(b'api-.a.example.com')
assert check.is_valid_host(b'api-._a.example.com')
assert check.is_valid_host(b'api-.a_.example.com')
assert check.is_valid_host(b'api-.ab.example.com')
# Test str
assert check.is_valid_host('example.tld')
assert not check.is_valid_host("foo..bar") # cannot be idna-encoded.
def test_is_valid_host():
assert not check.is_valid_host(b"")
assert check.is_valid_host(b"one.two")
assert not check.is_valid_host(b"one" * 255)
assert check.is_valid_host(b"one.two.")
