def test_ip_whitelist_blacklist_conflict(self):
"""Manual whitelist should take precedence over manual blacklist"""
validator = AddrValidator(
ip_whitelist=(ipaddress.ip_network("127.0.0.1"), ),
ip_blacklist=(ipaddress.ip_network("127.0.0.1"), ),
)
self.assertTrue(validator.is_ip_allowed("127.0.0.1"))
def test_advocate_wrapper_futures(self):
wrapper = RequestsAPIWrapper(validator=AddrValidator())
local_validator = AddrValidator(ip_whitelist={
ipaddress.ip_network("127.0.0.1"),
})
local_wrapper = RequestsAPIWrapper(validator=local_validator)
with self.assertRaises(UnacceptableAddressException):
sess = wrapper.FuturesSession()
sess.get("http://127.0.0.1/").result()
with self.assertRaises(Exception) as cm:
sess = local_wrapper.FuturesSession()
sess.get("http://127.0.0.1:1/").result()
# Check that we got a connection exception instead of a validation one
# This might be either exception depending on the requests version
self.assertRegexpMatches(
cm.exception.__class__.__name__,
r"\A(Connection|Protocol)Error",
)
with self.assertRaises(UnacceptableAddressException):
sess = wrapper.FuturesSession()
sess.get("http://localhost:1/").result()
with self.assertRaises(UnacceptableAddressException):
sess = wrapper.FuturesSession()
sess.get("https://localhost:1/").result()
def parse(self, key_path: str,
raw_config: config.RawConfig) -> RequestsContextFactory:
# use advocate to ensure this client only ever gets used
# with internal services over the internal network.
#
# the allowlist takes precedence. allow loopback and 10./8 private
# addresses, deny the rest.
validator = AddrValidator(
ip_whitelist={
ipaddress.ip_network("127.0.0.0/8"),
ipaddress.ip_network("10.0.0.0/8")
},
ip_blacklist={ipaddress.ip_network("0.0.0.0/0")},
port_blacklist=[
0
], # disable the default allowlist by giving an explicit denylist
allow_ipv6=False,
)
adapter = http_adapter_from_config(raw_config,
prefix=f"{key_path}.",
validator=validator,
**self.kwargs)
return RequestsContextFactory(adapter,
session_cls=InternalBaseplateSession)
def test_advocate_requests_api_wrapper_hostnames(self):
wrapper = RequestsAPIWrapper(
validator=AddrValidator(hostname_blacklist={"google.com"}, ))
self.assertRaises(
UnacceptableAddressException,
wrapper.get,
"https://google.com/",
)
def test_advocate_requests_api_wrapper_req_methods(self):
# Make sure all the convenience methods make requests with the correct
# methods
wrapper = RequestsAPIWrapper(AddrValidator())
request_methods = ("get", "options", "head", "post", "put", "patch",
"delete")
for method_name in request_methods:
with requests_mock.mock() as request_mock:
# This will fail if the request expected by `request_mock`
# isn't sent when calling the wrapper method
request_mock.request(method_name, "http://example.com/foo")
getattr(wrapper, method_name)("http://example.com/foo")
def permissive_validator(**kwargs):
default_options = dict(
ip_blacklist=None,
port_whitelist=None,
port_blacklist=None,
hostname_blacklist=None,
allow_ipv6=True,
allow_teredo=True,
allow_6to4=True,
allow_dns64=True,
autodetect_local_addresses=False,
)
default_options.update(**kwargs)
return AddrValidator(**default_options)
def test_safecurl_blacklist(self):
"""Test that we at least disallow everything SafeCurl does"""
# All IPs that SafeCurl would disallow
bad_netblocks = (ipaddress.ip_network(x)
for x in ('0.0.0.0/8', '10.0.0.0/8', '100.64.0.0/10',
'127.0.0.0/8', '169.254.0.0/16',
'172.16.0.0/12', '192.0.0.0/29',
'192.0.2.0/24', '192.88.99.0/24',
'192.168.0.0/16', '198.18.0.0/15',
'198.51.100.0/24', '203.0.113.0/24',
'224.0.0.0/4', '240.0.0.0/4'))
i = 0
validator = AddrValidator()
for bad_netblock in bad_netblocks:
num_ips = bad_netblock.num_addresses
# Don't test *every* IP in large netblocks
step_size = int(min(max(num_ips / 255, 1), 128))
for ip_idx in six.moves.range(0, num_ips, step_size):
i += 1
bad_ip = bad_netblock[ip_idx]
bad_ip_allowed = validator.is_ip_allowed(bad_ip)
if bad_ip_allowed:
print(i, bad_ip)
self.assertFalse(bad_ip_allowed)
def test_wrapper_session_pickle(self):
# Make sure the validator still works after a pickle round-trip
wrapper = RequestsAPIWrapper(validator=AddrValidator(ip_whitelist={
ipaddress.ip_network("127.0.0.1"),
}))
sess_instance = pickle.loads(pickle.dumps(wrapper.Session()))
with self.assertRaises(Exception) as cm:
sess_instance.get("http://127.0.0.1:1/")
self.assertRegexpMatches(
cm.exception.__class__.__name__,
r"\A(Connection|Protocol)Error",
)
self.assertRaises(UnacceptableAddressException, sess_instance.get,
"http://127.0.1.1:1/")
def test_manual_ip_blacklist(self):
"""Test manually blacklisting based on IP"""
validator = AddrValidator(
allow_ipv6=True,
ip_blacklist=(
ipaddress.ip_network("132.0.5.0/24"),
ipaddress.ip_network("152.0.0.0/8"),
ipaddress.ip_network("::1"),
),
)
self.assertFalse(validator.is_ip_allowed("132.0.5.1"))
self.assertFalse(validator.is_ip_allowed("152.254.90.1"))
self.assertTrue(validator.is_ip_allowed("178.254.90.1"))
self.assertFalse(validator.is_ip_allowed("::1"))
# Google, found via `dig google.com AAAA`
self.assertTrue(validator.is_ip_allowed("2607:f8b0:400a:807::200e"))
def test_wrapper_session_subclass(self):
# Make sure pickle doesn't explode if we try to pickle a subclass
# of `wrapper.Session`
wrapper = RequestsAPIWrapper(validator=AddrValidator(ip_whitelist={
ipaddress.ip_network("127.0.0.1"),
}))
class _SessionThing(wrapper.Session):
pass
sess_instance = pickle.loads(pickle.dumps(_SessionThing()))
with self.assertRaises(Exception) as cm:
sess_instance.get("http://127.0.0.1:1/")
self.assertRegexpMatches(
cm.exception.__class__.__name__,
r"\A(Connection|Protocol)Error",
)
self.assertRaises(UnacceptableAddressException, sess_instance.get,
"http://127.0.1.1:1/")
def http_adapter_from_config(
app_config: config.RawConfig, prefix: str, **kwargs: Any
) -> HTTPAdapter:
"""Make an HTTPAdapter from a configuration dictionary.
The keys useful to :py:func:`http_adapter_from_config` should be prefixed,
e.g. ``http.pool_connections``, ``http.max_retries``, etc. The ``prefix``
argument specifies the prefix used. Each key is mapped to a corresponding
keyword argument on the :py:class:`~requests.adapters.HTTPAdapter`
constructor.
Supported keys:
* ``pool_connections``: The number of connections to cache (default: 10).
* ``pool_maxsize``: The maximum number of connections to keep in the pool
(default: 10).
* ``max_retries``: How many times to retry DNS lookups or connection
attempts, but never sending data (default: 0).
* ``pool_block``: Whether the connection pool will block when trying to get
a connection (default: false).
Additionally, the rules for Advocate's address filtering can be configured
with the ``filter`` sub-keys:
* ``filter.ip_allowlist``: A comma-delimited list of IP addresses (1.2.3.4)
or CIDR-notation (1.2.3.0/24) ranges that the client can always connect to
(default: anything not on the local network).
* ``filter.ip_denylist``: A comma-delimited list of IP addresses or
CIDR-notation ranges the client may never connect to (default: the local network).
* ``filter.port_allowlist``: A comma-delimited list of TCP port numbers
that the client can connect to (default: 80, 8080, 443, 8443, 8000).
* ``filter.port_denylist``: A comma-delimited list of TCP port numbers that
the client may never connect to (default: none).
* ``filter.hostname_denylist``: A comma-delimited list of hostnames that
the client may never connect to (default: none).
* ``filter.allow_ipv6``: Should the client be allowed to connect to IPv6
hosts? (default: false, note: IPv6 is tricky to apply filtering rules
comprehensively to).
"""
assert prefix.endswith(".")
parser = config.SpecParser(
{
"pool_connections": config.Optional(config.Integer, default=10),
"pool_maxsize": config.Optional(config.Integer, default=10),
"max_retries": config.Optional(config.Integer, default=0),
"pool_block": config.Optional(config.Boolean, default=False),
"filter": {
"ip_allowlist": config.Optional(config.TupleOf(ipaddress.ip_network)),
"ip_denylist": config.Optional(config.TupleOf(ipaddress.ip_network)),
"port_allowlist": config.Optional(config.TupleOf(int)),
"port_denylist": config.Optional(config.TupleOf(int)),
"hostname_denylist": config.Optional(config.TupleOf(config.String)),
"allow_ipv6": config.Optional(config.Boolean, default=False),
},
}
)
options = parser.parse(prefix[:-1], app_config)
if options.pool_connections is not None:
kwargs.setdefault("pool_connections", options.pool_connections)
if options.pool_maxsize is not None:
kwargs.setdefault("pool_maxsize", options.pool_maxsize)
if options.max_retries is not None:
kwargs.setdefault("max_retries", options.max_retries)
if options.pool_block is not None:
kwargs.setdefault("pool_block", options.pool_block)
kwargs.setdefault(
"validator",
AddrValidator(
ip_whitelist=options.filter.ip_allowlist,
ip_blacklist=options.filter.ip_denylist,
port_whitelist=options.filter.port_allowlist,
port_blacklist=options.filter.port_denylist,
hostname_blacklist=options.filter.hostname_denylist,
allow_ipv6=options.filter.allow_ipv6,
),
)
return ValidatingHTTPAdapter(**kwargs)
def test_custom_validator(self):
validator = AddrValidator(hostname_blacklist={"example.org"})
sess = FuturesSession(validator=validator)
self.assertRaises(UnacceptableAddressException,
lambda: sess.get("http://example.org").result())
def test_wrapper_getattr_fallback(self):
# Make sure wrappers include everything in Advocate's `__init__.py`
wrapper = RequestsAPIWrapper(AddrValidator())
self.assertIsNotNone(wrapper.PreparedRequest)
def test_blacklist_hostname(self):
self.assertRaises(
UnacceptableAddressException,
advocate.get,
"https://google.com/",
validator=AddrValidator(hostname_blacklist={"google.com"}))
def test_ip_whitelist(self):
"""Test manually whitelisting based on IP"""
validator = AddrValidator(
ip_whitelist=(ipaddress.ip_network("127.0.0.1"), ), )
self.assertTrue(validator.is_ip_allowed("127.0.0.1"))
from advocate.connection import advocate_getaddrinfo
from advocate.exceptions import (
MountDisabledException,
NameserverException,
UnacceptableAddressException,
)
from advocate.packages import ipaddress
from advocate.futures import FuturesSession
# We use port 1 for testing because nothing is likely to legitimately listen
# on it.
AddrValidator.DEFAULT_PORT_WHITELIST.add(1)
RequestsAPIWrapper.SUPPORT_WRAPPER_PICKLING = True
global_wrapper = RequestsAPIWrapper(validator=AddrValidator(ip_whitelist={
ipaddress.ip_network("127.0.0.1"),
}))
RequestsAPIWrapper.SUPPORT_WRAPPER_PICKLING = False
class _WrapperSubclass(global_wrapper.Session):
def good_method(self):
return "foo"
def canonname_supported():
"""Check if the nameserver supports the AI_CANONNAME flag
travis-ci.org's Python 3 env doesn't seem to support it, so don't try
any of the test that rely on it.
"""
