def test_custom_not_before_not_after():
ca = CertificateAuthority(
"Test Custom CA",
TEST_CA_ROOT,
cert_not_before=-60 * 60,
cert_not_after=60 * 60 * 24 * 3,
)
# check PKCS12
buff_pk12 = ca.get_root_PKCS12()
assert len(buff_pk12) > 0
cert = crypto.load_pkcs12(buff_pk12).get_certificate()
expected_not_before = datetime.datetime.utcnow() - datetime.timedelta(
seconds=60 * 60)
expected_not_after = datetime.datetime.utcnow() + datetime.timedelta(
seconds=60 * 60 * 24 * 3)
actual_not_before = datetime.datetime.strptime(
cert.get_notBefore().decode("ascii"), "%Y%m%d%H%M%SZ")
actual_not_after = datetime.datetime.strptime(
cert.get_notAfter().decode("ascii"), "%Y%m%d%H%M%SZ")
time.mktime(expected_not_before.utctimetuple())
assert (abs((time.mktime(actual_not_before.utctimetuple()) -
time.mktime(expected_not_before.utctimetuple()))) < 10)
assert (abs((time.mktime(actual_not_after.utctimetuple()) -
time.mktime(expected_not_after.utctimetuple()))) < 10)
def test_explicit_wildcard():
ca = CertificateAuthority(TEST_CA_ROOT, TEST_CA_DIR, 'Test CA')
filename = ca.get_wildcard_cert('test.example.proxy')
certfile = os.path.join(TEST_CA_DIR, 'example.proxy.pem')
assert filename == certfile
assert os.path.isfile(certfile)
os.remove(certfile)
def do_CONNECT(self):
hostname = self.path.split(':')[0]
from certauth.certauth import CertificateAuthority
ca = CertificateAuthority('My Custom CA',
'ca/certs/ca.pem',
cert_cache='tmp/certs')
filename = ca.cert_for_host(hostname)
self.wfile.write("{0} {1} {2}\r\n\r\n".format(
'HTTP/1.1', 200, 'Connection Established').encode())
self.wfile.flush()
self.connection = ssl.wrap_socket(self.connection,
keyfile=filename,
certfile=filename,
server_side=True)
self.rfile = self.connection.makefile("rb", self.rbufsize)
self.wfile = self.connection.makefile("wb", self.wbufsize)
conntype = self.headers.get('Proxy-Connection', '')
if conntype.lower() != 'close':
self.close_connection = 0
else:
self.close_connection = 1
def create_or_update(self, refresh=False):
if not self.cache.get("!!root_ca"):
self.fail(
f"certificate for root ca '{self.ca_name}' does not exist.")
return
ca = CertificateAuthority(
self.ca_name,
ca_file_cache=self.cache,
cert_cache=self.cache,
overwrite=False,
)
cert, pkey, entry = ca.load_cert(
self.hostname,
overwrite=refresh,
wildcard=self.wildcard,
wildcard_use_parent=False,
include_cache_key=True,
)
public_cert_pem = crypto.dump_certificate(FILETYPE_PEM, cert)
self.set_attribute("CAName", self.ca_name)
self.set_attribute("Hostname", self.hostname)
self.set_attribute("Hash", hashlib.md5(public_cert_pem).hexdigest())
self.set_attribute("PublicCertPEM", public_cert_pem.decode("ascii"))
self.physical_resource_id = self.cache.parameter_name(self.hostname)
def __init__(self, routes, **kwargs):
self.error_view = kwargs.get('error_view')
proxy_options = kwargs.get('config', {})
if proxy_options:
proxy_options = proxy_options.get('proxy_options', {})
self.magic_name = proxy_options.get('magic_name')
if not self.magic_name:
self.magic_name = self.DEF_MAGIC_NAME
proxy_options['magic_name'] = self.magic_name
self.extra_headers = proxy_options.get('extra_headers')
if not self.extra_headers:
self.extra_headers = self.EXTRA_HEADERS
proxy_options['extra_headers'] = self.extra_headers
res_type = proxy_options.get('cookie_resolver', True)
if res_type == 'auth' or not res_type:
self.resolver = ProxyAuthResolver(routes, proxy_options)
elif res_type == 'ip':
self.resolver = IPCacheResolver(routes, proxy_options)
#elif res_type == True or res_type == 'cookie':
# self.resolver = CookieResolver(routes, proxy_options)
else:
self.resolver = CookieResolver(routes, proxy_options)
self.use_banner = proxy_options.get('use_banner', True)
self.use_wombat = proxy_options.get('use_client_rewrite', True)
self.proxy_cert_dl_view = proxy_options.get('proxy_cert_download_view')
if not proxy_options.get('enable_https_proxy'):
self.ca = None
return
try:
from certauth.certauth import CertificateAuthority
except ImportError: #pragma: no cover
print('HTTPS proxy is not available as the "certauth" module ' +
'is not installed')
print('Please install via "pip install certauth" ' +
'to enable HTTPS support')
self.ca = None
return
# HTTPS Only Options
ca_file = proxy_options.get('root_ca_file', self.CA_ROOT_FILE)
# attempt to create the root_ca_file if doesn't exist
# (generally recommended to create this seperately)
ca_name = proxy_options.get('root_ca_name', self.CA_ROOT_NAME)
certs_dir = proxy_options.get('certs_dir', self.CA_CERTS_DIR)
self.ca = CertificateAuthority(ca_file=ca_file,
certs_dir=certs_dir,
ca_name=ca_name)
self.use_wildcard = proxy_options.get('use_wildcard_certs', True)
def __init__(self,
wsgi,
prefix_resolver=None,
download_host=None,
proxy_host=None,
proxy_options=None,
proxy_apps=None):
self._wsgi = wsgi
self.set_connection_class()
if isinstance(prefix_resolver, str):
prefix_resolver = FixedResolver(prefix_resolver)
self.prefix_resolver = prefix_resolver or FixedResolver()
self.proxy_apps = proxy_apps or {}
self.proxy_host = proxy_host or self.DEFAULT_HOST
if self.proxy_host not in self.proxy_apps:
self.proxy_apps[self.proxy_host] = None
# HTTPS Only Options
proxy_options = proxy_options or {}
ca_name = proxy_options.get('ca_name', self.CA_ROOT_NAME)
ca_file_cache = proxy_options.get('ca_file_cache', self.CA_ROOT_FILE)
self.ca = CertificateAuthority(ca_name=ca_name,
ca_file_cache=ca_file_cache,
cert_cache=None,
cert_not_before=-3600)
self.keepalive_max = proxy_options.get('keepalive_max',
self.DEFAULT_MAX_TUNNELS)
self.keepalive_opts = hasattr(socket, 'TCP_KEEPIDLE')
self._tcp_keepidle = proxy_options.get('tcp_keepidle', 60)
self._tcp_keepintvl = proxy_options.get('tcp_keepintvl', 5)
self._tcp_keepcnt = proxy_options.get('tcp_keepcnt', 3)
self.num_open_tunnels = 0
try:
self.root_ca_file = self.ca.get_root_pem_filename()
except Exception as e:
self.root_ca_file = None
self.use_wildcard = proxy_options.get('use_wildcard_certs', True)
if proxy_options.get('enable_cert_download', True):
download_host = download_host or self.DEFAULT_HOST
self.proxy_apps[download_host] = CertDownloader(self.ca)
self.enable_ws = proxy_options.get('enable_websockets', True)
if WebSocketHandler == object:
self.enable_ws = None
def test_create_root_subdir():
# create a new cert in a subdirectory
subdir = os.path.join(TEST_CA_DIR, 'subdir')
ca = CertificateAuthority(TEST_CA_ROOT, subdir, 'Test CA')
assert os.path.isdir(subdir)
buff = ca.get_root_PKCS12()
assert len(buff) > 0
def test_in_mem_parent_wildcard_cert_2():
cert_cache = {}
ca = CertificateAuthority('Test CA', TEST_CA_ROOT, cert_cache)
cert, key = ca.load_cert('test.example.org.uk',
wildcard=True,
wildcard_use_parent=True)
assert 'example.org.uk' in cert_cache, cert_cache.keys()
cached_value = cert_cache['example.org.uk']
cert2, key2 = ca.load_cert('example.org.uk')
# assert underlying cache unchanged
assert cached_value == cert_cache['example.org.uk']
verify_cert_san(cert2, ['DNS:example.org.uk', 'DNS:*.example.org.uk'])
def test_in_mem_parent_wildcard_cert():
cert_cache = {}
ca = CertificateAuthority("Test CA", TEST_CA_ROOT, cert_cache)
cert, key = ca.load_cert("test.example.proxy",
wildcard=True,
wildcard_use_parent=True)
assert "example.proxy" in cert_cache, cert_cache.keys()
cached_value = cert_cache["example.proxy"]
cert2, key2 = ca.load_cert("example.proxy")
# assert underlying cache unchanged
assert cached_value == cert_cache["example.proxy"]
verify_cert_san(cert2, "DNS:example.proxy, DNS:*.example.proxy")
def test_create_root_subdir():
# create a new cert in a subdirectory
subdir = os.path.join(TEST_CA_DIR, "subdir")
ca_file = os.path.join(subdir, "certauth_test_ca.pem")
certs_dir = os.path.join(subdir, "certs")
ca = CertificateAuthority("Test CA", ca_file, certs_dir)
assert os.path.isdir(subdir)
assert os.path.isfile(ca_file)
assert os.path.isdir(certs_dir)
assert ca.get_root_pem_filename() == ca_file
def test_in_mem_cert():
cert_cache = {}
ca = CertificateAuthority('Test CA',
TEST_CA_ROOT,
cert_cache,
overwrite=True)
res = ca.load_cert('test.example.proxy')
assert 'test.example.proxy' in cert_cache, cert_cache.keys()
cached_value = cert_cache['test.example.proxy']
cert, key = ca.load_cert('test.example.proxy')
verify_cert_san(cert, ['DNS:test.example.proxy'])
# assert underlying cache unchanged
assert cached_value == cert_cache['test.example.proxy']
def test_in_mem_cert():
cert_cache = {}
ca = CertificateAuthority("Test CA",
TEST_CA_ROOT,
cert_cache,
overwrite=True)
res = ca.load_cert("test.example.proxy")
assert "test.example.proxy" in cert_cache, cert_cache.keys()
cached_value = cert_cache["test.example.proxy"]
cert, key = ca.load_cert("test.example.proxy")
verify_cert_san(cert, "DNS:test.example.proxy")
# assert underlying cache unchanged
assert cached_value == cert_cache["test.example.proxy"]
def __init__(self,
server_address=('localhost', 8000),
req_handler_class=WarcProxyHandler,
bind_and_activate=True,
ca=None,
recorded_url_q=None,
digest_algorithm='sha1',
method_filter=False):
http_server.HTTPServer.__init__(self, server_address,
req_handler_class, bind_and_activate)
self.digest_algorithm = digest_algorithm
if ca is not None:
self.ca = ca
else:
ca_name = 'Warcprox CA on {}'.format(socket.gethostname())[:64]
self.ca = CertificateAuthority(ca_file='warcprox-ca.pem',
certs_dir='./warcprox-ca',
ca_name=ca_name)
if recorded_url_q is not None:
self.recorded_url_q = recorded_url_q
else:
self.recorded_url_q = queue.Queue()
self.method_filter = method_filter
def __init__(self, routes, **kwargs):
self.error_view = kwargs.get('error_view')
proxy_options = kwargs.get('config', {})
if proxy_options:
proxy_options = proxy_options.get('proxy_options', {})
self.magic_name = proxy_options.get('magic_name')
if not self.magic_name:
self.magic_name = self.DEF_MAGIC_NAME
proxy_options['magic_name'] = self.magic_name
self.extra_headers = proxy_options.get('extra_headers')
if not self.extra_headers:
self.extra_headers = self.EXTRA_HEADERS
proxy_options['extra_headers'] = self.extra_headers
res_type = proxy_options.get('cookie_resolver', True)
if res_type == 'auth' or not res_type:
self.resolver = ProxyAuthResolver(routes, proxy_options)
elif res_type == 'ip':
self.resolver = IPCacheResolver(routes, proxy_options)
#elif res_type == True or res_type == 'cookie':
# self.resolver = CookieResolver(routes, proxy_options)
else:
self.resolver = CookieResolver(routes, proxy_options)
self.use_banner = proxy_options.get('use_banner', True)
self.use_wombat = proxy_options.get('use_client_rewrite', True)
self.proxy_cert_dl_view = proxy_options.get('proxy_cert_download_view')
if not proxy_options.get('enable_https_proxy'):
self.ca = None
return
try:
from certauth.certauth import CertificateAuthority
except ImportError: #pragma: no cover
print('HTTPS proxy is not available as the "certauth" module ' +
'is not installed')
print('Please install via "pip install certauth" ' +
'to enable HTTPS support')
self.ca = None
return
# HTTPS Only Options
ca_file = proxy_options.get('root_ca_file', self.CA_ROOT_FILE)
# attempt to create the root_ca_file if doesn't exist
# (generally recommended to create this seperately)
ca_name = proxy_options.get('root_ca_name', self.CA_ROOT_NAME)
certs_dir = proxy_options.get('certs_dir', self.CA_CERTS_DIR)
self.ca = CertificateAuthority(ca_file=ca_file,
certs_dir=certs_dir,
ca_name=ca_name)
self.use_wildcard = proxy_options.get('use_wildcard_certs', True)
def test_ca_cert_in_mem():
root_cert_dict = {}
ca = CertificateAuthority("Test CA", root_cert_dict, 10)
# check PEM
buff_pem = ca.get_root_pem()
assert len(buff_pem) > 0
# PEM stored in root_cert_dict
assert root_cert_dict[ROOT_CA] == buff_pem
cert_pem = crypto.load_certificate(FILETYPE_PEM, buff_pem)
# check PKCS12
buff_pk12 = ca.get_root_PKCS12()
assert len(buff_pk12) > 0
cert_pk12 = crypto.load_pkcs12(buff_pk12).get_certificate()
def generate_certs(host, w):
"""Generates private.key, cert.crt, and ca-bundle.crt..."""
WORKDIR = "/".join(__file__.split("/")[:-1])
with open(WORKDIR + "/config.json", "r") as config_file:
CONFIG = json.load(config_file)
path = "."
ca = CertificateAuthority(CONFIG["CERT_AUTH_NAME"],
CONFIG["CERT_AUTH_ROOT_FILE"],
cert_cache=path)
cert, key = ca.load_cert(host, wildcard=w)
## Extract root CA and host private keys and certificates from pem files ##
with open(CONFIG["CERT_AUTH_ROOT_FILE"], "r") as py_ca_file:
py_ca_data = "".join(py_ca_file.readlines())
ca_private_key, ca_root_cert = split_certs(py_ca_data)
with open(path + "/" + host + ".pem", "r") as host_file:
host_data = "".join(host_file.readlines())
host_private_key, host_cert = split_certs(host_data)
## Output certificates to host folder ##
with open(path + "/ca.crt", "w+") as py_ca_root_cert_file:
py_ca_root_cert_file.write(ca_root_cert)
print(f"Generating " + path + "/ca_bundle.crt")
with open(path + "/private.key", "w+") as host_private_key_file:
host_private_key_file.write(host_private_key)
print(f"Generating " + path + "/private.key")
with open(path + "/cert.crt", "w+") as host_cert_file:
host_cert_file.write(host_cert)
print(f"Generating " + path + "/cert.crt")
with open(path + "/domain-cert.crt", "w+") as concat_cert_file:
concat_cert_file.write(ca_root_cert + host_cert)
print(f"Generating " + path + "/concat.crt")
print("Finished!")
def test_ca_lru_cache():
lru = LRUCache(max_size=2)
ca = CertificateAuthority("Test CA LRU Cache", TEST_CA_ROOT, lru)
res = ca.load_cert("example.com")
assert "example.com" in lru
assert len(lru) == 1
res = ca.load_cert("ABC.example.com")
assert "ABC.example.com" in lru
assert "example.com" in lru
assert len(lru) == 2
res = ca.load_cert("XYZ.example.com", include_cache_key=True)
assert res[2] == "XYZ.example.com"
assert "XYZ.example.com" in lru
assert "ABC.example.com" in lru
assert len(lru) == 2
assert "example.com" not in lru
def test_ca_lru_cache():
lru = LRUCache(max_size=2)
ca = CertificateAuthority('Test CA LRU Cache', TEST_CA_ROOT, lru)
res = ca.load_cert('example.com')
assert 'example.com' in lru
assert len(lru) == 1
res = ca.load_cert('ABC.example.com')
assert 'ABC.example.com' in lru
assert 'example.com' in lru
assert len(lru) == 2
res = ca.load_cert('XYZ.example.com', include_cache_key=True)
assert res[2] == 'XYZ.example.com'
assert 'XYZ.example.com' in lru
assert 'ABC.example.com' in lru
assert len(lru) == 2
assert 'example.com' not in lru
def create_keys(temp_dir, common_name):
"""Create wildcard certificate for the proxy base url."""
ca = CertificateAuthority(
f'Riptide Proxy CA for {socket.gethostname()}',
get_ca_path(),
cert_cache=temp_dir,
# Make it valid for 364 days, as per MacOS Catalina requirements
cert_not_after=NOT_VALID_AFTER)
if ca.ca_cert.has_expired():
logger.warning("The CA certificate had to be re-generated because"
"it was no longer valid. You may need to re-import it,"
"please see the documentation.")
ca = CertificateAuthority(
f'Riptide Proxy CA for {socket.gethostname()}',
get_ca_path(),
cert_cache=temp_dir,
cert_not_after=NOT_VALID_AFTER,
overwrite=True)
return ca.get_wildcard_cert('*.' + common_name)
def __init__(self,
stats_db=None,
status_callback=None,
options=warcprox.Options()):
self.status_callback = status_callback
self.stats_db = stats_db
self.options = options
self.remote_connection_pool = PoolManager(
num_pools=max(round(options.max_threads /
6), 200) if options.max_threads else 200)
server_address = (options.address or 'localhost',
options.port if options.port is not None else 8000)
if options.onion_tor_socks_proxy:
try:
host, port = options.onion_tor_socks_proxy.split(':')
WarcProxyHandler.onion_tor_socks_proxy_host = host
WarcProxyHandler.onion_tor_socks_proxy_port = int(port)
except ValueError:
WarcProxyHandler.onion_tor_socks_proxy_host = options.onion_tor_socks_proxy
WarcProxyHandler.onion_tor_socks_proxy_port = None
if options.socket_timeout:
WarcProxyHandler._socket_timeout = options.socket_timeout
if options.max_resource_size:
WarcProxyHandler._max_resource_size = options.max_resource_size
if options.tmp_file_max_memory_size:
WarcProxyHandler._tmp_file_max_memory_size = options.tmp_file_max_memory_size
http_server.HTTPServer.__init__(self,
server_address,
WarcProxyHandler,
bind_and_activate=True)
self.digest_algorithm = options.digest_algorithm or 'sha1'
ca_name = ('Warcprox CA on %s' % socket.gethostname())[:64]
self.ca = CertificateAuthority(ca_file=options.cacert
or 'warcprox-ca.pem',
certs_dir=options.certs_dir
or './warcprox-ca',
ca_name=ca_name)
self.recorded_url_q = warcprox.TimestampedQueue(
maxsize=options.queue_size or 1000)
self.running_stats = warcprox.stats.RunningStats()
def __init__(self,
MitmProxyHandlerClass=MitmProxyHandler,
options=warcprox.Options()):
self.options = options
# TTLCache is not thread-safe. Access to the shared cache from multiple
# threads must be properly synchronized with an RLock according to ref:
# https://cachetools.readthedocs.io/en/latest/
self.bad_hostnames_ports = TTLCache(maxsize=1024, ttl=60)
self.bad_hostnames_ports_lock = RLock()
self.remote_connection_pool = PoolManager(num_pools=max(
(options.max_threads or 0) // 6, 400),
maxsize=6)
if options.onion_tor_socks_proxy:
try:
host, port = options.onion_tor_socks_proxy.split(':')
MitmProxyHandlerClass.onion_tor_socks_proxy_host = host
MitmProxyHandlerClass.onion_tor_socks_proxy_port = int(port)
except ValueError:
MitmProxyHandlerClass.onion_tor_socks_proxy_host = options.onion_tor_socks_proxy
MitmProxyHandlerClass.onion_tor_socks_proxy_port = None
if options.socket_timeout:
MitmProxyHandlerClass._socket_timeout = options.socket_timeout
if options.max_resource_size:
MitmProxyHandlerClass._max_resource_size = options.max_resource_size
if options.tmp_file_max_memory_size:
MitmProxyHandlerClass._tmp_file_max_memory_size = options.tmp_file_max_memory_size
self.digest_algorithm = options.digest_algorithm or 'sha1'
ca_name = ('Warcprox CA on %s' % socket.gethostname())[:64]
self.ca = CertificateAuthority(ca_file=options.cacert
or 'warcprox-ca.pem',
certs_dir=options.certs_dir
or './warcprox-ca',
ca_name=ca_name)
server_address = (options.address or 'localhost',
options.port if options.port is not None else 8000)
http_server.HTTPServer.__init__(self,
server_address,
MitmProxyHandlerClass,
bind_and_activate=True)
def __init__(self,
ca=None,
recorded_url_q=None,
stats_db=None,
options=warcprox.Options()):
server_address = (options.address or 'localhost',
options.port if options.port is not None else 8000)
if options.onion_tor_socks_proxy:
try:
host, port = options.onion_tor_socks_proxy.split(':')
WarcProxyHandler.onion_tor_socks_proxy_host = host
WarcProxyHandler.onion_tor_socks_proxy_port = int(port)
except ValueError:
WarcProxyHandler.onion_tor_socks_proxy_host = options.onion_tor_socks_proxy
WarcProxyHandler.onion_tor_socks_proxy_port = None
http_server.HTTPServer.__init__(self,
server_address,
WarcProxyHandler,
bind_and_activate=True)
self.digest_algorithm = options.digest_algorithm or 'sha1'
if ca is not None:
self.ca = ca
else:
ca_name = 'Warcprox CA on {}'.format(socket.gethostname())[:64]
self.ca = CertificateAuthority(ca_file='warcprox-ca.pem',
certs_dir='./warcprox-ca',
ca_name=ca_name)
if recorded_url_q is not None:
self.recorded_url_q = recorded_url_q
else:
self.recorded_url_q = warcprox.TimestampedQueue(
maxsize=options.queue_size or 1000)
self.stats_db = stats_db
self.options = options
self.running_stats = warcprox.stats.RunningStats()
def create_or_update(self, overwrite=False):
ca = CertificateAuthority(
self.ca_name,
ca_file_cache=self.cache,
cert_cache=self.cache,
overwrite=overwrite,
)
public_cert_pem = crypto.dump_certificate(FILETYPE_PEM, ca.ca_cert)
self.set_attribute("CAName", self.ca_name)
self.set_attribute("Hash", hashlib.md5(public_cert_pem).hexdigest())
self.set_attribute("PublicCertPEM", public_cert_pem.decode("ascii"))
_ = ssm.put_parameter(
Name=self.public_cert_parameter_name,
Value=self.get_attribute("PublicCertPEM"),
Overwrite=True,
Type="String",
)
self.physical_resource_id = self.cache.parameter_name("!!root_ca")
class ProxyRouter(object):
"""
A router which supports http proxy mode requests
Handles requests of the form: GET http://example.com
The router returns latest capture by default.
However, if Memento protocol support is enabled,
the memento Accept-Datetime header can be used
to select specific capture.
See: http://www.mementoweb.org/guide/rfc/#Pattern1.3
for more details.
"""
BLOCK_SIZE = 4096
DEF_MAGIC_NAME = 'pywb.proxy'
CERT_DL_PEM = '/pywb-ca.pem'
CERT_DL_P12 = '/pywb-ca.p12'
CA_ROOT_FILE = './ca/pywb-ca.pem'
CA_ROOT_NAME = 'pywb https proxy replay CA'
CA_CERTS_DIR = './ca/certs/'
EXTRA_HEADERS = {'cache-control': 'no-cache',
'connection': 'close',
'p3p': 'CP="NOI ADM DEV COM NAV OUR STP"'}
def __init__(self, routes, **kwargs):
self.error_view = kwargs.get('error_view')
proxy_options = kwargs.get('config', {})
if proxy_options:
proxy_options = proxy_options.get('proxy_options', {})
self.magic_name = proxy_options.get('magic_name')
if not self.magic_name:
self.magic_name = self.DEF_MAGIC_NAME
proxy_options['magic_name'] = self.magic_name
self.extra_headers = proxy_options.get('extra_headers')
if not self.extra_headers:
self.extra_headers = self.EXTRA_HEADERS
proxy_options['extra_headers'] = self.extra_headers
res_type = proxy_options.get('cookie_resolver')
if res_type == True or res_type == 'cookie':
self.resolver = CookieResolver(routes, proxy_options)
elif res_type == 'ip':
self.resolver = IPCacheResolver(routes, proxy_options)
else:
self.resolver = ProxyAuthResolver(routes, proxy_options)
self.use_banner = proxy_options.get('use_banner', True)
self.use_wombat = proxy_options.get('use_client_rewrite', True)
self.proxy_cert_dl_view = proxy_options.get('proxy_cert_download_view')
if not proxy_options.get('enable_https_proxy'):
self.ca = None
return
try:
from certauth.certauth import CertificateAuthority
except ImportError: #pragma: no cover
print('HTTPS proxy is not available as the "certauth" module ' +
'is not installed')
print('Please install via "pip install certauth" ' +
'to enable HTTPS support')
self.ca = None
return
# HTTPS Only Options
ca_file = proxy_options.get('root_ca_file', self.CA_ROOT_FILE)
# attempt to create the root_ca_file if doesn't exist
# (generally recommended to create this seperately)
ca_name = proxy_options.get('root_ca_name', self.CA_ROOT_NAME)
certs_dir = proxy_options.get('certs_dir', self.CA_CERTS_DIR)
self.ca = CertificateAuthority(ca_file=ca_file,
certs_dir=certs_dir,
ca_name=ca_name)
self.use_wildcard = proxy_options.get('use_wildcard_certs', True)
def __call__(self, env):
is_https = (env['REQUEST_METHOD'] == 'CONNECT')
ArchivalRouter.ensure_rel_uri_set(env)
# for non-https requests, check non-proxy urls
if not is_https:
url = env['REL_REQUEST_URI']
if not url.startswith(('http://', 'https://')):
return None
env['pywb.proxy_scheme'] = 'http'
route = None
coll = None
matcher = None
response = None
ts = None
# check resolver, for pre connect resolve
if self.resolver.pre_connect:
route, coll, matcher, ts, response = self.resolver.resolve(env)
if response:
return response
# do connect, then get updated url
if is_https:
response = self.handle_connect(env)
if response:
return response
url = env['REL_REQUEST_URI']
else:
parts = urlparse.urlsplit(env['REL_REQUEST_URI'])
hostport = parts.netloc.split(':', 1)
env['pywb.proxy_host'] = hostport[0]
env['pywb.proxy_port'] = hostport[1] if len(hostport) == 2 else ''
env['pywb.proxy_req_uri'] = parts.path
if parts.query:
env['pywb.proxy_req_uri'] += '?' + parts.query
env['pywb.proxy_query'] = parts.query
env['pywb_proxy_magic'] = self.magic_name
# route (static) and other resources to archival replay
if env['pywb.proxy_host'] == self.magic_name:
env['REL_REQUEST_URI'] = env['pywb.proxy_req_uri']
# special case for proxy install
response = self.handle_cert_install(env)
if response:
return response
return None
# check resolver, post connect
if not self.resolver.pre_connect:
route, coll, matcher, ts, response = self.resolver.resolve(env)
if response:
return response
host_prefix = env['pywb.proxy_scheme'] + '://' + self.magic_name
rel_prefix = ''
# special case for proxy calendar
if (env['pywb.proxy_host'] == 'query.' + self.magic_name):
url = env['pywb.proxy_req_uri'][1:]
rel_prefix = '/'
if ts is not None:
url = ts + '/' + url
wbrequest = route.request_class(env,
request_uri=url,
wb_url_str=url,
coll=coll,
host_prefix=host_prefix,
rel_prefix=rel_prefix,
wburl_class=route.handler.get_wburl_type(),
urlrewriter_class=HttpsUrlRewriter,
use_abs_prefix=False,
is_proxy=True)
if matcher:
route.apply_filters(wbrequest, matcher)
# full rewrite and banner
if self.use_wombat and self.use_banner:
wbrequest.wb_url.mod = ''
elif self.use_banner:
# banner only, no rewrite
wbrequest.wb_url.mod = 'bn_'
else:
# unaltered, no rewrite or banner
wbrequest.wb_url.mod = 'id_'
response = route.handler(wbrequest)
if wbrequest.wb_url and wbrequest.wb_url.is_replay():
response.status_headers.replace_headers(self.extra_headers)
return response
def get_request_socket(self, env):
if not self.ca:
return None
sock = None
if env.get('uwsgi.version'): # pragma: no cover
try:
import uwsgi
fd = uwsgi.connection_fd()
conn = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
sock = socket.socket(_sock=conn)
except Exception:
pass
elif env.get('gunicorn.socket'): # pragma: no cover
sock = env['gunicorn.socket']
if not sock:
# attempt to find socket from wsgi.input
input_ = env.get('wsgi.input')
if input_ and hasattr(input_, '_sock'):
sock = socket.socket(_sock=input_._sock)
return sock
def handle_connect(self, env):
sock = self.get_request_socket(env)
if not sock:
return WbResponse.text_response('HTTPS Proxy Not Supported',
'405 HTTPS Proxy Not Supported')
sock.send('HTTP/1.0 200 Connection Established\r\n')
sock.send('Server: pywb proxy\r\n')
sock.send('\r\n')
hostname, port = env['REL_REQUEST_URI'].split(':')
if not self.use_wildcard:
certfile = self.ca.cert_for_host(hostname)
else:
certfile = self.ca.get_wildcard_cert(hostname)
try:
ssl_sock = ssl.wrap_socket(sock,
server_side=True,
certfile=certfile,
#ciphers="ALL",
suppress_ragged_eofs=False,
ssl_version=ssl.PROTOCOL_SSLv23
)
env['pywb.proxy_ssl_sock'] = ssl_sock
buffreader = BufferedReader(ssl_sock, block_size=self.BLOCK_SIZE)
statusline = buffreader.readline().rstrip()
except Exception as se:
raise BadRequestException(se.message)
statusparts = statusline.split(' ')
if len(statusparts) < 3:
raise BadRequestException('Invalid Proxy Request: ' + statusline)
env['REQUEST_METHOD'] = statusparts[0]
env['REL_REQUEST_URI'] = ('https://' +
env['REL_REQUEST_URI'].replace(':443', '') +
statusparts[1])
env['SERVER_PROTOCOL'] = statusparts[2].strip()
env['pywb.proxy_scheme'] = 'https'
env['pywb.proxy_host'] = hostname
env['pywb.proxy_port'] = port
env['pywb.proxy_req_uri'] = statusparts[1]
queryparts = env['REL_REQUEST_URI'].split('?', 1)
env['PATH_INFO'] = queryparts[0]
env['QUERY_STRING'] = queryparts[1] if len(queryparts) > 1 else ''
env['pywb.proxy_query'] = env['QUERY_STRING']
while True:
line = buffreader.readline()
if line:
line = line.rstrip()
if not line:
break
parts = line.split(':', 1)
if len(parts) < 2:
continue
name = parts[0].strip()
value = parts[1].strip()
name = name.replace('-', '_').upper()
if name not in ('CONTENT_LENGTH', 'CONTENT_TYPE'):
name = 'HTTP_' + name
env[name] = value
remain = buffreader.rem_length()
if remain > 0:
remainder = buffreader.read(self.BLOCK_SIZE)
env['wsgi.input'] = BufferedReader(ssl_sock,
block_size=self.BLOCK_SIZE,
starting_data=remainder)
def handle_cert_install(self, env):
if env['pywb.proxy_req_uri'] in ('/', '/index.html', '/index.html'):
available = (self.ca is not None)
if self.proxy_cert_dl_view:
return (self.proxy_cert_dl_view.
render_response(available=available,
pem_path=self.CERT_DL_PEM,
p12_path=self.CERT_DL_P12))
elif env['pywb.proxy_req_uri'] == self.CERT_DL_PEM:
if not self.ca:
return None
buff = ''
with open(self.ca.ca_file, 'rb') as fh:
buff = fh.read()
content_type = 'application/x-x509-ca-cert'
return WbResponse.text_response(buff,
content_type=content_type)
elif env['pywb.proxy_req_uri'] == self.CERT_DL_P12:
if not self.ca:
return None
buff = self.ca.get_root_PKCS12()
content_type = 'application/x-pkcs12'
return WbResponse.text_response(buff,
content_type=content_type)
class ProxyRouter(object):
"""
A router which supports http proxy mode requests
Handles requests of the form: GET http://example.com
The router returns latest capture by default.
However, if Memento protocol support is enabled,
the memento Accept-Datetime header can be used
to select specific capture.
See: http://www.mementoweb.org/guide/rfc/#Pattern1.3
for more details.
"""
BLOCK_SIZE = 4096
DEF_MAGIC_NAME = 'pywb.proxy'
CERT_DL_PEM = '/pywb-ca.pem'
CERT_DL_P12 = '/pywb-ca.p12'
CA_ROOT_FILE = './ca/pywb-ca.pem'
CA_ROOT_NAME = 'pywb https proxy replay CA'
CA_CERTS_DIR = './ca/certs/'
EXTRA_HEADERS = {
'cache-control': 'no-cache',
'p3p': 'CP="NOI ADM DEV COM NAV OUR STP"'
}
def __init__(self, routes, **kwargs):
self.error_view = kwargs.get('error_view')
proxy_options = kwargs.get('config', {})
if proxy_options:
proxy_options = proxy_options.get('proxy_options', {})
self.magic_name = proxy_options.get('magic_name')
if not self.magic_name:
self.magic_name = self.DEF_MAGIC_NAME
proxy_options['magic_name'] = self.magic_name
self.extra_headers = proxy_options.get('extra_headers')
if not self.extra_headers:
self.extra_headers = self.EXTRA_HEADERS
proxy_options['extra_headers'] = self.extra_headers
if proxy_options.get('cookie_resolver', True):
self.resolver = CookieResolver(routes, proxy_options)
else:
self.resolver = ProxyAuthResolver(routes, proxy_options)
self.unaltered = proxy_options.get('unaltered_replay', False)
self.proxy_cert_dl_view = proxy_options.get('proxy_cert_download_view')
if not proxy_options.get('enable_https_proxy'):
self.ca = None
return
try:
from certauth.certauth import CertificateAuthority
except ImportError: #pragma: no cover
print('HTTPS proxy is not available as the "certauth" module ' +
'is not installed')
print('Please install via "pip install certauth" ' +
'to enable HTTPS support')
self.ca = None
return
# HTTPS Only Options
ca_file = proxy_options.get('root_ca_file', self.CA_ROOT_FILE)
# attempt to create the root_ca_file if doesn't exist
# (generally recommended to create this seperately)
ca_name = proxy_options.get('root_ca_name', self.CA_ROOT_NAME)
certs_dir = proxy_options.get('certs_dir', self.CA_CERTS_DIR)
self.ca = CertificateAuthority(ca_file=ca_file,
certs_dir=certs_dir,
ca_name=ca_name)
self.use_wildcard = proxy_options.get('use_wildcard_certs', True)
def __call__(self, env):
is_https = (env['REQUEST_METHOD'] == 'CONNECT')
ArchivalRouter.ensure_rel_uri_set(env)
# for non-https requests, check non-proxy urls
if not is_https:
url = env['REL_REQUEST_URI']
if not url.startswith(('http://', 'https://')):
return None
env['pywb.proxy_scheme'] = 'http'
route = None
coll = None
matcher = None
response = None
ts = None
# check resolver, for pre connect resolve
if self.resolver.pre_connect:
route, coll, matcher, ts, response = self.resolver.resolve(env)
if response:
return response
# do connect, then get updated url
if is_https:
response = self.handle_connect(env)
if response:
return response
url = env['REL_REQUEST_URI']
else:
parts = urlparse.urlsplit(env['REL_REQUEST_URI'])
hostport = parts.netloc.split(':', 1)
env['pywb.proxy_host'] = hostport[0]
env['pywb.proxy_port'] = hostport[1] if len(hostport) == 2 else ''
env['pywb.proxy_req_uri'] = parts.path
if parts.query:
env['pywb.proxy_req_uri'] += '?' + parts.query
env['pywb_proxy_magic'] = self.magic_name
# route (static) and other resources to archival replay
if env['pywb.proxy_host'] == self.magic_name:
env['REL_REQUEST_URI'] = env['pywb.proxy_req_uri']
# special case for proxy install
response = self.handle_cert_install(env)
if response:
return response
return None
# check resolver, post connect
if not self.resolver.pre_connect:
route, coll, matcher, ts, response = self.resolver.resolve(env)
if response:
return response
host_prefix = env['pywb.proxy_scheme'] + '://' + self.magic_name
rel_prefix = ''
# special case for proxy calendar
if (env['pywb.proxy_host'] == 'query.' + self.magic_name):
url = env['pywb.proxy_req_uri'][1:]
rel_prefix = '/'
if ts is not None:
url = ts + '/' + url
wbrequest = route.request_class(
env,
request_uri=url,
wb_url_str=url,
coll=coll,
host_prefix=host_prefix,
rel_prefix=rel_prefix,
wburl_class=route.handler.get_wburl_type(),
urlrewriter_class=HttpsUrlRewriter,
use_abs_prefix=False,
is_proxy=True)
if matcher:
route.apply_filters(wbrequest, matcher)
if self.unaltered:
wbrequest.wb_url.mod = 'id_'
elif is_https:
wbrequest.wb_url.mod = 'bn_'
response = route.handler(wbrequest)
if wbrequest.wb_url and wbrequest.wb_url.is_replay():
response.status_headers.replace_headers(self.extra_headers)
return response
def get_request_socket(self, env):
if not self.ca:
return None
sock = None
if env.get('uwsgi.version'): # pragma: no cover
try:
import uwsgi
fd = uwsgi.connection_fd()
conn = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
sock = socket.socket(_sock=conn)
except Exception:
pass
elif env.get('gunicorn.socket'): # pragma: no cover
sock = env['gunicorn.socket']
if not sock:
# attempt to find socket from wsgi.input
input_ = env.get('wsgi.input')
if input_ and hasattr(input_, '_sock'):
sock = socket.socket(_sock=input_._sock)
return sock
def handle_connect(self, env):
sock = self.get_request_socket(env)
if not sock:
return WbResponse.text_response('HTTPS Proxy Not Supported',
'405 HTTPS Proxy Not Supported')
sock.send('HTTP/1.0 200 Connection Established\r\n')
sock.send('Server: pywb proxy\r\n')
sock.send('\r\n')
hostname, port = env['REL_REQUEST_URI'].split(':')
if not self.use_wildcard:
certfile = self.ca.cert_for_host(hostname)
else:
certfile = self.ca.get_wildcard_cert(hostname)
try:
ssl_sock = ssl.wrap_socket(
sock,
server_side=True,
certfile=certfile,
#ciphers="ALL",
suppress_ragged_eofs=False,
ssl_version=ssl.PROTOCOL_SSLv23)
env['pywb.proxy_ssl_sock'] = ssl_sock
buffreader = BufferedReader(ssl_sock, block_size=self.BLOCK_SIZE)
statusline = buffreader.readline().rstrip()
except Exception as se:
raise BadRequestException(se.message)
statusparts = statusline.split(' ')
if len(statusparts) < 3:
raise BadRequestException('Invalid Proxy Request: ' + statusline)
env['REQUEST_METHOD'] = statusparts[0]
env['REL_REQUEST_URI'] = ('https://' +
env['REL_REQUEST_URI'].replace(':443', '') +
statusparts[1])
env['SERVER_PROTOCOL'] = statusparts[2].strip()
env['pywb.proxy_scheme'] = 'https'
env['pywb.proxy_host'] = hostname
env['pywb.proxy_port'] = port
env['pywb.proxy_req_uri'] = statusparts[1]
queryparts = env['REL_REQUEST_URI'].split('?', 1)
env['PATH_INFO'] = queryparts[0]
env['QUERY_STRING'] = queryparts[1] if len(queryparts) > 1 else ''
while True:
line = buffreader.readline()
if line:
line = line.rstrip()
if not line:
break
parts = line.split(':', 1)
if len(parts) < 2:
continue
name = parts[0].strip()
value = parts[1].strip()
name = name.replace('-', '_').upper()
if name not in ('CONTENT_LENGTH', 'CONTENT_TYPE'):
name = 'HTTP_' + name
env[name] = value
remain = buffreader.rem_length()
if remain > 0:
remainder = buffreader.read(self.BLOCK_SIZE)
env['wsgi.input'] = BufferedReader(ssl_sock,
block_size=self.BLOCK_SIZE,
starting_data=remainder)
def handle_cert_install(self, env):
if env['pywb.proxy_req_uri'] in ('/', '/index.html', '/index.html'):
available = (self.ca is not None)
if self.proxy_cert_dl_view:
return (self.proxy_cert_dl_view.render_response(
available=available,
pem_path=self.CERT_DL_PEM,
p12_path=self.CERT_DL_P12))
elif env['pywb.proxy_req_uri'] == self.CERT_DL_PEM:
if not self.ca:
return None
buff = ''
with open(self.ca.ca_file, 'rb') as fh:
buff = fh.read()
content_type = 'application/x-x509-ca-cert'
return WbResponse.text_response(buff, content_type=content_type)
elif env['pywb.proxy_req_uri'] == self.CERT_DL_P12:
if not self.ca:
return None
buff = self.ca.get_root_PKCS12()
content_type = 'application/x-pkcs12'
return WbResponse.text_response(buff, content_type=content_type)
#!/usr/bin/env python3
from certauth.certauth import main, CertificateAuthority, FileCache, LRUCache, ROOT_CA
ca = CertificateAuthority('My Custom CA', 'cert.pem', cert_cache='/tmp/certs')
filename = ca.cert_for_host('test.server')
print(ca)
print(filename)
def devpipostgresql_postgresql(request):
tmpdir = py.path.local(
tempfile.mkdtemp(prefix='test-', suffix='-devpi-postgresql'))
try:
cap = py.io.StdCaptureFD()
cap.startall()
subprocess.check_call(['initdb', tmpdir.strpath])
cap.reset()
postgresql_conf_lines = [
"fsync = off", "full_page_writes = off",
"synchronous_commit = off",
"unix_socket_directories = '{}'".format(tmpdir.strpath)
]
pg_ssl = request.config.option.backend_postgresql_ssl
host = 'localhost'
if pg_ssl:
# Make certificate authority and server certificate
ca = CertificateAuthority('Test CA',
tmpdir.join('ca.pem').strpath,
cert_cache=tmpdir.strpath)
server_cert = ca.cert_for_host(host)
if not sys.platform.startswith("win"):
# Postgres requires restrictive permissions on private key.
os.chmod(server_cert, 0o600)
postgresql_conf_lines.extend([
"ssl = on", "ssl_cert_file = '{}'".format(server_cert),
"ssl_key_file = '{}'".format(server_cert),
"ssl_ca_file = 'ca.pem'"
])
# Require SSL connections to be authenticated by client certificates.
with tmpdir.join('pg_hba.conf').open('w', encoding='ascii') as f:
f.write("\n".join([
# "local" is for Unix domain socket connections only
'local all all trust',
# IPv4 local connections:
'hostssl all all 127.0.0.1/32 cert',
'host all all 127.0.0.1/32 trust'
]))
with tmpdir.join('postgresql.conf').open('w+', encoding='ascii') as f:
f.write("\n".join(postgresql_conf_lines))
port = get_open_port(host)
cap = py.io.StdCaptureFD()
cap.startall()
p = subprocess.Popen(
['postgres', '-D', tmpdir.strpath, '-h', host, '-p',
str(port)])
wait_for_port(host, port)
cap.reset()
try:
cap = py.io.StdCaptureFD()
cap.startall()
subprocess.check_call(
['createdb', '-h', host, '-p',
str(port), 'devpi'])
cap.reset()
user = getpass.getuser()
settings = dict(host=host, port=port, user=user)
if pg_ssl:
# Make client certificate for user and authenticate with it.
client_cert = ca.cert_for_host(user)
settings['ssl_check_hostname'] = 'yes'
settings['ssl_ca_certs'] = tmpdir.join('ca.pem').strpath
settings['ssl_certfile'] = client_cert
main.Storage(tmpdir,
notify_on_commit=False,
cache_size=10000,
settings=settings)
yield settings
for conn, db, ts in Storage._connections:
try:
conn.close()
except AttributeError:
pass
# use a copy of the set, as it might be changed in another thread
for db in set(Storage._dbs_created):
try:
subprocess.check_call([
'dropdb', '-h', Storage.host, '-p',
str(Storage.port), db
])
except subprocess.CalledProcessError:
pass
finally:
p.terminate()
p.wait()
finally:
tmpdir.remove(ignore_errors=True)
def ca():
return CertificateAuthority("Test CA", TEST_CA_ROOT, TEST_CA_DIR)
class WSGIProxMiddleware(object):
DEFAULT_HOST = 'wsgiprox'
CA_ROOT_NAME = 'wsgiprox https proxy CA'
CA_ROOT_FILE = os.path.join('.', 'ca', 'wsgiprox-ca.pem')
SSL_BASIC_OPTIONS = (SSL.OP_CIPHER_SERVER_PREFERENCE)
SSL_DEFAULT_METHOD = SSL.SSLv23_METHOD
SSL_DEFAULT_OPTIONS = (SSL.OP_NO_TICKET | SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3
| SSL_BASIC_OPTIONS)
CONNECT_RESPONSE_1_1 = b'HTTP/1.1 200 Connection Established\r\n\r\n'
CONNECT_RESPONSE_1_0 = b'HTTP/1.0 200 Connection Established\r\n\r\n'
DEFAULT_MAX_TUNNELS = 50
@classmethod
def set_connection_class(cls):
try:
import gevent.socket
assert (gevent.socket.socket == socket.socket)
from wsgiprox.gevent_ssl import SSLConnection as SSLConnection
cls.is_gevent_ssl = True
except Exception as e: #pragma: no cover
logger.debug(str(e))
from OpenSSL.SSL import Connection as SSLConnection
cls.is_gevent_ssl = False
finally:
cls.SSLConnection = SSLConnection
def __init__(self,
wsgi,
prefix_resolver=None,
download_host=None,
proxy_host=None,
proxy_options=None,
proxy_apps=None):
self._wsgi = wsgi
self.set_connection_class()
if isinstance(prefix_resolver, str):
prefix_resolver = FixedResolver(prefix_resolver)
self.prefix_resolver = prefix_resolver or FixedResolver()
self.proxy_apps = proxy_apps or {}
self.proxy_host = proxy_host or self.DEFAULT_HOST
if self.proxy_host not in self.proxy_apps:
self.proxy_apps[self.proxy_host] = None
# HTTPS Only Options
proxy_options = proxy_options or {}
ca_name = proxy_options.get('ca_name', self.CA_ROOT_NAME)
ca_file_cache = proxy_options.get('ca_file_cache', self.CA_ROOT_FILE)
#self.ca = CertificateAuthority(ca_name=ca_name,
# ca_file_cache=ca_file_cache,
# cert_cache=None,
# cert_not_before=-3600)
self.ca = CertificateAuthority(ca_name=ca_name,
ca_file_cache=ca_file_cache,
cert_cache=50,
cert_not_before=-3600)
self.keepalive_max = proxy_options.get('keepalive_max',
self.DEFAULT_MAX_TUNNELS)
self.keepalive_opts = hasattr(socket, 'TCP_KEEPIDLE')
self._tcp_keepidle = proxy_options.get('tcp_keepidle', 60)
self._tcp_keepintvl = proxy_options.get('tcp_keepintvl', 5)
self._tcp_keepcnt = proxy_options.get('tcp_keepcnt', 3)
self.num_open_tunnels = 0
try:
self.root_ca_file = self.ca.get_root_pem_filename()
except Exception as e:
self.root_ca_file = None
self.use_wildcard = proxy_options.get('use_wildcard_certs', True)
if proxy_options.get('enable_cert_download', True):
download_host = download_host or self.DEFAULT_HOST
self.proxy_apps[download_host] = CertDownloader(self.ca)
self.enable_ws = proxy_options.get('enable_websockets', True)
if WebSocketHandler == object:
self.enable_ws = None
def wsgi(self, env, start_response):
# see if the host matches one of the proxy app hosts
# if so, try to see if there is an wsgi app set
# and if it returns something
hostname = env.get('wsgiprox.matched_proxy_host')
if hostname:
app = self.proxy_apps.get(hostname)
if app:
res = app(env, start_response)
if res is not None:
return res
# call upstream wsgi app
return self._wsgi(env, start_response)
def __call__(self, env, start_response):
if env['REQUEST_METHOD'] == 'CONNECT':
return self.handle_connect(env, start_response)
else:
self.ensure_request_uri(env)
if env['REQUEST_URI'].startswith('http://'):
return self.handle_http_proxy(env, start_response)
else:
return self.wsgi(env, start_response)
def handle_http_proxy(self, env, start_response):
res = self.require_auth(env, start_response)
if res is not None:
return res
handler = HttpProxyHandler(start_response, self.wsgi, self.resolve)
return handler(env)
def handle_connect(self, env, start_response):
raw_sock = self.get_raw_socket(env)
if not raw_sock:
start_response('405 HTTPS Proxy Not Supported',
[('Content-Length', '0')])
return []
res = self.require_auth(env, start_response)
if res is not None:
return res
connect_handler = None
curr_sock = None
try:
scheme, curr_sock = self.wrap_socket(env, raw_sock)
connect_handler = ConnectHandler(curr_sock, scheme, self.wsgi,
self.resolve)
self.num_open_tunnels += 1
connect_handler(env, self.enable_ws)
while self.keep_alive(connect_handler):
connect_handler(env, self.enable_ws)
except Exception as e:
logger.debug(str(e))
start_response('500 Unexpected Error', [('Content-Length', '0')])
finally:
if connect_handler:
self.num_open_tunnels -= 1
connect_handler.close()
if curr_sock and curr_sock != raw_sock:
# this seems to necessary to avoid tls data read later
# in the same gevent
try:
if self.is_gevent_ssl:
curr_sock.recv(0)
curr_sock.shutdown()
except:
pass
finally:
curr_sock.close()
start_response('200 OK', [])
return []
def keep_alive(self, connect_handler):
# keepalive disabled
if self.keepalive_max < 0:
return False
if not connect_handler.is_keepalive:
return False
# no max
if self.keepalive_max == 0:
return True
return (self.num_open_tunnels <= self.keepalive_max)
def _new_context(self):
context = SSL.Context(self.SSL_DEFAULT_METHOD)
context.set_options(self.SSL_DEFAULT_OPTIONS)
context.set_session_cache_mode(SSL.SESS_CACHE_OFF)
return context
def create_ssl_context(self, hostname):
cert, key = self.ca.load_cert(hostname,
wildcard=self.use_wildcard,
wildcard_use_parent=True)
context = self._new_context()
context.use_privatekey(key)
context.use_certificate(cert)
return context
def _get_connect_response(self, env):
if env.get('SERVER_PROTOCOL', 'HTTP/1.0') == 'HTTP/1.1':
return self.CONNECT_RESPONSE_1_1
else:
return self.CONNECT_RESPONSE_1_0
def wrap_socket(self, env, sock):
if self.keepalive_max >= 0:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
if self.keepalive_opts: #pragma: no cover
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE,
self._tcp_keepidle)
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL,
self._tcp_keepintvl)
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT,
self._tcp_keepcnt)
host_port = env['PATH_INFO']
hostname, port = host_port.split(':', 1)
env['wsgiprox.connect_host'] = hostname
sock.sendall(self._get_connect_response(env))
if port == '80':
return 'http', sock
if port != '443':
env['wsgiprox.connect_port'] = port
peek_buff = sock.recv(16, socket.MSG_PEEK)
# http websocket traffic would start with a GET
if peek_buff.startswith(b'GET '):
return 'http', sock
def sni_callback(connection):
sni_hostname = connection.get_servername()
# curl -k (unverified) mode results in empty hostname here
# requests unverified mode still includes an sni hostname
if not sni_hostname:
return
if six.PY3:
sni_hostname = sni_hostname.decode('iso-8859-1')
# if same host as CONNECT header, then just keep current context
if sni_hostname == hostname:
return
connection.set_context(self.create_ssl_context(sni_hostname))
env['wsgiprox.connect_host'] = sni_hostname
context = self.create_ssl_context(hostname)
context.set_tlsext_servername_callback(sni_callback)
ssl_sock = self.SSLConnection(context, sock)
ssl_sock.set_accept_state()
ssl_sock.do_handshake()
return 'https', ssl_sock
def require_auth(self, env, start_response):
if not hasattr(self.prefix_resolver, 'require_auth'):
return
auth_req = self.prefix_resolver.require_auth(env)
if not auth_req:
return
auth_req = 'Basic realm="{0}"'.format(auth_req)
headers = [('Proxy-Authenticate', auth_req),
('Proxy-Connection', 'close'), ('Content-Length', '0')]
start_response('407 Proxy Authentication', headers)
return []
def resolve(self, url, env, hostname):
if hostname in self.proxy_apps.keys():
parts = urlsplit(url)
full = parts.path
if parts.query:
full += '?' + parts.query
env['REQUEST_URI'] = full
env['wsgiprox.matched_proxy_host'] = hostname
env['wsgiprox.proxy_host'] = hostname
else:
env['REQUEST_URI'] = self.prefix_resolver(url, env)
env['wsgiprox.proxy_host'] = self.proxy_host
queryparts = env['REQUEST_URI'].split('?', 1)
env['PATH_INFO'] = queryparts[0]
env['QUERY_STRING'] = queryparts[1] if len(queryparts) > 1 else ''
def ensure_request_uri(self, env):
if 'REQUEST_URI' in env:
return
full_uri = env['PATH_INFO']
if env.get('QUERY_STRING'):
full_uri += '?' + env['QUERY_STRING']
env['REQUEST_URI'] = full_uri
@classmethod
def get_raw_socket(cls, env): #pragma: no cover
sock = None
if env.get('uwsgi.version'):
try:
import uwsgi
fd = uwsgi.connection_fd()
sock = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
except Exception as e:
pass
elif env.get('gunicorn.socket'):
sock = env['gunicorn.socket']
if not sock:
# attempt to find socket from wsgi.input
input_ = env.get('wsgi.input')
if input_:
if hasattr(input_, '_sock'):
raw = input_._sock
sock = socket.socket(_sock=raw)
elif hasattr(input_, 'raw'):
sock = input_.raw._sock
elif hasattr(input_, 'rfile'):
# PY3
if hasattr(input_.rfile, 'raw'):
sock = input_.rfile.raw._sock
# PY2
else:
sock = input_.rfile._sock
return sock
class ProxyRouter(object):
"""
A router which supports http proxy mode requests
Handles requests of the form: GET http://example.com
The router returns latest capture by default.
However, if Memento protocol support is enabled,
the memento Accept-Datetime header can be used
to select specific capture.
See: http://www.mementoweb.org/guide/rfc/#Pattern1.3
for more details.
"""
BLOCK_SIZE = 4096
DEF_MAGIC_NAME = 'pywb.proxy'
BUFF_RESPONSE_MEM_SIZE = 1024 * 1024
CERT_DL_PEM = '/pywb-ca.pem'
CERT_DL_P12 = '/pywb-ca.p12'
CA_ROOT_FILE = './ca/pywb-ca.pem'
CA_ROOT_NAME = 'pywb https proxy replay CA'
CA_CERTS_DIR = './ca/certs/'
EXTRA_HEADERS = {
'cache-control': 'no-cache',
'connection': 'close',
'p3p': 'CP="NOI ADM DEV COM NAV OUR STP"'
}
def __init__(self, routes, **kwargs):
self.error_view = kwargs.get('error_view')
proxy_options = kwargs.get('config', {})
if proxy_options:
proxy_options = proxy_options.get('proxy_options', {})
self.magic_name = proxy_options.get('magic_name')
if not self.magic_name:
self.magic_name = self.DEF_MAGIC_NAME
proxy_options['magic_name'] = self.magic_name
self.extra_headers = proxy_options.get('extra_headers')
if not self.extra_headers:
self.extra_headers = self.EXTRA_HEADERS
proxy_options['extra_headers'] = self.extra_headers
res_type = proxy_options.get('cookie_resolver', True)
if res_type == 'auth' or not res_type:
self.resolver = ProxyAuthResolver(routes, proxy_options)
elif res_type == 'ip':
self.resolver = IPCacheResolver(routes, proxy_options)
#elif res_type == True or res_type == 'cookie':
# self.resolver = CookieResolver(routes, proxy_options)
else:
self.resolver = CookieResolver(routes, proxy_options)
self.use_banner = proxy_options.get('use_banner', True)
self.use_wombat = proxy_options.get('use_client_rewrite', True)
self.proxy_cert_dl_view = proxy_options.get('proxy_cert_download_view')
if not proxy_options.get('enable_https_proxy'):
self.ca = None
return
try:
from certauth.certauth import CertificateAuthority
except ImportError: #pragma: no cover
print('HTTPS proxy is not available as the "certauth" module ' +
'is not installed')
print('Please install via "pip install certauth" ' +
'to enable HTTPS support')
self.ca = None
return
# HTTPS Only Options
ca_file = proxy_options.get('root_ca_file', self.CA_ROOT_FILE)
# attempt to create the root_ca_file if doesn't exist
# (generally recommended to create this seperately)
ca_name = proxy_options.get('root_ca_name', self.CA_ROOT_NAME)
certs_dir = proxy_options.get('certs_dir', self.CA_CERTS_DIR)
self.ca = CertificateAuthority(ca_file=ca_file,
certs_dir=certs_dir,
ca_name=ca_name)
self.use_wildcard = proxy_options.get('use_wildcard_certs', True)
def __call__(self, env):
is_https = (env['REQUEST_METHOD'] == 'CONNECT')
ArchivalRouter.ensure_rel_uri_set(env)
# for non-https requests, check non-proxy urls
if not is_https:
url = env['REL_REQUEST_URI']
if not url.startswith(('http://', 'https://')):
return None
env['pywb.proxy_scheme'] = 'http'
route = None
coll = None
matcher = None
response = None
ts = None
# check resolver, for pre connect resolve
if self.resolver.pre_connect:
route, coll, matcher, ts, response = self.resolver.resolve(env)
if response:
return response
# do connect, then get updated url
if is_https:
response = self.handle_connect(env)
if response:
return response
url = env['REL_REQUEST_URI']
else:
parts = urlsplit(env['REL_REQUEST_URI'])
hostport = parts.netloc.split(':', 1)
env['pywb.proxy_host'] = hostport[0]
env['pywb.proxy_port'] = hostport[1] if len(hostport) == 2 else ''
env['pywb.proxy_req_uri'] = parts.path
if parts.query:
env['pywb.proxy_req_uri'] += '?' + parts.query
env['pywb.proxy_query'] = parts.query
if self.resolver.supports_switching:
env['pywb_proxy_magic'] = self.magic_name
# route (static) and other resources to archival replay
if env['pywb.proxy_host'] == self.magic_name:
env['REL_REQUEST_URI'] = env['pywb.proxy_req_uri']
# special case for proxy install
response = self.handle_cert_install(env)
if response:
return response
return None
# check resolver, post connect
if not self.resolver.pre_connect:
route, coll, matcher, ts, response = self.resolver.resolve(env)
if response:
return response
rel_prefix = ''
custom_prefix = env.get('HTTP_PYWB_REWRITE_PREFIX', '')
if custom_prefix:
host_prefix = custom_prefix
urlrewriter_class = UrlRewriter
abs_prefix = True
# always rewrite to absolute here
rewrite_opts = dict(no_match_rel=True)
else:
host_prefix = env['pywb.proxy_scheme'] + '://' + self.magic_name
urlrewriter_class = SchemeOnlyUrlRewriter
abs_prefix = False
rewrite_opts = {}
# special case for proxy calendar
if (env['pywb.proxy_host'] == 'query.' + self.magic_name):
url = env['pywb.proxy_req_uri'][1:]
rel_prefix = '/'
if ts is not None:
url = ts + '/' + url
wbrequest = route.request_class(
env,
request_uri=url,
wb_url_str=url,
coll=coll,
host_prefix=host_prefix,
rel_prefix=rel_prefix,
wburl_class=route.handler.get_wburl_type(),
urlrewriter_class=urlrewriter_class,
use_abs_prefix=abs_prefix,
rewrite_opts=rewrite_opts,
is_proxy=True)
if matcher:
route.apply_filters(wbrequest, matcher)
# full rewrite and banner
if self.use_wombat and self.use_banner:
wbrequest.wb_url.mod = ''
elif self.use_banner:
# banner only, no rewrite
wbrequest.wb_url.mod = 'bn_'
else:
# unaltered, no rewrite or banner
wbrequest.wb_url.mod = 'uo_'
response = route.handler(wbrequest)
if not response:
return None
# add extra headers for replay responses
if wbrequest.wb_url and wbrequest.wb_url.is_replay():
response.status_headers.replace_headers(self.extra_headers)
# check for content-length
res = response.status_headers.get_header('content-length')
try:
if int(res) > 0:
return response
except:
pass
# need to either chunk or buffer to get content-length
if env.get('SERVER_PROTOCOL') == 'HTTP/1.1':
response.status_headers.remove_header('content-length')
response.status_headers.headers.append(
('Transfer-Encoding', 'chunked'))
response.body = self._chunk_encode(response.body)
else:
response.body = self._buffer_response(response.status_headers,
response.body)
return response
@staticmethod
def _chunk_encode(orig_iter):
for chunk in orig_iter:
if not len(chunk):
continue
chunk_len = b'%X\r\n' % len(chunk)
yield chunk_len
yield chunk
yield b'\r\n'
yield b'0\r\n\r\n'
@staticmethod
def _buffer_response(status_headers, iterator):
out = SpooledTemporaryFile(ProxyRouter.BUFF_RESPONSE_MEM_SIZE)
size = 0
for buff in iterator:
size += len(buff)
out.write(buff)
content_length_str = str(size)
# remove existing content length
status_headers.replace_header('Content-Length', content_length_str)
out.seek(0)
return RewriteContent.stream_to_gen(out)
def get_request_socket(self, env):
if not self.ca:
return None
sock = None
if env.get('uwsgi.version'): # pragma: no cover
try:
import uwsgi
fd = uwsgi.connection_fd()
conn = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
try:
sock = socket.socket(_sock=conn)
except:
sock = conn
except Exception as e:
pass
elif env.get('gunicorn.socket'): # pragma: no cover
sock = env['gunicorn.socket']
if not sock:
# attempt to find socket from wsgi.input
input_ = env.get('wsgi.input')
if input_:
if hasattr(input_, '_sock'): # pragma: no cover
raw = input_._sock
sock = socket.socket(_sock=raw) # pragma: no cover
elif hasattr(input_, 'raw'):
sock = input_.raw._sock
return sock
def handle_connect(self, env):
sock = self.get_request_socket(env)
if not sock:
return WbResponse.text_response('HTTPS Proxy Not Supported',
'405 HTTPS Proxy Not Supported')
sock.send(b'HTTP/1.0 200 Connection Established\r\n')
sock.send(b'Proxy-Connection: close\r\n')
sock.send(b'Server: pywb proxy\r\n')
sock.send(b'\r\n')
hostname, port = env['REL_REQUEST_URI'].split(':')
if not self.use_wildcard:
certfile = self.ca.cert_for_host(hostname)
else:
certfile = self.ca.get_wildcard_cert(hostname)
try:
ssl_sock = ssl.wrap_socket(
sock,
server_side=True,
certfile=certfile,
#ciphers="ALL",
suppress_ragged_eofs=False,
ssl_version=ssl.PROTOCOL_SSLv23)
env['pywb.proxy_ssl_sock'] = ssl_sock
buffreader = BufferedReader(ssl_sock, block_size=self.BLOCK_SIZE)
statusline = to_native_str(buffreader.readline().rstrip())
except Exception as se:
raise BadRequestException(se.message)
statusparts = statusline.split(' ')
if len(statusparts) < 3:
raise BadRequestException('Invalid Proxy Request: ' + statusline)
env['REQUEST_METHOD'] = statusparts[0]
env['REL_REQUEST_URI'] = ('https://' +
env['REL_REQUEST_URI'].replace(':443', '') +
statusparts[1])
env['SERVER_PROTOCOL'] = statusparts[2].strip()
env['pywb.proxy_scheme'] = 'https'
env['pywb.proxy_host'] = hostname
env['pywb.proxy_port'] = port
env['pywb.proxy_req_uri'] = statusparts[1]
queryparts = env['REL_REQUEST_URI'].split('?', 1)
env['PATH_INFO'] = queryparts[0]
env['QUERY_STRING'] = queryparts[1] if len(queryparts) > 1 else ''
env['pywb.proxy_query'] = env['QUERY_STRING']
while True:
line = to_native_str(buffreader.readline())
if line:
line = line.rstrip()
if not line:
break
parts = line.split(':', 1)
if len(parts) < 2:
continue
name = parts[0].strip()
value = parts[1].strip()
name = name.replace('-', '_').upper()
if name not in ('CONTENT_LENGTH', 'CONTENT_TYPE'):
name = 'HTTP_' + name
env[name] = value
env['wsgi.input'] = buffreader
#remain = buffreader.rem_length()
#if remain > 0:
#remainder = buffreader.read()
#env['wsgi.input'] = BufferedReader(BytesIO(remainder))
#remainder = buffreader.read(self.BLOCK_SIZE)
#env['wsgi.input'] = BufferedReader(ssl_sock,
# block_size=self.BLOCK_SIZE,
# starting_data=remainder)
def handle_cert_install(self, env):
if env['pywb.proxy_req_uri'] in ('/', '/index.html', '/index.html'):
available = (self.ca is not None)
if self.proxy_cert_dl_view:
return (self.proxy_cert_dl_view.render_response(
available=available,
pem_path=self.CERT_DL_PEM,
p12_path=self.CERT_DL_P12))
elif env['pywb.proxy_req_uri'] == self.CERT_DL_PEM:
if not self.ca:
return None
buff = b''
with open(self.ca.ca_file, 'rb') as fh:
buff = fh.read()
content_type = 'application/x-x509-ca-cert'
headers = [('Content-Length', str(len(buff)))]
return WbResponse.bin_stream([buff],
content_type=content_type,
headers=headers)
elif env['pywb.proxy_req_uri'] == self.CERT_DL_P12:
if not self.ca:
return None
buff = self.ca.get_root_PKCS12()
content_type = 'application/x-pkcs12'
headers = [('Content-Length', str(len(buff)))]
return WbResponse.bin_stream([buff],
content_type=content_type,
headers=headers)
def __generate_certificate(self):
ca = CertificateAuthority('Python digikey-api CA',
str(self._ca_cert),
cert_cache=str(self._storage_path))
return ca.cert_for_host('localhost')
