def test_custom_not_before_not_after(): ca = CertificateAuthority( "Test Custom CA", TEST_CA_ROOT, cert_not_before=-60 * 60, cert_not_after=60 * 60 * 24 * 3, ) # check PKCS12 buff_pk12 = ca.get_root_PKCS12() assert len(buff_pk12) > 0 cert = crypto.load_pkcs12(buff_pk12).get_certificate() expected_not_before = datetime.datetime.utcnow() - datetime.timedelta( seconds=60 * 60) expected_not_after = datetime.datetime.utcnow() + datetime.timedelta( seconds=60 * 60 * 24 * 3) actual_not_before = datetime.datetime.strptime( cert.get_notBefore().decode("ascii"), "%Y%m%d%H%M%SZ") actual_not_after = datetime.datetime.strptime( cert.get_notAfter().decode("ascii"), "%Y%m%d%H%M%SZ") time.mktime(expected_not_before.utctimetuple()) assert (abs((time.mktime(actual_not_before.utctimetuple()) - time.mktime(expected_not_before.utctimetuple()))) < 10) assert (abs((time.mktime(actual_not_after.utctimetuple()) - time.mktime(expected_not_after.utctimetuple()))) < 10)
def test_explicit_wildcard(): ca = CertificateAuthority(TEST_CA_ROOT, TEST_CA_DIR, 'Test CA') filename = ca.get_wildcard_cert('test.example.proxy') certfile = os.path.join(TEST_CA_DIR, 'example.proxy.pem') assert filename == certfile assert os.path.isfile(certfile) os.remove(certfile)
def do_CONNECT(self): hostname = self.path.split(':')[0] from certauth.certauth import CertificateAuthority ca = CertificateAuthority('My Custom CA', 'ca/certs/ca.pem', cert_cache='tmp/certs') filename = ca.cert_for_host(hostname) self.wfile.write("{0} {1} {2}\r\n\r\n".format( 'HTTP/1.1', 200, 'Connection Established').encode()) self.wfile.flush() self.connection = ssl.wrap_socket(self.connection, keyfile=filename, certfile=filename, server_side=True) self.rfile = self.connection.makefile("rb", self.rbufsize) self.wfile = self.connection.makefile("wb", self.wbufsize) conntype = self.headers.get('Proxy-Connection', '') if conntype.lower() != 'close': self.close_connection = 0 else: self.close_connection = 1
def create_or_update(self, refresh=False): if not self.cache.get("!!root_ca"): self.fail( f"certificate for root ca '{self.ca_name}' does not exist.") return ca = CertificateAuthority( self.ca_name, ca_file_cache=self.cache, cert_cache=self.cache, overwrite=False, ) cert, pkey, entry = ca.load_cert( self.hostname, overwrite=refresh, wildcard=self.wildcard, wildcard_use_parent=False, include_cache_key=True, ) public_cert_pem = crypto.dump_certificate(FILETYPE_PEM, cert) self.set_attribute("CAName", self.ca_name) self.set_attribute("Hostname", self.hostname) self.set_attribute("Hash", hashlib.md5(public_cert_pem).hexdigest()) self.set_attribute("PublicCertPEM", public_cert_pem.decode("ascii")) self.physical_resource_id = self.cache.parameter_name(self.hostname)
def __init__(self, routes, **kwargs): self.error_view = kwargs.get('error_view') proxy_options = kwargs.get('config', {}) if proxy_options: proxy_options = proxy_options.get('proxy_options', {}) self.magic_name = proxy_options.get('magic_name') if not self.magic_name: self.magic_name = self.DEF_MAGIC_NAME proxy_options['magic_name'] = self.magic_name self.extra_headers = proxy_options.get('extra_headers') if not self.extra_headers: self.extra_headers = self.EXTRA_HEADERS proxy_options['extra_headers'] = self.extra_headers res_type = proxy_options.get('cookie_resolver', True) if res_type == 'auth' or not res_type: self.resolver = ProxyAuthResolver(routes, proxy_options) elif res_type == 'ip': self.resolver = IPCacheResolver(routes, proxy_options) #elif res_type == True or res_type == 'cookie': # self.resolver = CookieResolver(routes, proxy_options) else: self.resolver = CookieResolver(routes, proxy_options) self.use_banner = proxy_options.get('use_banner', True) self.use_wombat = proxy_options.get('use_client_rewrite', True) self.proxy_cert_dl_view = proxy_options.get('proxy_cert_download_view') if not proxy_options.get('enable_https_proxy'): self.ca = None return try: from certauth.certauth import CertificateAuthority except ImportError: #pragma: no cover print('HTTPS proxy is not available as the "certauth" module ' + 'is not installed') print('Please install via "pip install certauth" ' + 'to enable HTTPS support') self.ca = None return # HTTPS Only Options ca_file = proxy_options.get('root_ca_file', self.CA_ROOT_FILE) # attempt to create the root_ca_file if doesn't exist # (generally recommended to create this seperately) ca_name = proxy_options.get('root_ca_name', self.CA_ROOT_NAME) certs_dir = proxy_options.get('certs_dir', self.CA_CERTS_DIR) self.ca = CertificateAuthority(ca_file=ca_file, certs_dir=certs_dir, ca_name=ca_name) self.use_wildcard = proxy_options.get('use_wildcard_certs', True)
def __init__(self, wsgi, prefix_resolver=None, download_host=None, proxy_host=None, proxy_options=None, proxy_apps=None): self._wsgi = wsgi self.set_connection_class() if isinstance(prefix_resolver, str): prefix_resolver = FixedResolver(prefix_resolver) self.prefix_resolver = prefix_resolver or FixedResolver() self.proxy_apps = proxy_apps or {} self.proxy_host = proxy_host or self.DEFAULT_HOST if self.proxy_host not in self.proxy_apps: self.proxy_apps[self.proxy_host] = None # HTTPS Only Options proxy_options = proxy_options or {} ca_name = proxy_options.get('ca_name', self.CA_ROOT_NAME) ca_file_cache = proxy_options.get('ca_file_cache', self.CA_ROOT_FILE) self.ca = CertificateAuthority(ca_name=ca_name, ca_file_cache=ca_file_cache, cert_cache=None, cert_not_before=-3600) self.keepalive_max = proxy_options.get('keepalive_max', self.DEFAULT_MAX_TUNNELS) self.keepalive_opts = hasattr(socket, 'TCP_KEEPIDLE') self._tcp_keepidle = proxy_options.get('tcp_keepidle', 60) self._tcp_keepintvl = proxy_options.get('tcp_keepintvl', 5) self._tcp_keepcnt = proxy_options.get('tcp_keepcnt', 3) self.num_open_tunnels = 0 try: self.root_ca_file = self.ca.get_root_pem_filename() except Exception as e: self.root_ca_file = None self.use_wildcard = proxy_options.get('use_wildcard_certs', True) if proxy_options.get('enable_cert_download', True): download_host = download_host or self.DEFAULT_HOST self.proxy_apps[download_host] = CertDownloader(self.ca) self.enable_ws = proxy_options.get('enable_websockets', True) if WebSocketHandler == object: self.enable_ws = None
def test_create_root_subdir(): # create a new cert in a subdirectory subdir = os.path.join(TEST_CA_DIR, 'subdir') ca = CertificateAuthority(TEST_CA_ROOT, subdir, 'Test CA') assert os.path.isdir(subdir) buff = ca.get_root_PKCS12() assert len(buff) > 0
def test_in_mem_parent_wildcard_cert_2(): cert_cache = {} ca = CertificateAuthority('Test CA', TEST_CA_ROOT, cert_cache) cert, key = ca.load_cert('test.example.org.uk', wildcard=True, wildcard_use_parent=True) assert 'example.org.uk' in cert_cache, cert_cache.keys() cached_value = cert_cache['example.org.uk'] cert2, key2 = ca.load_cert('example.org.uk') # assert underlying cache unchanged assert cached_value == cert_cache['example.org.uk'] verify_cert_san(cert2, ['DNS:example.org.uk', 'DNS:*.example.org.uk'])
def test_in_mem_parent_wildcard_cert(): cert_cache = {} ca = CertificateAuthority("Test CA", TEST_CA_ROOT, cert_cache) cert, key = ca.load_cert("test.example.proxy", wildcard=True, wildcard_use_parent=True) assert "example.proxy" in cert_cache, cert_cache.keys() cached_value = cert_cache["example.proxy"] cert2, key2 = ca.load_cert("example.proxy") # assert underlying cache unchanged assert cached_value == cert_cache["example.proxy"] verify_cert_san(cert2, "DNS:example.proxy, DNS:*.example.proxy")
def test_create_root_subdir(): # create a new cert in a subdirectory subdir = os.path.join(TEST_CA_DIR, "subdir") ca_file = os.path.join(subdir, "certauth_test_ca.pem") certs_dir = os.path.join(subdir, "certs") ca = CertificateAuthority("Test CA", ca_file, certs_dir) assert os.path.isdir(subdir) assert os.path.isfile(ca_file) assert os.path.isdir(certs_dir) assert ca.get_root_pem_filename() == ca_file
def test_in_mem_cert(): cert_cache = {} ca = CertificateAuthority('Test CA', TEST_CA_ROOT, cert_cache, overwrite=True) res = ca.load_cert('test.example.proxy') assert 'test.example.proxy' in cert_cache, cert_cache.keys() cached_value = cert_cache['test.example.proxy'] cert, key = ca.load_cert('test.example.proxy') verify_cert_san(cert, ['DNS:test.example.proxy']) # assert underlying cache unchanged assert cached_value == cert_cache['test.example.proxy']
def test_in_mem_cert(): cert_cache = {} ca = CertificateAuthority("Test CA", TEST_CA_ROOT, cert_cache, overwrite=True) res = ca.load_cert("test.example.proxy") assert "test.example.proxy" in cert_cache, cert_cache.keys() cached_value = cert_cache["test.example.proxy"] cert, key = ca.load_cert("test.example.proxy") verify_cert_san(cert, "DNS:test.example.proxy") # assert underlying cache unchanged assert cached_value == cert_cache["test.example.proxy"]
def __init__(self, server_address=('localhost', 8000), req_handler_class=WarcProxyHandler, bind_and_activate=True, ca=None, recorded_url_q=None, digest_algorithm='sha1', method_filter=False): http_server.HTTPServer.__init__(self, server_address, req_handler_class, bind_and_activate) self.digest_algorithm = digest_algorithm if ca is not None: self.ca = ca else: ca_name = 'Warcprox CA on {}'.format(socket.gethostname())[:64] self.ca = CertificateAuthority(ca_file='warcprox-ca.pem', certs_dir='./warcprox-ca', ca_name=ca_name) if recorded_url_q is not None: self.recorded_url_q = recorded_url_q else: self.recorded_url_q = queue.Queue() self.method_filter = method_filter
def test_ca_cert_in_mem(): root_cert_dict = {} ca = CertificateAuthority("Test CA", root_cert_dict, 10) # check PEM buff_pem = ca.get_root_pem() assert len(buff_pem) > 0 # PEM stored in root_cert_dict assert root_cert_dict[ROOT_CA] == buff_pem cert_pem = crypto.load_certificate(FILETYPE_PEM, buff_pem) # check PKCS12 buff_pk12 = ca.get_root_PKCS12() assert len(buff_pk12) > 0 cert_pk12 = crypto.load_pkcs12(buff_pk12).get_certificate()
def generate_certs(host, w): """Generates private.key, cert.crt, and ca-bundle.crt...""" WORKDIR = "/".join(__file__.split("/")[:-1]) with open(WORKDIR + "/config.json", "r") as config_file: CONFIG = json.load(config_file) path = "." ca = CertificateAuthority(CONFIG["CERT_AUTH_NAME"], CONFIG["CERT_AUTH_ROOT_FILE"], cert_cache=path) cert, key = ca.load_cert(host, wildcard=w) ## Extract root CA and host private keys and certificates from pem files ## with open(CONFIG["CERT_AUTH_ROOT_FILE"], "r") as py_ca_file: py_ca_data = "".join(py_ca_file.readlines()) ca_private_key, ca_root_cert = split_certs(py_ca_data) with open(path + "/" + host + ".pem", "r") as host_file: host_data = "".join(host_file.readlines()) host_private_key, host_cert = split_certs(host_data) ## Output certificates to host folder ## with open(path + "/ca.crt", "w+") as py_ca_root_cert_file: py_ca_root_cert_file.write(ca_root_cert) print(f"Generating " + path + "/ca_bundle.crt") with open(path + "/private.key", "w+") as host_private_key_file: host_private_key_file.write(host_private_key) print(f"Generating " + path + "/private.key") with open(path + "/cert.crt", "w+") as host_cert_file: host_cert_file.write(host_cert) print(f"Generating " + path + "/cert.crt") with open(path + "/domain-cert.crt", "w+") as concat_cert_file: concat_cert_file.write(ca_root_cert + host_cert) print(f"Generating " + path + "/concat.crt") print("Finished!")
def test_ca_lru_cache(): lru = LRUCache(max_size=2) ca = CertificateAuthority("Test CA LRU Cache", TEST_CA_ROOT, lru) res = ca.load_cert("example.com") assert "example.com" in lru assert len(lru) == 1 res = ca.load_cert("ABC.example.com") assert "ABC.example.com" in lru assert "example.com" in lru assert len(lru) == 2 res = ca.load_cert("XYZ.example.com", include_cache_key=True) assert res[2] == "XYZ.example.com" assert "XYZ.example.com" in lru assert "ABC.example.com" in lru assert len(lru) == 2 assert "example.com" not in lru
def test_ca_lru_cache(): lru = LRUCache(max_size=2) ca = CertificateAuthority('Test CA LRU Cache', TEST_CA_ROOT, lru) res = ca.load_cert('example.com') assert 'example.com' in lru assert len(lru) == 1 res = ca.load_cert('ABC.example.com') assert 'ABC.example.com' in lru assert 'example.com' in lru assert len(lru) == 2 res = ca.load_cert('XYZ.example.com', include_cache_key=True) assert res[2] == 'XYZ.example.com' assert 'XYZ.example.com' in lru assert 'ABC.example.com' in lru assert len(lru) == 2 assert 'example.com' not in lru
def create_keys(temp_dir, common_name): """Create wildcard certificate for the proxy base url.""" ca = CertificateAuthority( f'Riptide Proxy CA for {socket.gethostname()}', get_ca_path(), cert_cache=temp_dir, # Make it valid for 364 days, as per MacOS Catalina requirements cert_not_after=NOT_VALID_AFTER) if ca.ca_cert.has_expired(): logger.warning("The CA certificate had to be re-generated because" "it was no longer valid. You may need to re-import it," "please see the documentation.") ca = CertificateAuthority( f'Riptide Proxy CA for {socket.gethostname()}', get_ca_path(), cert_cache=temp_dir, cert_not_after=NOT_VALID_AFTER, overwrite=True) return ca.get_wildcard_cert('*.' + common_name)
def __init__(self, stats_db=None, status_callback=None, options=warcprox.Options()): self.status_callback = status_callback self.stats_db = stats_db self.options = options self.remote_connection_pool = PoolManager( num_pools=max(round(options.max_threads / 6), 200) if options.max_threads else 200) server_address = (options.address or 'localhost', options.port if options.port is not None else 8000) if options.onion_tor_socks_proxy: try: host, port = options.onion_tor_socks_proxy.split(':') WarcProxyHandler.onion_tor_socks_proxy_host = host WarcProxyHandler.onion_tor_socks_proxy_port = int(port) except ValueError: WarcProxyHandler.onion_tor_socks_proxy_host = options.onion_tor_socks_proxy WarcProxyHandler.onion_tor_socks_proxy_port = None if options.socket_timeout: WarcProxyHandler._socket_timeout = options.socket_timeout if options.max_resource_size: WarcProxyHandler._max_resource_size = options.max_resource_size if options.tmp_file_max_memory_size: WarcProxyHandler._tmp_file_max_memory_size = options.tmp_file_max_memory_size http_server.HTTPServer.__init__(self, server_address, WarcProxyHandler, bind_and_activate=True) self.digest_algorithm = options.digest_algorithm or 'sha1' ca_name = ('Warcprox CA on %s' % socket.gethostname())[:64] self.ca = CertificateAuthority(ca_file=options.cacert or 'warcprox-ca.pem', certs_dir=options.certs_dir or './warcprox-ca', ca_name=ca_name) self.recorded_url_q = warcprox.TimestampedQueue( maxsize=options.queue_size or 1000) self.running_stats = warcprox.stats.RunningStats()
def __init__(self, MitmProxyHandlerClass=MitmProxyHandler, options=warcprox.Options()): self.options = options # TTLCache is not thread-safe. Access to the shared cache from multiple # threads must be properly synchronized with an RLock according to ref: # https://cachetools.readthedocs.io/en/latest/ self.bad_hostnames_ports = TTLCache(maxsize=1024, ttl=60) self.bad_hostnames_ports_lock = RLock() self.remote_connection_pool = PoolManager(num_pools=max( (options.max_threads or 0) // 6, 400), maxsize=6) if options.onion_tor_socks_proxy: try: host, port = options.onion_tor_socks_proxy.split(':') MitmProxyHandlerClass.onion_tor_socks_proxy_host = host MitmProxyHandlerClass.onion_tor_socks_proxy_port = int(port) except ValueError: MitmProxyHandlerClass.onion_tor_socks_proxy_host = options.onion_tor_socks_proxy MitmProxyHandlerClass.onion_tor_socks_proxy_port = None if options.socket_timeout: MitmProxyHandlerClass._socket_timeout = options.socket_timeout if options.max_resource_size: MitmProxyHandlerClass._max_resource_size = options.max_resource_size if options.tmp_file_max_memory_size: MitmProxyHandlerClass._tmp_file_max_memory_size = options.tmp_file_max_memory_size self.digest_algorithm = options.digest_algorithm or 'sha1' ca_name = ('Warcprox CA on %s' % socket.gethostname())[:64] self.ca = CertificateAuthority(ca_file=options.cacert or 'warcprox-ca.pem', certs_dir=options.certs_dir or './warcprox-ca', ca_name=ca_name) server_address = (options.address or 'localhost', options.port if options.port is not None else 8000) http_server.HTTPServer.__init__(self, server_address, MitmProxyHandlerClass, bind_and_activate=True)
def __init__(self, ca=None, recorded_url_q=None, stats_db=None, options=warcprox.Options()): server_address = (options.address or 'localhost', options.port if options.port is not None else 8000) if options.onion_tor_socks_proxy: try: host, port = options.onion_tor_socks_proxy.split(':') WarcProxyHandler.onion_tor_socks_proxy_host = host WarcProxyHandler.onion_tor_socks_proxy_port = int(port) except ValueError: WarcProxyHandler.onion_tor_socks_proxy_host = options.onion_tor_socks_proxy WarcProxyHandler.onion_tor_socks_proxy_port = None http_server.HTTPServer.__init__(self, server_address, WarcProxyHandler, bind_and_activate=True) self.digest_algorithm = options.digest_algorithm or 'sha1' if ca is not None: self.ca = ca else: ca_name = 'Warcprox CA on {}'.format(socket.gethostname())[:64] self.ca = CertificateAuthority(ca_file='warcprox-ca.pem', certs_dir='./warcprox-ca', ca_name=ca_name) if recorded_url_q is not None: self.recorded_url_q = recorded_url_q else: self.recorded_url_q = warcprox.TimestampedQueue( maxsize=options.queue_size or 1000) self.stats_db = stats_db self.options = options self.running_stats = warcprox.stats.RunningStats()
def create_or_update(self, overwrite=False): ca = CertificateAuthority( self.ca_name, ca_file_cache=self.cache, cert_cache=self.cache, overwrite=overwrite, ) public_cert_pem = crypto.dump_certificate(FILETYPE_PEM, ca.ca_cert) self.set_attribute("CAName", self.ca_name) self.set_attribute("Hash", hashlib.md5(public_cert_pem).hexdigest()) self.set_attribute("PublicCertPEM", public_cert_pem.decode("ascii")) _ = ssm.put_parameter( Name=self.public_cert_parameter_name, Value=self.get_attribute("PublicCertPEM"), Overwrite=True, Type="String", ) self.physical_resource_id = self.cache.parameter_name("!!root_ca")
class ProxyRouter(object): """ A router which supports http proxy mode requests Handles requests of the form: GET http://example.com The router returns latest capture by default. However, if Memento protocol support is enabled, the memento Accept-Datetime header can be used to select specific capture. See: http://www.mementoweb.org/guide/rfc/#Pattern1.3 for more details. """ BLOCK_SIZE = 4096 DEF_MAGIC_NAME = 'pywb.proxy' CERT_DL_PEM = '/pywb-ca.pem' CERT_DL_P12 = '/pywb-ca.p12' CA_ROOT_FILE = './ca/pywb-ca.pem' CA_ROOT_NAME = 'pywb https proxy replay CA' CA_CERTS_DIR = './ca/certs/' EXTRA_HEADERS = {'cache-control': 'no-cache', 'connection': 'close', 'p3p': 'CP="NOI ADM DEV COM NAV OUR STP"'} def __init__(self, routes, **kwargs): self.error_view = kwargs.get('error_view') proxy_options = kwargs.get('config', {}) if proxy_options: proxy_options = proxy_options.get('proxy_options', {}) self.magic_name = proxy_options.get('magic_name') if not self.magic_name: self.magic_name = self.DEF_MAGIC_NAME proxy_options['magic_name'] = self.magic_name self.extra_headers = proxy_options.get('extra_headers') if not self.extra_headers: self.extra_headers = self.EXTRA_HEADERS proxy_options['extra_headers'] = self.extra_headers res_type = proxy_options.get('cookie_resolver') if res_type == True or res_type == 'cookie': self.resolver = CookieResolver(routes, proxy_options) elif res_type == 'ip': self.resolver = IPCacheResolver(routes, proxy_options) else: self.resolver = ProxyAuthResolver(routes, proxy_options) self.use_banner = proxy_options.get('use_banner', True) self.use_wombat = proxy_options.get('use_client_rewrite', True) self.proxy_cert_dl_view = proxy_options.get('proxy_cert_download_view') if not proxy_options.get('enable_https_proxy'): self.ca = None return try: from certauth.certauth import CertificateAuthority except ImportError: #pragma: no cover print('HTTPS proxy is not available as the "certauth" module ' + 'is not installed') print('Please install via "pip install certauth" ' + 'to enable HTTPS support') self.ca = None return # HTTPS Only Options ca_file = proxy_options.get('root_ca_file', self.CA_ROOT_FILE) # attempt to create the root_ca_file if doesn't exist # (generally recommended to create this seperately) ca_name = proxy_options.get('root_ca_name', self.CA_ROOT_NAME) certs_dir = proxy_options.get('certs_dir', self.CA_CERTS_DIR) self.ca = CertificateAuthority(ca_file=ca_file, certs_dir=certs_dir, ca_name=ca_name) self.use_wildcard = proxy_options.get('use_wildcard_certs', True) def __call__(self, env): is_https = (env['REQUEST_METHOD'] == 'CONNECT') ArchivalRouter.ensure_rel_uri_set(env) # for non-https requests, check non-proxy urls if not is_https: url = env['REL_REQUEST_URI'] if not url.startswith(('http://', 'https://')): return None env['pywb.proxy_scheme'] = 'http' route = None coll = None matcher = None response = None ts = None # check resolver, for pre connect resolve if self.resolver.pre_connect: route, coll, matcher, ts, response = self.resolver.resolve(env) if response: return response # do connect, then get updated url if is_https: response = self.handle_connect(env) if response: return response url = env['REL_REQUEST_URI'] else: parts = urlparse.urlsplit(env['REL_REQUEST_URI']) hostport = parts.netloc.split(':', 1) env['pywb.proxy_host'] = hostport[0] env['pywb.proxy_port'] = hostport[1] if len(hostport) == 2 else '' env['pywb.proxy_req_uri'] = parts.path if parts.query: env['pywb.proxy_req_uri'] += '?' + parts.query env['pywb.proxy_query'] = parts.query env['pywb_proxy_magic'] = self.magic_name # route (static) and other resources to archival replay if env['pywb.proxy_host'] == self.magic_name: env['REL_REQUEST_URI'] = env['pywb.proxy_req_uri'] # special case for proxy install response = self.handle_cert_install(env) if response: return response return None # check resolver, post connect if not self.resolver.pre_connect: route, coll, matcher, ts, response = self.resolver.resolve(env) if response: return response host_prefix = env['pywb.proxy_scheme'] + '://' + self.magic_name rel_prefix = '' # special case for proxy calendar if (env['pywb.proxy_host'] == 'query.' + self.magic_name): url = env['pywb.proxy_req_uri'][1:] rel_prefix = '/' if ts is not None: url = ts + '/' + url wbrequest = route.request_class(env, request_uri=url, wb_url_str=url, coll=coll, host_prefix=host_prefix, rel_prefix=rel_prefix, wburl_class=route.handler.get_wburl_type(), urlrewriter_class=HttpsUrlRewriter, use_abs_prefix=False, is_proxy=True) if matcher: route.apply_filters(wbrequest, matcher) # full rewrite and banner if self.use_wombat and self.use_banner: wbrequest.wb_url.mod = '' elif self.use_banner: # banner only, no rewrite wbrequest.wb_url.mod = 'bn_' else: # unaltered, no rewrite or banner wbrequest.wb_url.mod = 'id_' response = route.handler(wbrequest) if wbrequest.wb_url and wbrequest.wb_url.is_replay(): response.status_headers.replace_headers(self.extra_headers) return response def get_request_socket(self, env): if not self.ca: return None sock = None if env.get('uwsgi.version'): # pragma: no cover try: import uwsgi fd = uwsgi.connection_fd() conn = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) sock = socket.socket(_sock=conn) except Exception: pass elif env.get('gunicorn.socket'): # pragma: no cover sock = env['gunicorn.socket'] if not sock: # attempt to find socket from wsgi.input input_ = env.get('wsgi.input') if input_ and hasattr(input_, '_sock'): sock = socket.socket(_sock=input_._sock) return sock def handle_connect(self, env): sock = self.get_request_socket(env) if not sock: return WbResponse.text_response('HTTPS Proxy Not Supported', '405 HTTPS Proxy Not Supported') sock.send('HTTP/1.0 200 Connection Established\r\n') sock.send('Server: pywb proxy\r\n') sock.send('\r\n') hostname, port = env['REL_REQUEST_URI'].split(':') if not self.use_wildcard: certfile = self.ca.cert_for_host(hostname) else: certfile = self.ca.get_wildcard_cert(hostname) try: ssl_sock = ssl.wrap_socket(sock, server_side=True, certfile=certfile, #ciphers="ALL", suppress_ragged_eofs=False, ssl_version=ssl.PROTOCOL_SSLv23 ) env['pywb.proxy_ssl_sock'] = ssl_sock buffreader = BufferedReader(ssl_sock, block_size=self.BLOCK_SIZE) statusline = buffreader.readline().rstrip() except Exception as se: raise BadRequestException(se.message) statusparts = statusline.split(' ') if len(statusparts) < 3: raise BadRequestException('Invalid Proxy Request: ' + statusline) env['REQUEST_METHOD'] = statusparts[0] env['REL_REQUEST_URI'] = ('https://' + env['REL_REQUEST_URI'].replace(':443', '') + statusparts[1]) env['SERVER_PROTOCOL'] = statusparts[2].strip() env['pywb.proxy_scheme'] = 'https' env['pywb.proxy_host'] = hostname env['pywb.proxy_port'] = port env['pywb.proxy_req_uri'] = statusparts[1] queryparts = env['REL_REQUEST_URI'].split('?', 1) env['PATH_INFO'] = queryparts[0] env['QUERY_STRING'] = queryparts[1] if len(queryparts) > 1 else '' env['pywb.proxy_query'] = env['QUERY_STRING'] while True: line = buffreader.readline() if line: line = line.rstrip() if not line: break parts = line.split(':', 1) if len(parts) < 2: continue name = parts[0].strip() value = parts[1].strip() name = name.replace('-', '_').upper() if name not in ('CONTENT_LENGTH', 'CONTENT_TYPE'): name = 'HTTP_' + name env[name] = value remain = buffreader.rem_length() if remain > 0: remainder = buffreader.read(self.BLOCK_SIZE) env['wsgi.input'] = BufferedReader(ssl_sock, block_size=self.BLOCK_SIZE, starting_data=remainder) def handle_cert_install(self, env): if env['pywb.proxy_req_uri'] in ('/', '/index.html', '/index.html'): available = (self.ca is not None) if self.proxy_cert_dl_view: return (self.proxy_cert_dl_view. render_response(available=available, pem_path=self.CERT_DL_PEM, p12_path=self.CERT_DL_P12)) elif env['pywb.proxy_req_uri'] == self.CERT_DL_PEM: if not self.ca: return None buff = '' with open(self.ca.ca_file, 'rb') as fh: buff = fh.read() content_type = 'application/x-x509-ca-cert' return WbResponse.text_response(buff, content_type=content_type) elif env['pywb.proxy_req_uri'] == self.CERT_DL_P12: if not self.ca: return None buff = self.ca.get_root_PKCS12() content_type = 'application/x-pkcs12' return WbResponse.text_response(buff, content_type=content_type)
class ProxyRouter(object): """ A router which supports http proxy mode requests Handles requests of the form: GET http://example.com The router returns latest capture by default. However, if Memento protocol support is enabled, the memento Accept-Datetime header can be used to select specific capture. See: http://www.mementoweb.org/guide/rfc/#Pattern1.3 for more details. """ BLOCK_SIZE = 4096 DEF_MAGIC_NAME = 'pywb.proxy' CERT_DL_PEM = '/pywb-ca.pem' CERT_DL_P12 = '/pywb-ca.p12' CA_ROOT_FILE = './ca/pywb-ca.pem' CA_ROOT_NAME = 'pywb https proxy replay CA' CA_CERTS_DIR = './ca/certs/' EXTRA_HEADERS = { 'cache-control': 'no-cache', 'p3p': 'CP="NOI ADM DEV COM NAV OUR STP"' } def __init__(self, routes, **kwargs): self.error_view = kwargs.get('error_view') proxy_options = kwargs.get('config', {}) if proxy_options: proxy_options = proxy_options.get('proxy_options', {}) self.magic_name = proxy_options.get('magic_name') if not self.magic_name: self.magic_name = self.DEF_MAGIC_NAME proxy_options['magic_name'] = self.magic_name self.extra_headers = proxy_options.get('extra_headers') if not self.extra_headers: self.extra_headers = self.EXTRA_HEADERS proxy_options['extra_headers'] = self.extra_headers if proxy_options.get('cookie_resolver', True): self.resolver = CookieResolver(routes, proxy_options) else: self.resolver = ProxyAuthResolver(routes, proxy_options) self.unaltered = proxy_options.get('unaltered_replay', False) self.proxy_cert_dl_view = proxy_options.get('proxy_cert_download_view') if not proxy_options.get('enable_https_proxy'): self.ca = None return try: from certauth.certauth import CertificateAuthority except ImportError: #pragma: no cover print('HTTPS proxy is not available as the "certauth" module ' + 'is not installed') print('Please install via "pip install certauth" ' + 'to enable HTTPS support') self.ca = None return # HTTPS Only Options ca_file = proxy_options.get('root_ca_file', self.CA_ROOT_FILE) # attempt to create the root_ca_file if doesn't exist # (generally recommended to create this seperately) ca_name = proxy_options.get('root_ca_name', self.CA_ROOT_NAME) certs_dir = proxy_options.get('certs_dir', self.CA_CERTS_DIR) self.ca = CertificateAuthority(ca_file=ca_file, certs_dir=certs_dir, ca_name=ca_name) self.use_wildcard = proxy_options.get('use_wildcard_certs', True) def __call__(self, env): is_https = (env['REQUEST_METHOD'] == 'CONNECT') ArchivalRouter.ensure_rel_uri_set(env) # for non-https requests, check non-proxy urls if not is_https: url = env['REL_REQUEST_URI'] if not url.startswith(('http://', 'https://')): return None env['pywb.proxy_scheme'] = 'http' route = None coll = None matcher = None response = None ts = None # check resolver, for pre connect resolve if self.resolver.pre_connect: route, coll, matcher, ts, response = self.resolver.resolve(env) if response: return response # do connect, then get updated url if is_https: response = self.handle_connect(env) if response: return response url = env['REL_REQUEST_URI'] else: parts = urlparse.urlsplit(env['REL_REQUEST_URI']) hostport = parts.netloc.split(':', 1) env['pywb.proxy_host'] = hostport[0] env['pywb.proxy_port'] = hostport[1] if len(hostport) == 2 else '' env['pywb.proxy_req_uri'] = parts.path if parts.query: env['pywb.proxy_req_uri'] += '?' + parts.query env['pywb_proxy_magic'] = self.magic_name # route (static) and other resources to archival replay if env['pywb.proxy_host'] == self.magic_name: env['REL_REQUEST_URI'] = env['pywb.proxy_req_uri'] # special case for proxy install response = self.handle_cert_install(env) if response: return response return None # check resolver, post connect if not self.resolver.pre_connect: route, coll, matcher, ts, response = self.resolver.resolve(env) if response: return response host_prefix = env['pywb.proxy_scheme'] + '://' + self.magic_name rel_prefix = '' # special case for proxy calendar if (env['pywb.proxy_host'] == 'query.' + self.magic_name): url = env['pywb.proxy_req_uri'][1:] rel_prefix = '/' if ts is not None: url = ts + '/' + url wbrequest = route.request_class( env, request_uri=url, wb_url_str=url, coll=coll, host_prefix=host_prefix, rel_prefix=rel_prefix, wburl_class=route.handler.get_wburl_type(), urlrewriter_class=HttpsUrlRewriter, use_abs_prefix=False, is_proxy=True) if matcher: route.apply_filters(wbrequest, matcher) if self.unaltered: wbrequest.wb_url.mod = 'id_' elif is_https: wbrequest.wb_url.mod = 'bn_' response = route.handler(wbrequest) if wbrequest.wb_url and wbrequest.wb_url.is_replay(): response.status_headers.replace_headers(self.extra_headers) return response def get_request_socket(self, env): if not self.ca: return None sock = None if env.get('uwsgi.version'): # pragma: no cover try: import uwsgi fd = uwsgi.connection_fd() conn = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) sock = socket.socket(_sock=conn) except Exception: pass elif env.get('gunicorn.socket'): # pragma: no cover sock = env['gunicorn.socket'] if not sock: # attempt to find socket from wsgi.input input_ = env.get('wsgi.input') if input_ and hasattr(input_, '_sock'): sock = socket.socket(_sock=input_._sock) return sock def handle_connect(self, env): sock = self.get_request_socket(env) if not sock: return WbResponse.text_response('HTTPS Proxy Not Supported', '405 HTTPS Proxy Not Supported') sock.send('HTTP/1.0 200 Connection Established\r\n') sock.send('Server: pywb proxy\r\n') sock.send('\r\n') hostname, port = env['REL_REQUEST_URI'].split(':') if not self.use_wildcard: certfile = self.ca.cert_for_host(hostname) else: certfile = self.ca.get_wildcard_cert(hostname) try: ssl_sock = ssl.wrap_socket( sock, server_side=True, certfile=certfile, #ciphers="ALL", suppress_ragged_eofs=False, ssl_version=ssl.PROTOCOL_SSLv23) env['pywb.proxy_ssl_sock'] = ssl_sock buffreader = BufferedReader(ssl_sock, block_size=self.BLOCK_SIZE) statusline = buffreader.readline().rstrip() except Exception as se: raise BadRequestException(se.message) statusparts = statusline.split(' ') if len(statusparts) < 3: raise BadRequestException('Invalid Proxy Request: ' + statusline) env['REQUEST_METHOD'] = statusparts[0] env['REL_REQUEST_URI'] = ('https://' + env['REL_REQUEST_URI'].replace(':443', '') + statusparts[1]) env['SERVER_PROTOCOL'] = statusparts[2].strip() env['pywb.proxy_scheme'] = 'https' env['pywb.proxy_host'] = hostname env['pywb.proxy_port'] = port env['pywb.proxy_req_uri'] = statusparts[1] queryparts = env['REL_REQUEST_URI'].split('?', 1) env['PATH_INFO'] = queryparts[0] env['QUERY_STRING'] = queryparts[1] if len(queryparts) > 1 else '' while True: line = buffreader.readline() if line: line = line.rstrip() if not line: break parts = line.split(':', 1) if len(parts) < 2: continue name = parts[0].strip() value = parts[1].strip() name = name.replace('-', '_').upper() if name not in ('CONTENT_LENGTH', 'CONTENT_TYPE'): name = 'HTTP_' + name env[name] = value remain = buffreader.rem_length() if remain > 0: remainder = buffreader.read(self.BLOCK_SIZE) env['wsgi.input'] = BufferedReader(ssl_sock, block_size=self.BLOCK_SIZE, starting_data=remainder) def handle_cert_install(self, env): if env['pywb.proxy_req_uri'] in ('/', '/index.html', '/index.html'): available = (self.ca is not None) if self.proxy_cert_dl_view: return (self.proxy_cert_dl_view.render_response( available=available, pem_path=self.CERT_DL_PEM, p12_path=self.CERT_DL_P12)) elif env['pywb.proxy_req_uri'] == self.CERT_DL_PEM: if not self.ca: return None buff = '' with open(self.ca.ca_file, 'rb') as fh: buff = fh.read() content_type = 'application/x-x509-ca-cert' return WbResponse.text_response(buff, content_type=content_type) elif env['pywb.proxy_req_uri'] == self.CERT_DL_P12: if not self.ca: return None buff = self.ca.get_root_PKCS12() content_type = 'application/x-pkcs12' return WbResponse.text_response(buff, content_type=content_type)
#!/usr/bin/env python3 from certauth.certauth import main, CertificateAuthority, FileCache, LRUCache, ROOT_CA ca = CertificateAuthority('My Custom CA', 'cert.pem', cert_cache='/tmp/certs') filename = ca.cert_for_host('test.server') print(ca) print(filename)
def devpipostgresql_postgresql(request): tmpdir = py.path.local( tempfile.mkdtemp(prefix='test-', suffix='-devpi-postgresql')) try: cap = py.io.StdCaptureFD() cap.startall() subprocess.check_call(['initdb', tmpdir.strpath]) cap.reset() postgresql_conf_lines = [ "fsync = off", "full_page_writes = off", "synchronous_commit = off", "unix_socket_directories = '{}'".format(tmpdir.strpath) ] pg_ssl = request.config.option.backend_postgresql_ssl host = 'localhost' if pg_ssl: # Make certificate authority and server certificate ca = CertificateAuthority('Test CA', tmpdir.join('ca.pem').strpath, cert_cache=tmpdir.strpath) server_cert = ca.cert_for_host(host) if not sys.platform.startswith("win"): # Postgres requires restrictive permissions on private key. os.chmod(server_cert, 0o600) postgresql_conf_lines.extend([ "ssl = on", "ssl_cert_file = '{}'".format(server_cert), "ssl_key_file = '{}'".format(server_cert), "ssl_ca_file = 'ca.pem'" ]) # Require SSL connections to be authenticated by client certificates. with tmpdir.join('pg_hba.conf').open('w', encoding='ascii') as f: f.write("\n".join([ # "local" is for Unix domain socket connections only 'local all all trust', # IPv4 local connections: 'hostssl all all 127.0.0.1/32 cert', 'host all all 127.0.0.1/32 trust' ])) with tmpdir.join('postgresql.conf').open('w+', encoding='ascii') as f: f.write("\n".join(postgresql_conf_lines)) port = get_open_port(host) cap = py.io.StdCaptureFD() cap.startall() p = subprocess.Popen( ['postgres', '-D', tmpdir.strpath, '-h', host, '-p', str(port)]) wait_for_port(host, port) cap.reset() try: cap = py.io.StdCaptureFD() cap.startall() subprocess.check_call( ['createdb', '-h', host, '-p', str(port), 'devpi']) cap.reset() user = getpass.getuser() settings = dict(host=host, port=port, user=user) if pg_ssl: # Make client certificate for user and authenticate with it. client_cert = ca.cert_for_host(user) settings['ssl_check_hostname'] = 'yes' settings['ssl_ca_certs'] = tmpdir.join('ca.pem').strpath settings['ssl_certfile'] = client_cert main.Storage(tmpdir, notify_on_commit=False, cache_size=10000, settings=settings) yield settings for conn, db, ts in Storage._connections: try: conn.close() except AttributeError: pass # use a copy of the set, as it might be changed in another thread for db in set(Storage._dbs_created): try: subprocess.check_call([ 'dropdb', '-h', Storage.host, '-p', str(Storage.port), db ]) except subprocess.CalledProcessError: pass finally: p.terminate() p.wait() finally: tmpdir.remove(ignore_errors=True)
def ca(): return CertificateAuthority("Test CA", TEST_CA_ROOT, TEST_CA_DIR)
class WSGIProxMiddleware(object): DEFAULT_HOST = 'wsgiprox' CA_ROOT_NAME = 'wsgiprox https proxy CA' CA_ROOT_FILE = os.path.join('.', 'ca', 'wsgiprox-ca.pem') SSL_BASIC_OPTIONS = (SSL.OP_CIPHER_SERVER_PREFERENCE) SSL_DEFAULT_METHOD = SSL.SSLv23_METHOD SSL_DEFAULT_OPTIONS = (SSL.OP_NO_TICKET | SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3 | SSL_BASIC_OPTIONS) CONNECT_RESPONSE_1_1 = b'HTTP/1.1 200 Connection Established\r\n\r\n' CONNECT_RESPONSE_1_0 = b'HTTP/1.0 200 Connection Established\r\n\r\n' DEFAULT_MAX_TUNNELS = 50 @classmethod def set_connection_class(cls): try: import gevent.socket assert (gevent.socket.socket == socket.socket) from wsgiprox.gevent_ssl import SSLConnection as SSLConnection cls.is_gevent_ssl = True except Exception as e: #pragma: no cover logger.debug(str(e)) from OpenSSL.SSL import Connection as SSLConnection cls.is_gevent_ssl = False finally: cls.SSLConnection = SSLConnection def __init__(self, wsgi, prefix_resolver=None, download_host=None, proxy_host=None, proxy_options=None, proxy_apps=None): self._wsgi = wsgi self.set_connection_class() if isinstance(prefix_resolver, str): prefix_resolver = FixedResolver(prefix_resolver) self.prefix_resolver = prefix_resolver or FixedResolver() self.proxy_apps = proxy_apps or {} self.proxy_host = proxy_host or self.DEFAULT_HOST if self.proxy_host not in self.proxy_apps: self.proxy_apps[self.proxy_host] = None # HTTPS Only Options proxy_options = proxy_options or {} ca_name = proxy_options.get('ca_name', self.CA_ROOT_NAME) ca_file_cache = proxy_options.get('ca_file_cache', self.CA_ROOT_FILE) #self.ca = CertificateAuthority(ca_name=ca_name, # ca_file_cache=ca_file_cache, # cert_cache=None, # cert_not_before=-3600) self.ca = CertificateAuthority(ca_name=ca_name, ca_file_cache=ca_file_cache, cert_cache=50, cert_not_before=-3600) self.keepalive_max = proxy_options.get('keepalive_max', self.DEFAULT_MAX_TUNNELS) self.keepalive_opts = hasattr(socket, 'TCP_KEEPIDLE') self._tcp_keepidle = proxy_options.get('tcp_keepidle', 60) self._tcp_keepintvl = proxy_options.get('tcp_keepintvl', 5) self._tcp_keepcnt = proxy_options.get('tcp_keepcnt', 3) self.num_open_tunnels = 0 try: self.root_ca_file = self.ca.get_root_pem_filename() except Exception as e: self.root_ca_file = None self.use_wildcard = proxy_options.get('use_wildcard_certs', True) if proxy_options.get('enable_cert_download', True): download_host = download_host or self.DEFAULT_HOST self.proxy_apps[download_host] = CertDownloader(self.ca) self.enable_ws = proxy_options.get('enable_websockets', True) if WebSocketHandler == object: self.enable_ws = None def wsgi(self, env, start_response): # see if the host matches one of the proxy app hosts # if so, try to see if there is an wsgi app set # and if it returns something hostname = env.get('wsgiprox.matched_proxy_host') if hostname: app = self.proxy_apps.get(hostname) if app: res = app(env, start_response) if res is not None: return res # call upstream wsgi app return self._wsgi(env, start_response) def __call__(self, env, start_response): if env['REQUEST_METHOD'] == 'CONNECT': return self.handle_connect(env, start_response) else: self.ensure_request_uri(env) if env['REQUEST_URI'].startswith('http://'): return self.handle_http_proxy(env, start_response) else: return self.wsgi(env, start_response) def handle_http_proxy(self, env, start_response): res = self.require_auth(env, start_response) if res is not None: return res handler = HttpProxyHandler(start_response, self.wsgi, self.resolve) return handler(env) def handle_connect(self, env, start_response): raw_sock = self.get_raw_socket(env) if not raw_sock: start_response('405 HTTPS Proxy Not Supported', [('Content-Length', '0')]) return [] res = self.require_auth(env, start_response) if res is not None: return res connect_handler = None curr_sock = None try: scheme, curr_sock = self.wrap_socket(env, raw_sock) connect_handler = ConnectHandler(curr_sock, scheme, self.wsgi, self.resolve) self.num_open_tunnels += 1 connect_handler(env, self.enable_ws) while self.keep_alive(connect_handler): connect_handler(env, self.enable_ws) except Exception as e: logger.debug(str(e)) start_response('500 Unexpected Error', [('Content-Length', '0')]) finally: if connect_handler: self.num_open_tunnels -= 1 connect_handler.close() if curr_sock and curr_sock != raw_sock: # this seems to necessary to avoid tls data read later # in the same gevent try: if self.is_gevent_ssl: curr_sock.recv(0) curr_sock.shutdown() except: pass finally: curr_sock.close() start_response('200 OK', []) return [] def keep_alive(self, connect_handler): # keepalive disabled if self.keepalive_max < 0: return False if not connect_handler.is_keepalive: return False # no max if self.keepalive_max == 0: return True return (self.num_open_tunnels <= self.keepalive_max) def _new_context(self): context = SSL.Context(self.SSL_DEFAULT_METHOD) context.set_options(self.SSL_DEFAULT_OPTIONS) context.set_session_cache_mode(SSL.SESS_CACHE_OFF) return context def create_ssl_context(self, hostname): cert, key = self.ca.load_cert(hostname, wildcard=self.use_wildcard, wildcard_use_parent=True) context = self._new_context() context.use_privatekey(key) context.use_certificate(cert) return context def _get_connect_response(self, env): if env.get('SERVER_PROTOCOL', 'HTTP/1.0') == 'HTTP/1.1': return self.CONNECT_RESPONSE_1_1 else: return self.CONNECT_RESPONSE_1_0 def wrap_socket(self, env, sock): if self.keepalive_max >= 0: sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) if self.keepalive_opts: #pragma: no cover sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, self._tcp_keepidle) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, self._tcp_keepintvl) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, self._tcp_keepcnt) host_port = env['PATH_INFO'] hostname, port = host_port.split(':', 1) env['wsgiprox.connect_host'] = hostname sock.sendall(self._get_connect_response(env)) if port == '80': return 'http', sock if port != '443': env['wsgiprox.connect_port'] = port peek_buff = sock.recv(16, socket.MSG_PEEK) # http websocket traffic would start with a GET if peek_buff.startswith(b'GET '): return 'http', sock def sni_callback(connection): sni_hostname = connection.get_servername() # curl -k (unverified) mode results in empty hostname here # requests unverified mode still includes an sni hostname if not sni_hostname: return if six.PY3: sni_hostname = sni_hostname.decode('iso-8859-1') # if same host as CONNECT header, then just keep current context if sni_hostname == hostname: return connection.set_context(self.create_ssl_context(sni_hostname)) env['wsgiprox.connect_host'] = sni_hostname context = self.create_ssl_context(hostname) context.set_tlsext_servername_callback(sni_callback) ssl_sock = self.SSLConnection(context, sock) ssl_sock.set_accept_state() ssl_sock.do_handshake() return 'https', ssl_sock def require_auth(self, env, start_response): if not hasattr(self.prefix_resolver, 'require_auth'): return auth_req = self.prefix_resolver.require_auth(env) if not auth_req: return auth_req = 'Basic realm="{0}"'.format(auth_req) headers = [('Proxy-Authenticate', auth_req), ('Proxy-Connection', 'close'), ('Content-Length', '0')] start_response('407 Proxy Authentication', headers) return [] def resolve(self, url, env, hostname): if hostname in self.proxy_apps.keys(): parts = urlsplit(url) full = parts.path if parts.query: full += '?' + parts.query env['REQUEST_URI'] = full env['wsgiprox.matched_proxy_host'] = hostname env['wsgiprox.proxy_host'] = hostname else: env['REQUEST_URI'] = self.prefix_resolver(url, env) env['wsgiprox.proxy_host'] = self.proxy_host queryparts = env['REQUEST_URI'].split('?', 1) env['PATH_INFO'] = queryparts[0] env['QUERY_STRING'] = queryparts[1] if len(queryparts) > 1 else '' def ensure_request_uri(self, env): if 'REQUEST_URI' in env: return full_uri = env['PATH_INFO'] if env.get('QUERY_STRING'): full_uri += '?' + env['QUERY_STRING'] env['REQUEST_URI'] = full_uri @classmethod def get_raw_socket(cls, env): #pragma: no cover sock = None if env.get('uwsgi.version'): try: import uwsgi fd = uwsgi.connection_fd() sock = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) except Exception as e: pass elif env.get('gunicorn.socket'): sock = env['gunicorn.socket'] if not sock: # attempt to find socket from wsgi.input input_ = env.get('wsgi.input') if input_: if hasattr(input_, '_sock'): raw = input_._sock sock = socket.socket(_sock=raw) elif hasattr(input_, 'raw'): sock = input_.raw._sock elif hasattr(input_, 'rfile'): # PY3 if hasattr(input_.rfile, 'raw'): sock = input_.rfile.raw._sock # PY2 else: sock = input_.rfile._sock return sock
class ProxyRouter(object): """ A router which supports http proxy mode requests Handles requests of the form: GET http://example.com The router returns latest capture by default. However, if Memento protocol support is enabled, the memento Accept-Datetime header can be used to select specific capture. See: http://www.mementoweb.org/guide/rfc/#Pattern1.3 for more details. """ BLOCK_SIZE = 4096 DEF_MAGIC_NAME = 'pywb.proxy' BUFF_RESPONSE_MEM_SIZE = 1024 * 1024 CERT_DL_PEM = '/pywb-ca.pem' CERT_DL_P12 = '/pywb-ca.p12' CA_ROOT_FILE = './ca/pywb-ca.pem' CA_ROOT_NAME = 'pywb https proxy replay CA' CA_CERTS_DIR = './ca/certs/' EXTRA_HEADERS = { 'cache-control': 'no-cache', 'connection': 'close', 'p3p': 'CP="NOI ADM DEV COM NAV OUR STP"' } def __init__(self, routes, **kwargs): self.error_view = kwargs.get('error_view') proxy_options = kwargs.get('config', {}) if proxy_options: proxy_options = proxy_options.get('proxy_options', {}) self.magic_name = proxy_options.get('magic_name') if not self.magic_name: self.magic_name = self.DEF_MAGIC_NAME proxy_options['magic_name'] = self.magic_name self.extra_headers = proxy_options.get('extra_headers') if not self.extra_headers: self.extra_headers = self.EXTRA_HEADERS proxy_options['extra_headers'] = self.extra_headers res_type = proxy_options.get('cookie_resolver', True) if res_type == 'auth' or not res_type: self.resolver = ProxyAuthResolver(routes, proxy_options) elif res_type == 'ip': self.resolver = IPCacheResolver(routes, proxy_options) #elif res_type == True or res_type == 'cookie': # self.resolver = CookieResolver(routes, proxy_options) else: self.resolver = CookieResolver(routes, proxy_options) self.use_banner = proxy_options.get('use_banner', True) self.use_wombat = proxy_options.get('use_client_rewrite', True) self.proxy_cert_dl_view = proxy_options.get('proxy_cert_download_view') if not proxy_options.get('enable_https_proxy'): self.ca = None return try: from certauth.certauth import CertificateAuthority except ImportError: #pragma: no cover print('HTTPS proxy is not available as the "certauth" module ' + 'is not installed') print('Please install via "pip install certauth" ' + 'to enable HTTPS support') self.ca = None return # HTTPS Only Options ca_file = proxy_options.get('root_ca_file', self.CA_ROOT_FILE) # attempt to create the root_ca_file if doesn't exist # (generally recommended to create this seperately) ca_name = proxy_options.get('root_ca_name', self.CA_ROOT_NAME) certs_dir = proxy_options.get('certs_dir', self.CA_CERTS_DIR) self.ca = CertificateAuthority(ca_file=ca_file, certs_dir=certs_dir, ca_name=ca_name) self.use_wildcard = proxy_options.get('use_wildcard_certs', True) def __call__(self, env): is_https = (env['REQUEST_METHOD'] == 'CONNECT') ArchivalRouter.ensure_rel_uri_set(env) # for non-https requests, check non-proxy urls if not is_https: url = env['REL_REQUEST_URI'] if not url.startswith(('http://', 'https://')): return None env['pywb.proxy_scheme'] = 'http' route = None coll = None matcher = None response = None ts = None # check resolver, for pre connect resolve if self.resolver.pre_connect: route, coll, matcher, ts, response = self.resolver.resolve(env) if response: return response # do connect, then get updated url if is_https: response = self.handle_connect(env) if response: return response url = env['REL_REQUEST_URI'] else: parts = urlsplit(env['REL_REQUEST_URI']) hostport = parts.netloc.split(':', 1) env['pywb.proxy_host'] = hostport[0] env['pywb.proxy_port'] = hostport[1] if len(hostport) == 2 else '' env['pywb.proxy_req_uri'] = parts.path if parts.query: env['pywb.proxy_req_uri'] += '?' + parts.query env['pywb.proxy_query'] = parts.query if self.resolver.supports_switching: env['pywb_proxy_magic'] = self.magic_name # route (static) and other resources to archival replay if env['pywb.proxy_host'] == self.magic_name: env['REL_REQUEST_URI'] = env['pywb.proxy_req_uri'] # special case for proxy install response = self.handle_cert_install(env) if response: return response return None # check resolver, post connect if not self.resolver.pre_connect: route, coll, matcher, ts, response = self.resolver.resolve(env) if response: return response rel_prefix = '' custom_prefix = env.get('HTTP_PYWB_REWRITE_PREFIX', '') if custom_prefix: host_prefix = custom_prefix urlrewriter_class = UrlRewriter abs_prefix = True # always rewrite to absolute here rewrite_opts = dict(no_match_rel=True) else: host_prefix = env['pywb.proxy_scheme'] + '://' + self.magic_name urlrewriter_class = SchemeOnlyUrlRewriter abs_prefix = False rewrite_opts = {} # special case for proxy calendar if (env['pywb.proxy_host'] == 'query.' + self.magic_name): url = env['pywb.proxy_req_uri'][1:] rel_prefix = '/' if ts is not None: url = ts + '/' + url wbrequest = route.request_class( env, request_uri=url, wb_url_str=url, coll=coll, host_prefix=host_prefix, rel_prefix=rel_prefix, wburl_class=route.handler.get_wburl_type(), urlrewriter_class=urlrewriter_class, use_abs_prefix=abs_prefix, rewrite_opts=rewrite_opts, is_proxy=True) if matcher: route.apply_filters(wbrequest, matcher) # full rewrite and banner if self.use_wombat and self.use_banner: wbrequest.wb_url.mod = '' elif self.use_banner: # banner only, no rewrite wbrequest.wb_url.mod = 'bn_' else: # unaltered, no rewrite or banner wbrequest.wb_url.mod = 'uo_' response = route.handler(wbrequest) if not response: return None # add extra headers for replay responses if wbrequest.wb_url and wbrequest.wb_url.is_replay(): response.status_headers.replace_headers(self.extra_headers) # check for content-length res = response.status_headers.get_header('content-length') try: if int(res) > 0: return response except: pass # need to either chunk or buffer to get content-length if env.get('SERVER_PROTOCOL') == 'HTTP/1.1': response.status_headers.remove_header('content-length') response.status_headers.headers.append( ('Transfer-Encoding', 'chunked')) response.body = self._chunk_encode(response.body) else: response.body = self._buffer_response(response.status_headers, response.body) return response @staticmethod def _chunk_encode(orig_iter): for chunk in orig_iter: if not len(chunk): continue chunk_len = b'%X\r\n' % len(chunk) yield chunk_len yield chunk yield b'\r\n' yield b'0\r\n\r\n' @staticmethod def _buffer_response(status_headers, iterator): out = SpooledTemporaryFile(ProxyRouter.BUFF_RESPONSE_MEM_SIZE) size = 0 for buff in iterator: size += len(buff) out.write(buff) content_length_str = str(size) # remove existing content length status_headers.replace_header('Content-Length', content_length_str) out.seek(0) return RewriteContent.stream_to_gen(out) def get_request_socket(self, env): if not self.ca: return None sock = None if env.get('uwsgi.version'): # pragma: no cover try: import uwsgi fd = uwsgi.connection_fd() conn = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) try: sock = socket.socket(_sock=conn) except: sock = conn except Exception as e: pass elif env.get('gunicorn.socket'): # pragma: no cover sock = env['gunicorn.socket'] if not sock: # attempt to find socket from wsgi.input input_ = env.get('wsgi.input') if input_: if hasattr(input_, '_sock'): # pragma: no cover raw = input_._sock sock = socket.socket(_sock=raw) # pragma: no cover elif hasattr(input_, 'raw'): sock = input_.raw._sock return sock def handle_connect(self, env): sock = self.get_request_socket(env) if not sock: return WbResponse.text_response('HTTPS Proxy Not Supported', '405 HTTPS Proxy Not Supported') sock.send(b'HTTP/1.0 200 Connection Established\r\n') sock.send(b'Proxy-Connection: close\r\n') sock.send(b'Server: pywb proxy\r\n') sock.send(b'\r\n') hostname, port = env['REL_REQUEST_URI'].split(':') if not self.use_wildcard: certfile = self.ca.cert_for_host(hostname) else: certfile = self.ca.get_wildcard_cert(hostname) try: ssl_sock = ssl.wrap_socket( sock, server_side=True, certfile=certfile, #ciphers="ALL", suppress_ragged_eofs=False, ssl_version=ssl.PROTOCOL_SSLv23) env['pywb.proxy_ssl_sock'] = ssl_sock buffreader = BufferedReader(ssl_sock, block_size=self.BLOCK_SIZE) statusline = to_native_str(buffreader.readline().rstrip()) except Exception as se: raise BadRequestException(se.message) statusparts = statusline.split(' ') if len(statusparts) < 3: raise BadRequestException('Invalid Proxy Request: ' + statusline) env['REQUEST_METHOD'] = statusparts[0] env['REL_REQUEST_URI'] = ('https://' + env['REL_REQUEST_URI'].replace(':443', '') + statusparts[1]) env['SERVER_PROTOCOL'] = statusparts[2].strip() env['pywb.proxy_scheme'] = 'https' env['pywb.proxy_host'] = hostname env['pywb.proxy_port'] = port env['pywb.proxy_req_uri'] = statusparts[1] queryparts = env['REL_REQUEST_URI'].split('?', 1) env['PATH_INFO'] = queryparts[0] env['QUERY_STRING'] = queryparts[1] if len(queryparts) > 1 else '' env['pywb.proxy_query'] = env['QUERY_STRING'] while True: line = to_native_str(buffreader.readline()) if line: line = line.rstrip() if not line: break parts = line.split(':', 1) if len(parts) < 2: continue name = parts[0].strip() value = parts[1].strip() name = name.replace('-', '_').upper() if name not in ('CONTENT_LENGTH', 'CONTENT_TYPE'): name = 'HTTP_' + name env[name] = value env['wsgi.input'] = buffreader #remain = buffreader.rem_length() #if remain > 0: #remainder = buffreader.read() #env['wsgi.input'] = BufferedReader(BytesIO(remainder)) #remainder = buffreader.read(self.BLOCK_SIZE) #env['wsgi.input'] = BufferedReader(ssl_sock, # block_size=self.BLOCK_SIZE, # starting_data=remainder) def handle_cert_install(self, env): if env['pywb.proxy_req_uri'] in ('/', '/index.html', '/index.html'): available = (self.ca is not None) if self.proxy_cert_dl_view: return (self.proxy_cert_dl_view.render_response( available=available, pem_path=self.CERT_DL_PEM, p12_path=self.CERT_DL_P12)) elif env['pywb.proxy_req_uri'] == self.CERT_DL_PEM: if not self.ca: return None buff = b'' with open(self.ca.ca_file, 'rb') as fh: buff = fh.read() content_type = 'application/x-x509-ca-cert' headers = [('Content-Length', str(len(buff)))] return WbResponse.bin_stream([buff], content_type=content_type, headers=headers) elif env['pywb.proxy_req_uri'] == self.CERT_DL_P12: if not self.ca: return None buff = self.ca.get_root_PKCS12() content_type = 'application/x-pkcs12' headers = [('Content-Length', str(len(buff)))] return WbResponse.bin_stream([buff], content_type=content_type, headers=headers)
def __generate_certificate(self): ca = CertificateAuthority('Python digikey-api CA', str(self._ca_cert), cert_cache=str(self._storage_path)) return ca.cert_for_host('localhost')