def __init__(self, config):
self._config = config
# Check if we're using a custom list of a CA certificates
trust_root = config.federation_ca_trust_root
if trust_root is None:
# Use CA root certs provided by OpenSSL
trust_root = platformTrust()
# "insecurelyLowerMinimumTo" is the argument that will go lower than
# Twisted's default, which is why it is marked as "insecure" (since
# Twisted's defaults are reasonably secure). But, since Twisted is
# moving to TLS 1.2 by default, we want to respect the config option if
# it is set to 1.0 (which the alternate option, raiseMinimumTo, will not
# let us do).
minTLS = _TLS_VERSION_MAP[config.federation_client_minimum_tls_version]
_verify_ssl = CertificateOptions(trustRoot=trust_root,
insecurelyLowerMinimumTo=minTLS)
self._verify_ssl_context = _verify_ssl.getContext()
self._verify_ssl_context.set_info_callback(_context_info_cb)
_no_verify_ssl = CertificateOptions(insecurelyLowerMinimumTo=minTLS)
self._no_verify_ssl_context = _no_verify_ssl.getContext()
self._no_verify_ssl_context.set_info_callback(_context_info_cb)
def start_ssl(self):
log.debug("Enabling SSL with PKey: %s, Cert: %s", self.pkey, self.cert)
check_ssl_keys()
with open(configmanager.get_config_dir(self.cert)) as cert:
certificate = Certificate.loadPEM(cert.read()).original
with open(configmanager.get_config_dir(self.pkey)) as pkey:
private_key = KeyPair.load(pkey.read(), FILETYPE_PEM).original
options = CertificateOptions(privateKey=private_key, certificate=certificate, method=SSL.SSLv23_METHOD)
options.getContext().set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)
self.socket = reactor.listenSSL(self.port, self.site, options)
log.info("Serving on %s:%s view at https://127.0.0.1:%s", "0.0.0.0", self.port, self.port)
def creatorForNetloc(self, hostname, port):
certificateOptions = CertificateOptions(
trustRoot=self._trustRoot)
return PermissiveClientTLSOptions(
hostname.decode("ascii"),
certificateOptions.getContext())
class GeminiDownloadHandler:
"""
Scrapy download handler for gemini:// scheme URLs.
This implementation is *heavily* based on scrapy's HTTP 1.1 and telnet
handlers as references. I did, however, make several attempts to simplify
the code and use idiomatic twisted patterns. Some integrity checks had to
be removed since gemini does not use a Content-Length or checksum. Since
scrapy is built around HTTP requests/responses, this code will take the
gemini response and generate a pseudo-HTTP response with an equivalent
status code and headers. This is necessary to retain compatibility with
most of the library's middleware.
"""
lazy = False
def __init__(self, settings, crawler=None):
self.crawler = crawler
self.default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
self.default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
self.fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')
self.context_factory = CertificateOptions(
verify=False,
raiseMinimumTo=TLSVersion.TLSv1_2,
fixBrokenPeers=True,
)
@classmethod
def from_crawler(cls, crawler):
return cls(crawler.settings, crawler)
def download_request(self, request, spider):
bindaddress = request.meta.get('bindaddress')
timeout = request.meta.get('download_timeout')
maxsize = getattr(spider, 'download_maxsize', self.default_maxsize)
warnsize = getattr(spider, 'download_warnsize', self.default_warnsize)
parts = urlparse(request.url)
remote_host = bindaddress or parts.hostname
remote_port = parts.port or 1965
hostname = HostnameEndpoint(reactor, remote_host, remote_port)
# The recommended helper method for this (optionsForClientTLS) does not
# allow setting up a client context that accepts unverified certificates.
# So we are forced to use the private ClientTLSOptions method instead.
options = ScrapyClientTLSOptions(remote_host, self.context_factory.getContext())
# noinspection PyTypeChecker
endpoint = wrapClientTLS(options, hostname)
logger.debug(f"Creating download request for {request.url}")
protocol = GeminiClientProtocol(request, maxsize, warnsize, timeout)
# If the connection fails (DNS lookup, etc.) propagate the error so
# that scrapy knows the request has completed.
connected = connectProtocol(endpoint, protocol)
connected.addErrback(protocol.finished.errback)
return protocol.finished
def getContext(self) -> SSL.Context:
def always_validate(conn, cert, errno, depth, preverify_ok):
# This function is called to validate the certificate received by
# the other end. OpenSSL calls it multiple times, for each errno
# for each certificate.
# We do not care about certificate authorities or revocation
# lists, we just want to know that the certificate has a valid
# signature and follow the chain back to one which is
# self-signed. We need to protect against forged signatures, but
# not the usual TLS concerns about invalid CAs or revoked
# certificates.
things_are_ok = (
_OPENSSL.X509_V_OK,
_OPENSSL.X509_V_ERR_CERT_NOT_YET_VALID,
_OPENSSL.X509_V_ERR_CERT_HAS_EXPIRED,
_OPENSSL.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT,
_OPENSSL.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN,
)
# TODO can we do this once instead of multiple times?
if errno in things_are_ok and timing_safe_compare(
get_spki_hash(cert.to_cryptography()),
self.expected_spki_hash):
return 1
# TODO: log the details of the error, because otherwise they get
# lost in the PyOpenSSL exception that will eventually be raised
# (possibly OpenSSL.SSL.Error: certificate verify failed)
return 0
ctx = CertificateOptions.getContext(self)
# VERIFY_PEER means we ask the the other end for their certificate.
ctx.set_verify(SSL.VERIFY_PEER, always_validate)
return ctx
def creatorForNetloc(self, hostname, port):
certificateOptions = CertificateOptions(
trustRoot=self._trustRoot)
return PermissiveClientTLSOptions(
hostname.decode("ascii"),
certificateOptions.getContext())
def getContext(self, host, port):
ctx = CertificateOptions.getContext(self)
ctx.set_verify_depth(0)
ctx.set_verify(
OpenSSL.SSL.VERIFY_PEER | OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
self.verifyHostname)
return ctx
class MyWebClientContextFactory(object):
def __init__(self):
self._options = CertificateOptions()
def getContext(self, hostname, port):
return self._options.getContext()
def start_ssl(self):
log.debug("Enabling SSL with PKey: %s, Cert: %s", self.pkey, self.cert)
check_ssl_keys()
with open(configmanager.get_config_dir(self.cert)) as cert:
certificate = Certificate.loadPEM(cert.read()).original
with open(configmanager.get_config_dir(self.pkey)) as pkey:
private_key = KeyPair.load(pkey.read(), FILETYPE_PEM).original
options = CertificateOptions(privateKey=private_key,
certificate=certificate,
method=SSL.SSLv23_METHOD)
options.getContext().set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)
self.socket = reactor.listenSSL(self.port, self.site, options)
log.info("Serving on %s:%s view at https://127.0.0.1:%s", "0.0.0.0",
self.port, self.port)
def get_context_factory(cert_path, pkey_path):
"""OpenSSL context factory.
Generates an OpenSSL context factory using Twisted's CertificateOptions class.
This will keep a server cipher order.
Args:
cert_path (string): The path to the certificate file
pkey_path (string): The path to the private key file
Returns:
twisted.internet.ssl.CertificateOptions: An OpenSSL context factory
"""
with open(cert_path) as cert:
certificate = Certificate.loadPEM(cert.read()).original
with open(pkey_path) as pkey:
private_key = KeyPair.load(pkey.read(), FILETYPE_PEM).original
ciphers = AcceptableCiphers.fromOpenSSLCipherString(TLS_CIPHERS)
cert_options = CertificateOptions(
privateKey=private_key,
certificate=certificate,
raiseMinimumTo=TLSVersion.TLSv1_2,
acceptableCiphers=ciphers,
)
ctx = cert_options.getContext()
ctx.use_certificate_chain_file(cert_path)
ctx.set_options(SSL_OP_NO_RENEGOTIATION)
return cert_options
def test_snimap_default(self):
"""
SNIMap preferentially loads the DEFAULT value from the mapping if it's
present.
"""
options = CertificateOptions()
mapping = {'DEFAULT': options}
sni_map = SNIMap(mapping)
conn = sni_map.serverConnectionForTLS(protocol.Protocol())
self.assertIs(conn.get_context()._obj, options.getContext())
def test_snimap_default(self):
"""
SNIMap preferentially loads the DEFAULT value from the mapping if it's
present.
"""
options = CertificateOptions()
mapping = {'DEFAULT': options}
sni_map = SNIMap(mapping)
conn = sni_map.serverConnectionForTLS(protocol.Protocol())
self.assertIs(conn.get_context()._obj, options.getContext())
class ClientTLSOptionsFactory(object):
"""Factory for Twisted SSLClientConnectionCreators that are used to make connections
to remote servers for federation.
Uses one of two OpenSSL context objects for all connections, depending on whether
we should do SSL certificate verification.
get_options decides whether we should do SSL certificate verification and
constructs an SSLClientConnectionCreator factory accordingly.
"""
def __init__(self):
# Use CA root certs provided by OpenSSL
trust_root = platformTrust()
# "insecurelyLowerMinimumTo" is the argument that will go lower than
# Twisted's default, which is why it is marked as "insecure" (since
# Twisted's defaults are reasonably secure). But, since Twisted is
# moving to TLS 1.2 by default, we want to respect the config option if
# it is set to 1.0 (which the alternate option, raiseMinimumTo, will not
# let us do).
minTLS = TLSVersion.TLSv1_2
self._verify_ssl = CertificateOptions(
trustRoot=trust_root, insecurelyLowerMinimumTo=minTLS
)
self._verify_ssl_context = self._verify_ssl.getContext()
self._verify_ssl_context.set_info_callback(self._context_info_cb)
def get_options(self, host):
ssl_context = self._verify_ssl_context
return SSLClientConnectionCreator(host, ssl_context)
@staticmethod
def _context_info_cb(ssl_connection, where, ret):
"""The 'information callback' for our openssl context object."""
# we assume that the app_data on the connection object has been set to
# a TLSMemoryBIOProtocol object. (This is done by SSLClientConnectionCreator)
tls_protocol = ssl_connection.get_app_data()
try:
# ... we further assume that SSLClientConnectionCreator has set the
# '_synapse_tls_verifier' attribute to a ConnectionVerifier object.
tls_protocol._synapse_tls_verifier.verify_context_info_cb(
ssl_connection, where
)
except: # noqa: E722, taken from the twisted implementation
logger.exception("Error during info_callback")
f = Failure()
tls_protocol.failVerification(f)
def creatorForNetloc(self, hostname, port):
"""Implements the IPolicyForHTTPS interace so that this can be passed
directly to agents.
"""
return self.get_options(hostname)
def test_snimap_makes_its_own_defaults(self):
"""
If passed a mapping without a DEFAULT key, SNIMap will make its own
default context.
"""
options = CertificateOptions()
mapping = {'example.com': options}
sni_map = SNIMap(mapping)
conn = sni_map.serverConnectionForTLS(protocol.Protocol())
self.assertIsNot(conn.get_context(), options.getContext())
self.assertIsNotNone(conn.get_context())
def test_snimap_makes_its_own_defaults(self):
"""
If passed a mapping without a DEFAULT key, SNIMap will make its own
default context.
"""
options = CertificateOptions()
mapping = {'example.com': options}
sni_map = SNIMap(mapping)
conn = sni_map.serverConnectionForTLS(protocol.Protocol())
self.assertIsNot(conn.get_context(), options.getContext())
self.assertIsNotNone(conn.get_context())
def getContext(self):
ctx = CertificateOptions.getContext(self)
# VERIFY_PEER means we ask the the other end for their certificate.
# not adding VERIFY_FAIL_IF_NO_PEER_CERT means it's ok if they don't
# give us one (i.e. if an anonymous client connects to an
# authenticated server). I don't know what VERIFY_CLIENT_ONCE does.
ctx.set_verify(SSL.VERIFY_PEER |
#SSL.VERIFY_FAIL_IF_NO_PEER_CERT |
SSL.VERIFY_CLIENT_ONCE,
alwaysValidate)
return ctx
def getContext(self):
ctx = CertificateOptions.getContext(self)
# VERIFY_PEER means we ask the the other end for their certificate.
# not adding VERIFY_FAIL_IF_NO_PEER_CERT means it's ok if they don't
# give us one (i.e. if an anonymous client connects to an
# authenticated server). I don't know what VERIFY_CLIENT_ONCE does.
ctx.set_verify(
SSL.VERIFY_PEER |
#SSL.VERIFY_FAIL_IF_NO_PEER_CERT |
SSL.VERIFY_CLIENT_ONCE,
alwaysValidate)
return ctx
def start_ssl(self):
check_ssl_keys()
log.debug('Enabling SSL with PKey: %s, Cert: %s', self.pkey, self.cert)
with open(configmanager.get_config_dir(self.cert)) as cert:
certificate = Certificate.loadPEM(cert.read()).original
with open(configmanager.get_config_dir(self.pkey)) as pkey:
private_key = KeyPair.load(pkey.read(), FILETYPE_PEM).original
options = CertificateOptions(privateKey=private_key, certificate=certificate, method=SSL.SSLv23_METHOD)
ctx = options.getContext()
ctx.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)
ctx.use_certificate_chain_file(configmanager.get_config_dir(self.cert))
self.socket = reactor.listenSSL(self.port, self.site, options, interface=self.interface)
ip = self.socket.getHost().host
ip = '[%s]' % ip if is_ipv6(ip) else ip
log.info('Serving at https://%s:%s%s', ip, self.port, self.base)
def start_ssl(self):
check_ssl_keys()
log.debug('Enabling SSL with PKey: %s, Cert: %s', self.pkey, self.cert)
with open(configmanager.get_config_dir(self.cert)) as cert:
certificate = Certificate.loadPEM(cert.read()).original
with open(configmanager.get_config_dir(self.pkey)) as pkey:
private_key = KeyPair.load(pkey.read(), FILETYPE_PEM).original
options = CertificateOptions(privateKey=private_key,
certificate=certificate,
method=SSL.SSLv23_METHOD)
ctx = options.getContext()
ctx.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)
ctx.use_certificate_chain_file(configmanager.get_config_dir(self.cert))
self.socket = reactor.listenSSL(self.port,
self.site,
options,
interface=self.interface)
ip = self.socket.getHost().host
ip = '[%s]' % ip if is_ipv6(ip) else ip
log.info('Serving at https://%s:%s%s', ip, self.port, self.base)
def getContext(self, hostname, port):
opts = CertificateOptions(verify=True, caCerts=[self._cacert])
return opts.getContext()
def creatorForNetloc(self, hostname, port):
options = CertificateOptions(trustRoot=None)
ascii_hostname = hostname.decode("ascii")
context = options.getContext()
return IgnoreHostnameClientTLSOptions(ascii_hostname, context)
def getContext(self, hostname, port):
return CertificateOptions.getContext(self)
def getContext(self, host, port):
ctx = CertificateOptions.getContext(self)
ctx.set_verify_depth(0)
ctx.set_verify(OpenSSL.SSL.VERIFY_PEER | OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, self.verifyHostname)
return ctx
def __init__(self, clientCert=None):
if clientCert is None:
options = CertificateOptions()
else:
options = clientCert.options()
self._ctx = options.getContext()
def getContext(self, hostname, port):
return CertificateOptions.getContext(self)
class ClientTLSOptionsFactory(object):
"""Factory for Twisted SSLClientConnectionCreators that are used to make connections
to remote servers for federation.
Uses one of two OpenSSL context objects for all connections, depending on whether
we should do SSL certificate verification.
get_options decides whether we should do SSL certificate verification and
constructs an SSLClientConnectionCreator factory accordingly.
"""
def __init__(self, config):
self._config = config
# Check if we're using a custom list of a CA certificates
trust_root = config.federation_ca_trust_root
if trust_root is None:
# Use CA root certs provided by OpenSSL
trust_root = platformTrust()
# "insecurelyLowerMinimumTo" is the argument that will go lower than
# Twisted's default, which is why it is marked as "insecure" (since
# Twisted's defaults are reasonably secure). But, since Twisted is
# moving to TLS 1.2 by default, we want to respect the config option if
# it is set to 1.0 (which the alternate option, raiseMinimumTo, will not
# let us do).
minTLS = _TLS_VERSION_MAP[config.federation_client_minimum_tls_version]
self._verify_ssl = CertificateOptions(trustRoot=trust_root,
insecurelyLowerMinimumTo=minTLS)
self._verify_ssl_context = self._verify_ssl.getContext()
self._verify_ssl_context.set_info_callback(self._context_info_cb)
self._no_verify_ssl = CertificateOptions(
insecurelyLowerMinimumTo=minTLS)
self._no_verify_ssl_context = self._no_verify_ssl.getContext()
self._no_verify_ssl_context.set_info_callback(self._context_info_cb)
def get_options(self, host: bytes):
# IPolicyForHTTPS.get_options takes bytes, but we want to compare
# against the str whitelist. The hostnames in the whitelist are already
# IDNA-encoded like the hosts will be here.
ascii_host = host.decode("ascii")
# Check if certificate verification has been enabled
should_verify = self._config.federation_verify_certificates
# Check if we've disabled certificate verification for this host
if should_verify:
for regex in self._config.federation_certificate_verification_whitelist:
if regex.match(ascii_host):
should_verify = False
break
ssl_context = (self._verify_ssl_context
if should_verify else self._no_verify_ssl_context)
return SSLClientConnectionCreator(host, ssl_context, should_verify)
@staticmethod
def _context_info_cb(ssl_connection, where, ret):
"""The 'information callback' for our openssl context object."""
# we assume that the app_data on the connection object has been set to
# a TLSMemoryBIOProtocol object. (This is done by SSLClientConnectionCreator)
tls_protocol = ssl_connection.get_app_data()
try:
# ... we further assume that SSLClientConnectionCreator has set the
# '_synapse_tls_verifier' attribute to a ConnectionVerifier object.
tls_protocol._synapse_tls_verifier.verify_context_info_cb(
ssl_connection, where)
except: # noqa: E722, taken from the twisted implementation
logger.exception("Error during info_callback")
f = Failure()
tls_protocol.failVerification(f)
def creatorForNetloc(self, hostname, port):
"""Implements the IPolicyForHTTPS interace so that this can be passed
directly to agents.
"""
return self.get_options(hostname)
class MyWebClientContextFactory(object):
def __init__(self):
self._options = CertificateOptions()
def getContext(self, hostname, port):
return self._options.getContext()