def __init__(self, config): self._config = config # Check if we're using a custom list of a CA certificates trust_root = config.federation_ca_trust_root if trust_root is None: # Use CA root certs provided by OpenSSL trust_root = platformTrust() # "insecurelyLowerMinimumTo" is the argument that will go lower than # Twisted's default, which is why it is marked as "insecure" (since # Twisted's defaults are reasonably secure). But, since Twisted is # moving to TLS 1.2 by default, we want to respect the config option if # it is set to 1.0 (which the alternate option, raiseMinimumTo, will not # let us do). minTLS = _TLS_VERSION_MAP[config.federation_client_minimum_tls_version] _verify_ssl = CertificateOptions(trustRoot=trust_root, insecurelyLowerMinimumTo=minTLS) self._verify_ssl_context = _verify_ssl.getContext() self._verify_ssl_context.set_info_callback(_context_info_cb) _no_verify_ssl = CertificateOptions(insecurelyLowerMinimumTo=minTLS) self._no_verify_ssl_context = _no_verify_ssl.getContext() self._no_verify_ssl_context.set_info_callback(_context_info_cb)
def start_ssl(self): log.debug("Enabling SSL with PKey: %s, Cert: %s", self.pkey, self.cert) check_ssl_keys() with open(configmanager.get_config_dir(self.cert)) as cert: certificate = Certificate.loadPEM(cert.read()).original with open(configmanager.get_config_dir(self.pkey)) as pkey: private_key = KeyPair.load(pkey.read(), FILETYPE_PEM).original options = CertificateOptions(privateKey=private_key, certificate=certificate, method=SSL.SSLv23_METHOD) options.getContext().set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) self.socket = reactor.listenSSL(self.port, self.site, options) log.info("Serving on %s:%s view at https://127.0.0.1:%s", "0.0.0.0", self.port, self.port)
def creatorForNetloc(self, hostname, port): certificateOptions = CertificateOptions( trustRoot=self._trustRoot) return PermissiveClientTLSOptions( hostname.decode("ascii"), certificateOptions.getContext())
class GeminiDownloadHandler: """ Scrapy download handler for gemini:// scheme URLs. This implementation is *heavily* based on scrapy's HTTP 1.1 and telnet handlers as references. I did, however, make several attempts to simplify the code and use idiomatic twisted patterns. Some integrity checks had to be removed since gemini does not use a Content-Length or checksum. Since scrapy is built around HTTP requests/responses, this code will take the gemini response and generate a pseudo-HTTP response with an equivalent status code and headers. This is necessary to retain compatibility with most of the library's middleware. """ lazy = False def __init__(self, settings, crawler=None): self.crawler = crawler self.default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self.default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') self.fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS') self.context_factory = CertificateOptions( verify=False, raiseMinimumTo=TLSVersion.TLSv1_2, fixBrokenPeers=True, ) @classmethod def from_crawler(cls, crawler): return cls(crawler.settings, crawler) def download_request(self, request, spider): bindaddress = request.meta.get('bindaddress') timeout = request.meta.get('download_timeout') maxsize = getattr(spider, 'download_maxsize', self.default_maxsize) warnsize = getattr(spider, 'download_warnsize', self.default_warnsize) parts = urlparse(request.url) remote_host = bindaddress or parts.hostname remote_port = parts.port or 1965 hostname = HostnameEndpoint(reactor, remote_host, remote_port) # The recommended helper method for this (optionsForClientTLS) does not # allow setting up a client context that accepts unverified certificates. # So we are forced to use the private ClientTLSOptions method instead. options = ScrapyClientTLSOptions(remote_host, self.context_factory.getContext()) # noinspection PyTypeChecker endpoint = wrapClientTLS(options, hostname) logger.debug(f"Creating download request for {request.url}") protocol = GeminiClientProtocol(request, maxsize, warnsize, timeout) # If the connection fails (DNS lookup, etc.) propagate the error so # that scrapy knows the request has completed. connected = connectProtocol(endpoint, protocol) connected.addErrback(protocol.finished.errback) return protocol.finished
def getContext(self) -> SSL.Context: def always_validate(conn, cert, errno, depth, preverify_ok): # This function is called to validate the certificate received by # the other end. OpenSSL calls it multiple times, for each errno # for each certificate. # We do not care about certificate authorities or revocation # lists, we just want to know that the certificate has a valid # signature and follow the chain back to one which is # self-signed. We need to protect against forged signatures, but # not the usual TLS concerns about invalid CAs or revoked # certificates. things_are_ok = ( _OPENSSL.X509_V_OK, _OPENSSL.X509_V_ERR_CERT_NOT_YET_VALID, _OPENSSL.X509_V_ERR_CERT_HAS_EXPIRED, _OPENSSL.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT, _OPENSSL.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN, ) # TODO can we do this once instead of multiple times? if errno in things_are_ok and timing_safe_compare( get_spki_hash(cert.to_cryptography()), self.expected_spki_hash): return 1 # TODO: log the details of the error, because otherwise they get # lost in the PyOpenSSL exception that will eventually be raised # (possibly OpenSSL.SSL.Error: certificate verify failed) return 0 ctx = CertificateOptions.getContext(self) # VERIFY_PEER means we ask the the other end for their certificate. ctx.set_verify(SSL.VERIFY_PEER, always_validate) return ctx
def getContext(self, host, port): ctx = CertificateOptions.getContext(self) ctx.set_verify_depth(0) ctx.set_verify( OpenSSL.SSL.VERIFY_PEER | OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, self.verifyHostname) return ctx
class MyWebClientContextFactory(object): def __init__(self): self._options = CertificateOptions() def getContext(self, hostname, port): return self._options.getContext()
def get_context_factory(cert_path, pkey_path): """OpenSSL context factory. Generates an OpenSSL context factory using Twisted's CertificateOptions class. This will keep a server cipher order. Args: cert_path (string): The path to the certificate file pkey_path (string): The path to the private key file Returns: twisted.internet.ssl.CertificateOptions: An OpenSSL context factory """ with open(cert_path) as cert: certificate = Certificate.loadPEM(cert.read()).original with open(pkey_path) as pkey: private_key = KeyPair.load(pkey.read(), FILETYPE_PEM).original ciphers = AcceptableCiphers.fromOpenSSLCipherString(TLS_CIPHERS) cert_options = CertificateOptions( privateKey=private_key, certificate=certificate, raiseMinimumTo=TLSVersion.TLSv1_2, acceptableCiphers=ciphers, ) ctx = cert_options.getContext() ctx.use_certificate_chain_file(cert_path) ctx.set_options(SSL_OP_NO_RENEGOTIATION) return cert_options
def test_snimap_default(self): """ SNIMap preferentially loads the DEFAULT value from the mapping if it's present. """ options = CertificateOptions() mapping = {'DEFAULT': options} sni_map = SNIMap(mapping) conn = sni_map.serverConnectionForTLS(protocol.Protocol()) self.assertIs(conn.get_context()._obj, options.getContext())
class ClientTLSOptionsFactory(object): """Factory for Twisted SSLClientConnectionCreators that are used to make connections to remote servers for federation. Uses one of two OpenSSL context objects for all connections, depending on whether we should do SSL certificate verification. get_options decides whether we should do SSL certificate verification and constructs an SSLClientConnectionCreator factory accordingly. """ def __init__(self): # Use CA root certs provided by OpenSSL trust_root = platformTrust() # "insecurelyLowerMinimumTo" is the argument that will go lower than # Twisted's default, which is why it is marked as "insecure" (since # Twisted's defaults are reasonably secure). But, since Twisted is # moving to TLS 1.2 by default, we want to respect the config option if # it is set to 1.0 (which the alternate option, raiseMinimumTo, will not # let us do). minTLS = TLSVersion.TLSv1_2 self._verify_ssl = CertificateOptions( trustRoot=trust_root, insecurelyLowerMinimumTo=minTLS ) self._verify_ssl_context = self._verify_ssl.getContext() self._verify_ssl_context.set_info_callback(self._context_info_cb) def get_options(self, host): ssl_context = self._verify_ssl_context return SSLClientConnectionCreator(host, ssl_context) @staticmethod def _context_info_cb(ssl_connection, where, ret): """The 'information callback' for our openssl context object.""" # we assume that the app_data on the connection object has been set to # a TLSMemoryBIOProtocol object. (This is done by SSLClientConnectionCreator) tls_protocol = ssl_connection.get_app_data() try: # ... we further assume that SSLClientConnectionCreator has set the # '_synapse_tls_verifier' attribute to a ConnectionVerifier object. tls_protocol._synapse_tls_verifier.verify_context_info_cb( ssl_connection, where ) except: # noqa: E722, taken from the twisted implementation logger.exception("Error during info_callback") f = Failure() tls_protocol.failVerification(f) def creatorForNetloc(self, hostname, port): """Implements the IPolicyForHTTPS interace so that this can be passed directly to agents. """ return self.get_options(hostname)
def test_snimap_makes_its_own_defaults(self): """ If passed a mapping without a DEFAULT key, SNIMap will make its own default context. """ options = CertificateOptions() mapping = {'example.com': options} sni_map = SNIMap(mapping) conn = sni_map.serverConnectionForTLS(protocol.Protocol()) self.assertIsNot(conn.get_context(), options.getContext()) self.assertIsNotNone(conn.get_context())
def getContext(self): ctx = CertificateOptions.getContext(self) # VERIFY_PEER means we ask the the other end for their certificate. # not adding VERIFY_FAIL_IF_NO_PEER_CERT means it's ok if they don't # give us one (i.e. if an anonymous client connects to an # authenticated server). I don't know what VERIFY_CLIENT_ONCE does. ctx.set_verify(SSL.VERIFY_PEER | #SSL.VERIFY_FAIL_IF_NO_PEER_CERT | SSL.VERIFY_CLIENT_ONCE, alwaysValidate) return ctx
def getContext(self): ctx = CertificateOptions.getContext(self) # VERIFY_PEER means we ask the the other end for their certificate. # not adding VERIFY_FAIL_IF_NO_PEER_CERT means it's ok if they don't # give us one (i.e. if an anonymous client connects to an # authenticated server). I don't know what VERIFY_CLIENT_ONCE does. ctx.set_verify( SSL.VERIFY_PEER | #SSL.VERIFY_FAIL_IF_NO_PEER_CERT | SSL.VERIFY_CLIENT_ONCE, alwaysValidate) return ctx
def start_ssl(self): check_ssl_keys() log.debug('Enabling SSL with PKey: %s, Cert: %s', self.pkey, self.cert) with open(configmanager.get_config_dir(self.cert)) as cert: certificate = Certificate.loadPEM(cert.read()).original with open(configmanager.get_config_dir(self.pkey)) as pkey: private_key = KeyPair.load(pkey.read(), FILETYPE_PEM).original options = CertificateOptions(privateKey=private_key, certificate=certificate, method=SSL.SSLv23_METHOD) ctx = options.getContext() ctx.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) ctx.use_certificate_chain_file(configmanager.get_config_dir(self.cert)) self.socket = reactor.listenSSL(self.port, self.site, options, interface=self.interface) ip = self.socket.getHost().host ip = '[%s]' % ip if is_ipv6(ip) else ip log.info('Serving at https://%s:%s%s', ip, self.port, self.base)
def getContext(self, hostname, port): opts = CertificateOptions(verify=True, caCerts=[self._cacert]) return opts.getContext()
def creatorForNetloc(self, hostname, port): options = CertificateOptions(trustRoot=None) ascii_hostname = hostname.decode("ascii") context = options.getContext() return IgnoreHostnameClientTLSOptions(ascii_hostname, context)
def getContext(self, hostname, port): return CertificateOptions.getContext(self)
def getContext(self, host, port): ctx = CertificateOptions.getContext(self) ctx.set_verify_depth(0) ctx.set_verify(OpenSSL.SSL.VERIFY_PEER | OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, self.verifyHostname) return ctx
def __init__(self, clientCert=None): if clientCert is None: options = CertificateOptions() else: options = clientCert.options() self._ctx = options.getContext()
class ClientTLSOptionsFactory(object): """Factory for Twisted SSLClientConnectionCreators that are used to make connections to remote servers for federation. Uses one of two OpenSSL context objects for all connections, depending on whether we should do SSL certificate verification. get_options decides whether we should do SSL certificate verification and constructs an SSLClientConnectionCreator factory accordingly. """ def __init__(self, config): self._config = config # Check if we're using a custom list of a CA certificates trust_root = config.federation_ca_trust_root if trust_root is None: # Use CA root certs provided by OpenSSL trust_root = platformTrust() # "insecurelyLowerMinimumTo" is the argument that will go lower than # Twisted's default, which is why it is marked as "insecure" (since # Twisted's defaults are reasonably secure). But, since Twisted is # moving to TLS 1.2 by default, we want to respect the config option if # it is set to 1.0 (which the alternate option, raiseMinimumTo, will not # let us do). minTLS = _TLS_VERSION_MAP[config.federation_client_minimum_tls_version] self._verify_ssl = CertificateOptions(trustRoot=trust_root, insecurelyLowerMinimumTo=minTLS) self._verify_ssl_context = self._verify_ssl.getContext() self._verify_ssl_context.set_info_callback(self._context_info_cb) self._no_verify_ssl = CertificateOptions( insecurelyLowerMinimumTo=minTLS) self._no_verify_ssl_context = self._no_verify_ssl.getContext() self._no_verify_ssl_context.set_info_callback(self._context_info_cb) def get_options(self, host: bytes): # IPolicyForHTTPS.get_options takes bytes, but we want to compare # against the str whitelist. The hostnames in the whitelist are already # IDNA-encoded like the hosts will be here. ascii_host = host.decode("ascii") # Check if certificate verification has been enabled should_verify = self._config.federation_verify_certificates # Check if we've disabled certificate verification for this host if should_verify: for regex in self._config.federation_certificate_verification_whitelist: if regex.match(ascii_host): should_verify = False break ssl_context = (self._verify_ssl_context if should_verify else self._no_verify_ssl_context) return SSLClientConnectionCreator(host, ssl_context, should_verify) @staticmethod def _context_info_cb(ssl_connection, where, ret): """The 'information callback' for our openssl context object.""" # we assume that the app_data on the connection object has been set to # a TLSMemoryBIOProtocol object. (This is done by SSLClientConnectionCreator) tls_protocol = ssl_connection.get_app_data() try: # ... we further assume that SSLClientConnectionCreator has set the # '_synapse_tls_verifier' attribute to a ConnectionVerifier object. tls_protocol._synapse_tls_verifier.verify_context_info_cb( ssl_connection, where) except: # noqa: E722, taken from the twisted implementation logger.exception("Error during info_callback") f = Failure() tls_protocol.failVerification(f) def creatorForNetloc(self, hostname, port): """Implements the IPolicyForHTTPS interace so that this can be passed directly to agents. """ return self.get_options(hostname)