Example #1
0
    def __init__(self, config):
        self._config = config

        # Check if we're using a custom list of a CA certificates
        trust_root = config.federation_ca_trust_root
        if trust_root is None:
            # Use CA root certs provided by OpenSSL
            trust_root = platformTrust()

        # "insecurelyLowerMinimumTo" is the argument that will go lower than
        # Twisted's default, which is why it is marked as "insecure" (since
        # Twisted's defaults are reasonably secure). But, since Twisted is
        # moving to TLS 1.2 by default, we want to respect the config option if
        # it is set to 1.0 (which the alternate option, raiseMinimumTo, will not
        # let us do).
        minTLS = _TLS_VERSION_MAP[config.federation_client_minimum_tls_version]

        _verify_ssl = CertificateOptions(trustRoot=trust_root,
                                         insecurelyLowerMinimumTo=minTLS)
        self._verify_ssl_context = _verify_ssl.getContext()
        self._verify_ssl_context.set_info_callback(_context_info_cb)

        _no_verify_ssl = CertificateOptions(insecurelyLowerMinimumTo=minTLS)
        self._no_verify_ssl_context = _no_verify_ssl.getContext()
        self._no_verify_ssl_context.set_info_callback(_context_info_cb)
Example #2
0
    def start_ssl(self):
        log.debug("Enabling SSL with PKey: %s, Cert: %s", self.pkey, self.cert)
        check_ssl_keys()

        with open(configmanager.get_config_dir(self.cert)) as cert:
            certificate = Certificate.loadPEM(cert.read()).original
        with open(configmanager.get_config_dir(self.pkey)) as pkey:
            private_key = KeyPair.load(pkey.read(), FILETYPE_PEM).original
        options = CertificateOptions(privateKey=private_key, certificate=certificate, method=SSL.SSLv23_METHOD)
        options.getContext().set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)

        self.socket = reactor.listenSSL(self.port, self.site, options)
        log.info("Serving on %s:%s view at https://127.0.0.1:%s", "0.0.0.0", self.port, self.port)
Example #3
0
    def creatorForNetloc(self, hostname, port):
        certificateOptions = CertificateOptions(
            trustRoot=self._trustRoot)

        return PermissiveClientTLSOptions(
            hostname.decode("ascii"),
            certificateOptions.getContext())
Example #4
0
class GeminiDownloadHandler:
    """
    Scrapy download handler for gemini:// scheme URLs.

    This implementation is *heavily* based on scrapy's HTTP 1.1 and telnet
    handlers as references. I did, however, make several attempts to simplify
    the code and use idiomatic twisted patterns. Some integrity checks had to
    be removed since gemini does not use a Content-Length or checksum. Since
    scrapy is built around HTTP requests/responses, this code will take the
    gemini response and generate a pseudo-HTTP response with an equivalent
    status code and headers. This is necessary to retain compatibility with
    most of the library's middleware.
    """
    lazy = False

    def __init__(self, settings, crawler=None):
        self.crawler = crawler

        self.default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
        self.default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
        self.fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')

        self.context_factory = CertificateOptions(
            verify=False,
            raiseMinimumTo=TLSVersion.TLSv1_2,
            fixBrokenPeers=True,
        )

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler.settings, crawler)

    def download_request(self, request, spider):
        bindaddress = request.meta.get('bindaddress')
        timeout = request.meta.get('download_timeout')

        maxsize = getattr(spider, 'download_maxsize', self.default_maxsize)
        warnsize = getattr(spider, 'download_warnsize', self.default_warnsize)

        parts = urlparse(request.url)
        remote_host = bindaddress or parts.hostname
        remote_port = parts.port or 1965

        hostname = HostnameEndpoint(reactor, remote_host, remote_port)
        # The recommended helper method for this (optionsForClientTLS) does not
        # allow setting up a client context that accepts unverified certificates.
        # So we are forced to use the private ClientTLSOptions method instead.
        options = ScrapyClientTLSOptions(remote_host, self.context_factory.getContext())
        # noinspection PyTypeChecker
        endpoint = wrapClientTLS(options, hostname)

        logger.debug(f"Creating download request for {request.url}")
        protocol = GeminiClientProtocol(request, maxsize, warnsize, timeout)

        # If the connection fails (DNS lookup, etc.) propagate the error so
        # that scrapy knows the request has completed.
        connected = connectProtocol(endpoint, protocol)
        connected.addErrback(protocol.finished.errback)

        return protocol.finished
Example #5
0
    def getContext(self) -> SSL.Context:
        def always_validate(conn, cert, errno, depth, preverify_ok):
            # This function is called to validate the certificate received by
            # the other end. OpenSSL calls it multiple times, for each errno
            # for each certificate.

            # We do not care about certificate authorities or revocation
            # lists, we just want to know that the certificate has a valid
            # signature and follow the chain back to one which is
            # self-signed. We need to protect against forged signatures, but
            # not the usual TLS concerns about invalid CAs or revoked
            # certificates.
            things_are_ok = (
                _OPENSSL.X509_V_OK,
                _OPENSSL.X509_V_ERR_CERT_NOT_YET_VALID,
                _OPENSSL.X509_V_ERR_CERT_HAS_EXPIRED,
                _OPENSSL.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT,
                _OPENSSL.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN,
            )
            # TODO can we do this once instead of multiple times?
            if errno in things_are_ok and timing_safe_compare(
                    get_spki_hash(cert.to_cryptography()),
                    self.expected_spki_hash):
                return 1
            # TODO: log the details of the error, because otherwise they get
            # lost in the PyOpenSSL exception that will eventually be raised
            # (possibly OpenSSL.SSL.Error: certificate verify failed)
            return 0

        ctx = CertificateOptions.getContext(self)

        # VERIFY_PEER means we ask the the other end for their certificate.
        ctx.set_verify(SSL.VERIFY_PEER, always_validate)
        return ctx
    def creatorForNetloc(self, hostname, port):
        certificateOptions = CertificateOptions(
            trustRoot=self._trustRoot)

        return PermissiveClientTLSOptions(
            hostname.decode("ascii"),
            certificateOptions.getContext())
Example #7
0
 def getContext(self, host, port):
     ctx = CertificateOptions.getContext(self)
     ctx.set_verify_depth(0)
     ctx.set_verify(
         OpenSSL.SSL.VERIFY_PEER | OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
         self.verifyHostname)
     return ctx
Example #8
0
class MyWebClientContextFactory(object):

    def __init__(self):
        self._options = CertificateOptions()

    def getContext(self, hostname, port):
        return self._options.getContext()
Example #9
0
    def start_ssl(self):
        log.debug("Enabling SSL with PKey: %s, Cert: %s", self.pkey, self.cert)
        check_ssl_keys()

        with open(configmanager.get_config_dir(self.cert)) as cert:
            certificate = Certificate.loadPEM(cert.read()).original
        with open(configmanager.get_config_dir(self.pkey)) as pkey:
            private_key = KeyPair.load(pkey.read(), FILETYPE_PEM).original
        options = CertificateOptions(privateKey=private_key,
                                     certificate=certificate,
                                     method=SSL.SSLv23_METHOD)
        options.getContext().set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)

        self.socket = reactor.listenSSL(self.port, self.site, options)
        log.info("Serving on %s:%s view at https://127.0.0.1:%s", "0.0.0.0",
                 self.port, self.port)
Example #10
0
def get_context_factory(cert_path, pkey_path):
    """OpenSSL context factory.

    Generates an OpenSSL context factory using Twisted's CertificateOptions class.
    This will keep a server cipher order.

    Args:
        cert_path (string): The path to the certificate file
        pkey_path (string): The path to the private key file

    Returns:
        twisted.internet.ssl.CertificateOptions: An OpenSSL context factory
    """

    with open(cert_path) as cert:
        certificate = Certificate.loadPEM(cert.read()).original
    with open(pkey_path) as pkey:
        private_key = KeyPair.load(pkey.read(), FILETYPE_PEM).original
    ciphers = AcceptableCiphers.fromOpenSSLCipherString(TLS_CIPHERS)
    cert_options = CertificateOptions(
        privateKey=private_key,
        certificate=certificate,
        raiseMinimumTo=TLSVersion.TLSv1_2,
        acceptableCiphers=ciphers,
    )
    ctx = cert_options.getContext()
    ctx.use_certificate_chain_file(cert_path)
    ctx.set_options(SSL_OP_NO_RENEGOTIATION)

    return cert_options
Example #11
0
    def test_snimap_default(self):
        """
        SNIMap preferentially loads the DEFAULT value from the mapping if it's
        present.
        """
        options = CertificateOptions()
        mapping = {'DEFAULT': options}
        sni_map = SNIMap(mapping)

        conn = sni_map.serverConnectionForTLS(protocol.Protocol())
        self.assertIs(conn.get_context()._obj, options.getContext())
Example #12
0
    def test_snimap_default(self):
        """
        SNIMap preferentially loads the DEFAULT value from the mapping if it's
        present.
        """
        options = CertificateOptions()
        mapping = {'DEFAULT': options}
        sni_map = SNIMap(mapping)

        conn = sni_map.serverConnectionForTLS(protocol.Protocol())
        self.assertIs(conn.get_context()._obj, options.getContext())
Example #13
0
class ClientTLSOptionsFactory(object):
    """Factory for Twisted SSLClientConnectionCreators that are used to make connections
    to remote servers for federation.
    Uses one of two OpenSSL context objects for all connections, depending on whether
    we should do SSL certificate verification.
    get_options decides whether we should do SSL certificate verification and
    constructs an SSLClientConnectionCreator factory accordingly.
    """

    def __init__(self):
        # Use CA root certs provided by OpenSSL
        trust_root = platformTrust()

        # "insecurelyLowerMinimumTo" is the argument that will go lower than
        # Twisted's default, which is why it is marked as "insecure" (since
        # Twisted's defaults are reasonably secure). But, since Twisted is
        # moving to TLS 1.2 by default, we want to respect the config option if
        # it is set to 1.0 (which the alternate option, raiseMinimumTo, will not
        # let us do).
        minTLS = TLSVersion.TLSv1_2

        self._verify_ssl = CertificateOptions(
            trustRoot=trust_root, insecurelyLowerMinimumTo=minTLS
        )
        self._verify_ssl_context = self._verify_ssl.getContext()
        self._verify_ssl_context.set_info_callback(self._context_info_cb)

    def get_options(self, host):
        ssl_context = self._verify_ssl_context

        return SSLClientConnectionCreator(host, ssl_context)

    @staticmethod
    def _context_info_cb(ssl_connection, where, ret):
        """The 'information callback' for our openssl context object."""
        # we assume that the app_data on the connection object has been set to
        # a TLSMemoryBIOProtocol object. (This is done by SSLClientConnectionCreator)
        tls_protocol = ssl_connection.get_app_data()
        try:
            # ... we further assume that SSLClientConnectionCreator has set the
            # '_synapse_tls_verifier' attribute to a ConnectionVerifier object.
            tls_protocol._synapse_tls_verifier.verify_context_info_cb(
                ssl_connection, where
            )
        except:  # noqa: E722, taken from the twisted implementation
            logger.exception("Error during info_callback")
            f = Failure()
            tls_protocol.failVerification(f)

    def creatorForNetloc(self, hostname, port):
        """Implements the IPolicyForHTTPS interace so that this can be passed
        directly to agents.
        """
        return self.get_options(hostname)
Example #14
0
    def test_snimap_makes_its_own_defaults(self):
        """
        If passed a mapping without a DEFAULT key, SNIMap will make its own
        default context.
        """
        options = CertificateOptions()
        mapping = {'example.com': options}
        sni_map = SNIMap(mapping)

        conn = sni_map.serverConnectionForTLS(protocol.Protocol())
        self.assertIsNot(conn.get_context(), options.getContext())
        self.assertIsNotNone(conn.get_context())
Example #15
0
    def test_snimap_makes_its_own_defaults(self):
        """
        If passed a mapping without a DEFAULT key, SNIMap will make its own
        default context.
        """
        options = CertificateOptions()
        mapping = {'example.com': options}
        sni_map = SNIMap(mapping)

        conn = sni_map.serverConnectionForTLS(protocol.Protocol())
        self.assertIsNot(conn.get_context(), options.getContext())
        self.assertIsNotNone(conn.get_context())
Example #16
0
    def getContext(self):
        ctx = CertificateOptions.getContext(self)

        # VERIFY_PEER means we ask the the other end for their certificate.
        # not adding VERIFY_FAIL_IF_NO_PEER_CERT means it's ok if they don't
        # give us one (i.e. if an anonymous client connects to an
        # authenticated server). I don't know what VERIFY_CLIENT_ONCE does.
        ctx.set_verify(SSL.VERIFY_PEER |
                       #SSL.VERIFY_FAIL_IF_NO_PEER_CERT |
                       SSL.VERIFY_CLIENT_ONCE,
                       alwaysValidate)
        return ctx
Example #17
0
    def getContext(self):
        ctx = CertificateOptions.getContext(self)

        # VERIFY_PEER means we ask the the other end for their certificate.
        # not adding VERIFY_FAIL_IF_NO_PEER_CERT means it's ok if they don't
        # give us one (i.e. if an anonymous client connects to an
        # authenticated server). I don't know what VERIFY_CLIENT_ONCE does.
        ctx.set_verify(
            SSL.VERIFY_PEER |
            #SSL.VERIFY_FAIL_IF_NO_PEER_CERT |
            SSL.VERIFY_CLIENT_ONCE,
            alwaysValidate)
        return ctx
Example #18
0
    def start_ssl(self):
        check_ssl_keys()
        log.debug('Enabling SSL with PKey: %s, Cert: %s', self.pkey, self.cert)

        with open(configmanager.get_config_dir(self.cert)) as cert:
            certificate = Certificate.loadPEM(cert.read()).original
        with open(configmanager.get_config_dir(self.pkey)) as pkey:
            private_key = KeyPair.load(pkey.read(), FILETYPE_PEM).original
        options = CertificateOptions(privateKey=private_key, certificate=certificate, method=SSL.SSLv23_METHOD)
        ctx = options.getContext()
        ctx.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)
        ctx.use_certificate_chain_file(configmanager.get_config_dir(self.cert))

        self.socket = reactor.listenSSL(self.port, self.site, options, interface=self.interface)
        ip = self.socket.getHost().host
        ip = '[%s]' % ip if is_ipv6(ip) else ip
        log.info('Serving at https://%s:%s%s', ip, self.port, self.base)
Example #19
0
    def start_ssl(self):
        check_ssl_keys()
        log.debug('Enabling SSL with PKey: %s, Cert: %s', self.pkey, self.cert)

        with open(configmanager.get_config_dir(self.cert)) as cert:
            certificate = Certificate.loadPEM(cert.read()).original
        with open(configmanager.get_config_dir(self.pkey)) as pkey:
            private_key = KeyPair.load(pkey.read(), FILETYPE_PEM).original
        options = CertificateOptions(privateKey=private_key,
                                     certificate=certificate,
                                     method=SSL.SSLv23_METHOD)
        ctx = options.getContext()
        ctx.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)
        ctx.use_certificate_chain_file(configmanager.get_config_dir(self.cert))

        self.socket = reactor.listenSSL(self.port,
                                        self.site,
                                        options,
                                        interface=self.interface)
        ip = self.socket.getHost().host
        ip = '[%s]' % ip if is_ipv6(ip) else ip
        log.info('Serving at https://%s:%s%s', ip, self.port, self.base)
Example #20
0
 def getContext(self, hostname, port):
     opts = CertificateOptions(verify=True, caCerts=[self._cacert])
     return opts.getContext()
Example #21
0
        def creatorForNetloc(self, hostname, port):
            options = CertificateOptions(trustRoot=None)
            ascii_hostname = hostname.decode("ascii")
            context = options.getContext()

            return IgnoreHostnameClientTLSOptions(ascii_hostname, context)
Example #22
0
 def getContext(self, hostname, port):
     return CertificateOptions.getContext(self)
 def getContext(self, host, port):
     ctx = CertificateOptions.getContext(self)
     ctx.set_verify_depth(0)
     ctx.set_verify(OpenSSL.SSL.VERIFY_PEER | OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, self.verifyHostname)
     return ctx
Example #24
0
 def __init__(self, clientCert=None):
     if clientCert is None:
         options = CertificateOptions()
     else:
         options = clientCert.options()
     self._ctx = options.getContext()
 def getContext(self, hostname, port):
     return CertificateOptions.getContext(self)
class ClientTLSOptionsFactory(object):
    """Factory for Twisted SSLClientConnectionCreators that are used to make connections
    to remote servers for federation.

    Uses one of two OpenSSL context objects for all connections, depending on whether
    we should do SSL certificate verification.

    get_options decides whether we should do SSL certificate verification and
    constructs an SSLClientConnectionCreator factory accordingly.
    """
    def __init__(self, config):
        self._config = config

        # Check if we're using a custom list of a CA certificates
        trust_root = config.federation_ca_trust_root
        if trust_root is None:
            # Use CA root certs provided by OpenSSL
            trust_root = platformTrust()

        # "insecurelyLowerMinimumTo" is the argument that will go lower than
        # Twisted's default, which is why it is marked as "insecure" (since
        # Twisted's defaults are reasonably secure). But, since Twisted is
        # moving to TLS 1.2 by default, we want to respect the config option if
        # it is set to 1.0 (which the alternate option, raiseMinimumTo, will not
        # let us do).
        minTLS = _TLS_VERSION_MAP[config.federation_client_minimum_tls_version]

        self._verify_ssl = CertificateOptions(trustRoot=trust_root,
                                              insecurelyLowerMinimumTo=minTLS)
        self._verify_ssl_context = self._verify_ssl.getContext()
        self._verify_ssl_context.set_info_callback(self._context_info_cb)

        self._no_verify_ssl = CertificateOptions(
            insecurelyLowerMinimumTo=minTLS)
        self._no_verify_ssl_context = self._no_verify_ssl.getContext()
        self._no_verify_ssl_context.set_info_callback(self._context_info_cb)

    def get_options(self, host: bytes):

        # IPolicyForHTTPS.get_options takes bytes, but we want to compare
        # against the str whitelist. The hostnames in the whitelist are already
        # IDNA-encoded like the hosts will be here.
        ascii_host = host.decode("ascii")

        # Check if certificate verification has been enabled
        should_verify = self._config.federation_verify_certificates

        # Check if we've disabled certificate verification for this host
        if should_verify:
            for regex in self._config.federation_certificate_verification_whitelist:
                if regex.match(ascii_host):
                    should_verify = False
                    break

        ssl_context = (self._verify_ssl_context
                       if should_verify else self._no_verify_ssl_context)

        return SSLClientConnectionCreator(host, ssl_context, should_verify)

    @staticmethod
    def _context_info_cb(ssl_connection, where, ret):
        """The 'information callback' for our openssl context object."""
        # we assume that the app_data on the connection object has been set to
        # a TLSMemoryBIOProtocol object. (This is done by SSLClientConnectionCreator)
        tls_protocol = ssl_connection.get_app_data()
        try:
            # ... we further assume that SSLClientConnectionCreator has set the
            # '_synapse_tls_verifier' attribute to a ConnectionVerifier object.
            tls_protocol._synapse_tls_verifier.verify_context_info_cb(
                ssl_connection, where)
        except:  # noqa: E722, taken from the twisted implementation
            logger.exception("Error during info_callback")
            f = Failure()
            tls_protocol.failVerification(f)

    def creatorForNetloc(self, hostname, port):
        """Implements the IPolicyForHTTPS interace so that this can be passed
        directly to agents.
        """
        return self.get_options(hostname)
Example #27
0
class MyWebClientContextFactory(object):
    def __init__(self):
        self._options = CertificateOptions()

    def getContext(self, hostname, port):
        return self._options.getContext()