Example #1
0
def _idna_encode(value):
    # Retain prefixes '*.' for common/alt names and '.' for name constraints
    for prefix in ['*.', '.']:
        if value.startswith(prefix):
            value = value[len(prefix):]
            return prefix + idna.encode(value).decode("ascii")
    return idna.encode(value).decode("ascii")
Example #2
0
    def __init__(self, value):
        if not isinstance(value, six.text_type):
            raise TypeError("value must be a unicode string")

        parsed = urllib_parse.urlparse(value)
        if not parsed.hostname:
            netloc = ""
        elif parsed.port:
            netloc = (
                idna.encode(parsed.hostname) +
                ":{0}".format(parsed.port).encode("ascii")
            ).decode("ascii")
        else:
            netloc = idna.encode(parsed.hostname).decode("ascii")

        # Note that building a URL in this fashion means it should be
        # semantically indistinguishable from the original but is not
        # guaranteed to be exactly the same.
        uri = urllib_parse.urlunparse((
            parsed.scheme,
            netloc,
            parsed.path,
            parsed.params,
            parsed.query,
            parsed.fragment
        )).encode("ascii")

        self._value = value
        self._encoded = uri
Example #3
0
 def idna_encode(name):
     """
     Borrowed wholesale from the Python Cryptography Project. It turns out
     that we can't just safely call `idna.encode`: it can explode for
     wildcard names. This avoids that problem.
     """
     for prefix in [u'*.', u'.']:
         if name.startswith(prefix):
             name = name[len(prefix):]
             return prefix.encode('ascii') + idna.encode(name)
     return idna.encode(name)
Example #4
0
 def requires_non_ascii(self):
     """
     Can the address be converted to an ASCII compatible encoding?
     """
     if not is_pure_ascii(self.mailbox):
         return True
     if not is_pure_ascii(self.hostname):
         try:
             idna.encode(self.hostname)
         except idna.IDNAError:
             return True
     return False
Example #5
0
 def get_object(self, **kwargs):
     qs = self.get_queryset(**kwargs)
     domain = self.kwargs['domain']
     # support IDN domains, i.e. accept Unicode encoding too
     try:
         import idna
         if domain.startswith("*."):
             ace_domain = "*." + idna.encode(domain[2:]).decode("utf-8", "strict")
         else:
             ace_domain = idna.encode(domain).decode("utf-8", "strict")
     except:
         ace_domain = domain
     return get_object_or_404(qs, domain=ace_domain)
Example #6
0
    def __init__(self, raw_display_name=None, raw_addr_spec=None,
                 _display_name=None, _mailbox=None, _hostname=None):
        raw_display_name = _to_parser_input(raw_display_name)
        raw_addr_spec = _to_parser_input(raw_addr_spec)

        if raw_display_name and raw_addr_spec:
            mailbox = addr_spec_parser.parse(raw_addr_spec, lexer.clone())
            self._display_name = _to_text(raw_display_name)
            self._mailbox = _to_text(mailbox.local_part)
            self._hostname = _to_text(mailbox.domain)

        elif raw_display_name:
            mailbox = mailbox_parser.parse(raw_display_name, lexer.clone())
            self._display_name = _to_text(mailbox.display_name)
            self._mailbox = _to_text(mailbox.local_part)
            self._hostname = _to_text(mailbox.domain)

        elif raw_addr_spec:
            mailbox = addr_spec_parser.parse(raw_addr_spec, lexer.clone())
            self._display_name = u''
            self._mailbox = _to_text(mailbox.local_part)
            self._hostname = _to_text(mailbox.domain)

        elif _mailbox and _hostname:
            self._display_name = _display_name or u''
            self._mailbox = _mailbox
            self._hostname = _hostname

        else:
            raise SyntaxError('failed to create EmailAddress: bad parameters')

        # Convert display name to decoded unicode string.
        if (self._display_name.startswith('=?') and
                self._display_name.endswith('?=')):
            self._display_name = mime_to_unicode(self._display_name)
        if (self._display_name.startswith('"') and
                self._display_name.endswith('"') and
                len(self._display_name) > 2):
            self._display_name = smart_unquote(self._display_name)

        # Convert hostname to lowercase unicode string.
        self._hostname = self._hostname.lower()
        if self._hostname.startswith('xn--') or '.xn--' in self._hostname:
            self._hostname = idna.decode(self._hostname)
        if not is_pure_ascii(self._hostname):
            idna.encode(self._hostname)

        assert isinstance(self._display_name, six.text_type)
        assert isinstance(self._mailbox, six.text_type)
        assert isinstance(self._hostname, six.text_type)
Example #7
0
    def full_spec(self):
        """
        Returns an ASCII-compatable encoding of an email address or raises a
        ValueError. Display name and domain parts will be converted to
        ASCII-compatable encoding. The transformed address will be ASCII-only
        and RFC-2822 compliant.

           >>> EmailAddress("Ev K", "*****@*****.**").full_spec()
           'Ev K <*****@*****.**>'
           >>> EmailAddress("Жека", "*****@*****.**").full_spec()
           '=?utf-8?b?0JbQtdC60LA=?= <*****@*****.**>'
        """
        if not is_pure_ascii(self.mailbox):
            raise ValueError(
                'address {} has no ASCII-compatable encoding'.format(
                    self.address.encode('utf-8')))
        if not is_pure_ascii(self.hostname):
            try:
                ace_hostname = idna.encode(self.hostname)
            except idna.IDNAError:
                raise ValueError(
                    'address {} has no ASCII-compatable encoding'.format(
                        self.address.encode('utf-8')))
        else:
            ace_hostname = self.hostname
        if self.display_name:
            ace_display_name = smart_quote(
                encode_string(None,
                              self.display_name,
                              maxlinelen=MAX_ADDRESS_LENGTH))
            return u'{} <{}@{}>'.format(ace_display_name, self.mailbox,
                                        ace_hostname)
        return u'{}@{}'.format(self.mailbox, ace_hostname)
Example #8
0
def get_certificate(hostname, port):
    hostname_idna = idna.encode(hostname)
    sock = socket()
    sock.settimeout(5)

    sock.connect((hostname, 443))
    sock.setblocking(1)

    
    peername = sock.getpeername()
    ctx = SSL.Context(SSL.SSLv23_METHOD) # most compatible
    ctx.check_hostname = False
    ctx.verify_mode = SSL.VERIFY_NONE

    sock_ssl = SSL.Connection(ctx, sock)
    sock_ssl.set_connect_state()
    sock_ssl.set_tlsext_host_name(hostname_idna)
    sock_ssl.do_handshake()
    cert = sock_ssl.get_peer_certificate()
    crypto_cert = cert.to_cryptography()

    import pprint
    pprint.pprint(crypto_cert)

    sock_ssl.close()
    sock.close()

    return HostInfo(cert=crypto_cert, peername=peername, hostname=hostname)
Example #9
0
    def gen(self):
        full_url = self.full_url

        for char in self.domain:
            if char in self.glyphs:
                full_url = full_url.replace(
                    char, "[{} x {}]".format(char, len(self.glyphs[char])))
                special_chars = self.glyphs[char]
                for special_char in special_chars:
                    full_domain = "{}.{}".format(
                        idna.encode(self.domain.replace(
                            char, "{}".format(special_char)),
                                    uts46=True).decode('utf8'),
                        self.url_extract.suffix)
                    ip = domains_tool.Domain.check_ip(full_domain)
                    if ip:
                        hopp = domains_tool.Domain.check_hopp(full_domain)
                    else:
                        hopp = '?'

                    if self.show_all:
                        print("sign: {}, {}.{} => {} [{}], {}".format(
                            char,
                            self.domain.replace(char,
                                                "{}".format(special_char)),
                            self.url_extract.suffix, full_domain, ip, hopp))
                    elif ip:
                        print("sign: {}, {}.{} => {} [{}], {}".format(
                            char,
                            self.domain.replace(char,
                                                "{}".format(special_char)),
                            self.url_extract.suffix, full_domain, ip, hopp))
Example #10
0
def patched_match_hostname(cert, hostname):
    try:
        hostname = idna.encode(hostname, uts46=True).decode('ascii')
    except UnicodeError:
        hostname = hostname.encode('idna').decode('ascii')

    return real_match_hostname(cert, hostname)
Example #11
0
def normalise_domain(domain):
    domain = domain.lower()

    # Handle users copying domains with the scheme attached.
    # Only allow these two schemes - GPC is for HTTP(s).
    if domain.startswith('https://'):
        domain = domain[8:]
    elif domain.startswith('http://'):
        domain = domain[7:]

    # Similar to handling schemes, handle one slash at the end of the domain.
    if domain.endswith('/'):
        domain = domain[:-1]

    # Strip any optional trailing period from the domain.
    if domain.endswith('.'):
        domain = domain[:-1]

    try:
        # Convert to and from IDNA encoding with compatibility mapping enabled to normalise.
        domain = idna.decode(idna.encode(domain, uts46=True))
    except idna.IDNAError:
        # Ignore IDNA errors and return the domain without IDNA normalisation.
        # Any IDNA error will cause check_domain() to fail anyway.
        pass

    return domain
Example #12
0
def dmarc_fetch_public_suffix_list():
    """
    Fetch the list from the configured URL and parse it leaving out comments,
    empty lines and invalid lines.

    """
    public_suffix_list = []
    r = http_get(settings.PUBLIC_SUFFIX_LIST_URL)
    if not r:
        return public_suffix_list

    lines = r.text.split("\n")
    for line in lines:
        labels = []
        line = line.rstrip()
        if line and not line.startswith(("//", " ")):
            exception = False
            if line.startswith("!"):
                exception = True
                line = line[1:]
            # Convert to punnycode.
            # This is how we are going to compare domain names later.
            try:
                for label in line.split('.'):
                    if label == '*':
                        labels.append(label)
                    else:
                        labels.append(idna.encode(label).decode('ascii'))
                public_suffix_list.append((labels[::-1], exception))
            except (UnicodeError, ValueError, idna.IDNAError):
                pass
    return public_suffix_list
Example #13
0
def patched_match_hostname(cert, hostname):
    try:
        hostname = idna.encode(hostname, uts46=True).decode('ascii')
    except UnicodeError:
        hostname = hostname.encode('idna').decode('ascii')

    return real_match_hostname(cert, hostname)
Example #14
0
def normalize_url(url):
    purl = rfc3986.urlparse(url)

    if purl.scheme is None and purl.host is None and purl.path is not None:
        # no protocol, no // : it is a path according to the rfc3986
        # but we know it is a host
        purl = rfc3986.urlparse('//' + url)

    if purl.scheme is None:
        # The url starts with //
        # Add https (or http for .onion or i2p TLD)
        if model.host_use_http(purl.host):
            purl = purl.copy_with(scheme='http')
        else:
            purl = purl.copy_with(scheme='https')

    # first normalization
    # * idna encoding to avoid misleading host
    # * remove query and fragment
    # * remove empty path
    purl = purl.copy_with(scheme=purl.scheme.lower(),
                          host=idna.encode(purl.host).decode('utf-8').lower(),
                          path='' if purl.path == '/' else purl.path,
                          query=None,
                          fragment=None)

    # only https (exception: http for .onion and .i2p TLD)
    if (purl.scheme == 'https' and not model.host_use_http(purl.host)) or\
       (purl.scheme == 'http' and model.host_use_http(purl.host)):
        # normalize the URL
        return rfc3986.normalize_uri(purl.geturl())

    #
    return None
Example #15
0
def request_certificate():
    try:
        domain = request.form.get('domain', None)
        if domain is not None:
            domain = domain.strip()
        is_ok = Domain.query.filter_by(name=domain).filter_by(belongs=session['username']).first()
        if is_ok is None:
            return jsonify({'error': 'domain non exist!'}), 404
        if not is_ok.validated:
            return jsonify({'error': 'domain not validated'}), 404
        latest_cert = Certificates.query.filter_by(domain=domain).order_by(desc(Certificates.create_time)).first()
        if latest_cert is not None and datetime.now() - latest_cert.create_time < timedelta(days=60):
            return jsonify({'error': 'no reason to issue new certificate, validity over 60 days'}), 429
        privkey = request.form.get('privkey', None)  # PEM encoded
        if latest_cert is not None and privkey is None:
            privkey = latest_cert.certificate_key
        client = Client(Account(KEY1, '*****@*****.**'))
        client.account_register()
        # IDN encode
        domain = idna.encode(domain).decode()
        cert, cert_key = client.obtain_certificate([domain, "www." + domain], privkey)
        cert_now = Certificates(domain, cert, cert_key)
        db.session.add(cert_now)
        db.session.commit()
        return jsonify({}), 200
    except ValueError as e:
        return jsonify({'error': str(e)}), 403
Example #16
0
def fqdn_check(match: regex.Match):
    mnfo = match.groupdict()
    valu = mnfo.get('valu')

    nval = unicodedata.normalize('NFKC', valu)
    nval = regex.sub(udots, '.', nval)
    nval = nval.strip().strip('.')

    try:
        idna.encode(nval, uts46=True).decode('utf8')
    except idna.IDNAError:
        try:
            nval.encode('idna').decode('utf8').lower()
        except UnicodeError:
            return None, {}
    return valu, {}
Example #17
0
def get_expire_date(hostname):

    ## Encode in domains with IDN compilance
    hostname_idna = idna.encode(hostname)

    ## Open and connect socket
    sock = socket()
    sock.connect((hostname, 443))

    ## Add context
    contex = SSL.Context(SSL.SSLv23_METHOD)
    contex.check_hostname = False
    contex.verify_mode = SSL.VERIFY_NONE

    ## HTTP request with SSL
    sock_ssl = SSL.Connection(contex, sock)
    sock_ssl.set_connect_state()
    sock_ssl.set_tlsext_host_name(hostname_idna)
    sock_ssl.do_handshake()

    ## Get certificate infos
    cert = sock_ssl.get_peer_certificate()
    crypto_cert = cert.to_cryptography()

    ## Close conection SSL
    sock_ssl.close()

    ## Close socket
    sock.close()

    ## Get expire date from certificate
    return crypto_cert.not_valid_after
Example #18
0
    def get_surt_host(url):
        try:
            host = urlparse(url).hostname
        except:
            # self.get_logger().debug("Failed to parse URL {}: {}".format(url, e))
            return None
        if host is None or ExtractHostLinksJob.ip_pattern.match(host):
            return None
        host = host.strip().lower()
        if len(host) < 1 or len(host) > 253:
            return None
        parts = host.split('.')
        if parts[-1] == '':
            # trailing dot is allowed, strip it
            parts = parts[0:-1]
        if parts[0] == 'www' and len(parts) > 1:
            # strip leading 'www' to reduce number of "duplicate" hosts
            parts = parts[1:]
        for i in range(0, len(parts)):
            part = parts[i]
            if not ExtractHostLinksJob.host_part_pattern.match(part):
                try:
                    idn = idna.encode(part).decode('ascii')
                except (idna.IDNAError, UnicodeDecodeError, IndexError,
                        UnicodeEncodeError, Exception):
                    # self.get_logger().debug("Invalid host name: {}".format(url))
                    return None

                if ExtractHostLinksJob.host_part_pattern.match(idn):
                    parts[i] = idn
                else:
                    # self.get_logger().debug("Invalid host name: {}".format(url))
                    return None
        parts.reverse()
        return '.'.join(parts)
    def _idna_encode(self, value):
        idna = _lazy_import_idna()
        parsed = urllib_parse.urlparse(value)
        if parsed.port:
            netloc = (
                idna.encode(parsed.hostname) +
                ":{0}".format(parsed.port).encode("ascii")).decode("ascii")
        else:
            netloc = idna.encode(parsed.hostname).decode("ascii")

        # Note that building a URL in this fashion means it should be
        # semantically indistinguishable from the original but is not
        # guaranteed to be exactly the same.
        return urllib_parse.urlunparse(
            (parsed.scheme, netloc, parsed.path, parsed.params, parsed.query,
             parsed.fragment))
Example #20
0
def make_query_args(address,
                    exact_extension=True,
                    wildcard=None,
                    domain_search=False):
    assert isinstance(address, six.text_type),\
        "address should be of type %s" % six.text_type.__name__
    conf = dict(param_tools.get_global_parameters("modoboa_amavis"))
    local_part, domain = split_address(address)
    if not conf["localpart_is_case_sensitive"]:
        local_part = local_part.lower()
    if domain:
        domain = domain.lstrip("@").rstrip(".")
        domain = domain.lower()
        orig_domain = domain
        domain = idna.encode(domain, uts46=True).decode("ascii")
    delimiter = conf["recipient_delimiter"]
    local_part, extension = split_local_part(local_part, delimiter=delimiter)
    query_args = []
    if (conf["localpart_is_case_sensitive"]
            or (domain and domain != orig_domain)):
        query_args.append(address)
    if extension:
        query_args.append("%s%s%s@%s" %
                          (local_part, delimiter, extension, domain))
    if delimiter and not exact_extension and wildcard:
        query_args.append("%s%s%s@%s" %
                          (local_part, delimiter, wildcard, domain))
    query_args.append("%s@%s" % (local_part, domain))
    if domain_search:
        query_args.append("@%s" % domain)
        query_args.append("@.")

    return query_args
Example #21
0
    def normalize(self, hostname):
        hostname = Hostname.check_type(hostname)
        if not hostname:
            raise ObservableValidationError("Invalid Hostname (check_type={}): {}".format(Hostname.check_type(hostname), hostname))

        self.idna = unicode(idna.encode(hostname.lower()))
        self.value = unicode(idna.decode(hostname.lower()))
Example #22
0
    def get_certificate(self):
        try:
            hostname_idna = idna.encode(self.domain)
            sock = socket()

            sock.settimeout(10)
            sock.connect((self.domain, 443))
            sock.settimeout(None)
            ctx = SSL.Context(SSL.SSLv23_METHOD)  # most compatible
            ctx.check_hostname = False
            ctx.verify_mode = SSL.VERIFY_NONE

            sock_ssl = SSL.Connection(ctx, sock)
            sock_ssl.set_connect_state()
            sock_ssl.set_tlsext_host_name(hostname_idna)
            sock_ssl.do_handshake()
            cert = sock_ssl.get_peer_certificate()
            self.crypto_cert = cert.to_cryptography()
            sock_ssl.close()
            sock.close()

            self.get_issuer()

        except:
            pass
    def with_host(self, host):
        """Return a new URL with host replaced.

        Autoencode host if needed.

        Changing host for relative URLs is not allowed, use .join()
        instead.

        """
        # N.B. doesn't cleanup query/fragment
        if not isinstance(host, str):
            raise TypeError("Invalid host type")
        if not self.is_absolute():
            raise ValueError("host replacement is not allowed "
                             "for relative URLs")
        if not host:
            raise ValueError("host removing is not allowed")
        try:
            ip = ip_address(host)
        except ValueError:
            host = idna.encode(host, uts46=True).decode('ascii')
        else:
            if ip.version == 6:
                host = '[' + host + ']'
        val = self._val
        return URL(self._val._replace(netloc=self._make_netloc(
            val.username, val.password, host, val.port)),
                   encoded=True)
Example #24
0
    def idna_encode(name):
        """
        Borrowed wholesale from the Python Cryptography Project. It turns out
        that we can't just safely call `idna.encode`: it can explode for
        wildcard names. This avoids that problem.
        """
        import idna

        try:
            for prefix in [u"*.", u"."]:
                if name.startswith(prefix):
                    name = name[len(prefix) :]
                    return prefix.encode("ascii") + idna.encode(name)
            return idna.encode(name)
        except idna.core.IDNAError:
            return None
Example #25
0
def get_certificate(hostname, port):
    hostname_idna = idna.encode(hostname)
    sock = socket()

    sock.connect((hostname, port))
    peername = sock.getpeername()
    ctx = SSL.Context(SSL.SSLv23_METHOD)  # most compatible
    ctx.check_hostname = False
    ctx.verify_mode = SSL.VERIFY_NONE

    sock_ssl = SSL.Connection(ctx, sock)
    sock_ssl.set_connect_state()
    sock_ssl.set_tlsext_host_name(hostname_idna)
    sock_ssl.do_handshake()
    cert = sock_ssl.get_peer_certificate()
    crypto_cert = cert.to_cryptography()

    certs = sock_ssl.get_peer_cert_chain()
    for pos, cert in enumerate(certs):
        print("====SSL session certs[" + str(pos) + "]===")
        fd = os.open("cert" + str(pos) + ".pem", os.O_RDWR | os.O_CREAT)
        os.write(fd, crypto.dump_certificate(crypto.FILETYPE_PEM, cert))
        os.close(fd)
        print_cert_info(cert.to_cryptography())
    sock_ssl.close()
    sock.close()

    return HostInfo(cert=crypto_cert, peername=peername, hostname=hostname)
 def _get_object(self, handle: str) -> Any:
     objects = {}
     with suppress(OBJECT_NOT_FOUND, INVALID_HANDLE):
         objects['contact'] = WHOIS.get_contact_by_handle(handle)
     with suppress(OBJECT_NOT_FOUND, INVALID_HANDLE):
         objects['nsset'] = WHOIS.get_nsset_by_handle(handle)
     with suppress(OBJECT_NOT_FOUND, INVALID_HANDLE):
         objects['keyset'] = WHOIS.get_keyset_by_handle(handle)
     with suppress(OBJECT_NOT_FOUND, INVALID_HANDLE):
         objects['registrar'] = WHOIS.get_registrar_by_handle(handle)
     if not handle.startswith("."):
         with suppress(OBJECT_NOT_FOUND, UNMANAGED_ZONE, INVALID_LABEL,
                       TOO_MANY_LABELS, idna.IDNAError):
             idna_handle = idna.encode(handle).decode()
             try:
                 objects['domain'] = WHOIS.get_domain_by_handle(idna_handle)
             except OBJECT_DELETE_CANDIDATE:
                 objects['domain'] = None
     if not objects:
         raise WebwhoisError(
             code="OBJECT_NOT_FOUND",
             title=_("Record not found"),
             message=self.message_with_handle_in_html(
                 _("%s does not match any record."), handle),
             object_not_found=True,
         )
     return objects
Example #27
0
    def execute(self, domain_name):

        idns = []
        oldidns = []

        # Walk through all characters in the domain name
        domain = domain_name.rsplit('.', 1)[0]

        # Initially, replace single instances of homoglyphs
        idns = idns + self.replace_homoglyphs([domain])

        # Now iterate over this list again, to replace additional homoglyphs and always remove duplicates
        while not set(oldidns) == set(idns):
            oldidns = idns
            idns = list(set(idns + self.replace_homoglyphs(idns)))

        domains = []
        for idn in idns:
            try:
                clear = (idn + '.' +
                         domain_name.rsplit('.', 1)[1]).encode('utf-8')
                puni = idna.encode(clear.decode('utf-8'))
                print clear, puni
                domains.append(puni)
            except:
                pass
Example #28
0
    def get_certificate(host_name, port, proxy_ip='', proxy_port=8080):
        sock = socks.socksocket(
        )  # Same API as socket.socket in the standard lib
        if proxy_ip:
            sock.set_proxy(socks.HTTP, proxy_ip, proxy_port)

        hostname_idna = idna.encode(host_name)
        # sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        # sock = socket.socket()

        sock.connect((host_name, port))
        peer_name = sock.getpeername()
        ctx = SSL.Context(SSL.SSLv23_METHOD)  # most compatible
        ctx.check_hostname = False
        ctx.verify_mode = SSL.VERIFY_NONE

        sock_ssl = SSL.Connection(ctx, sock)
        sock_ssl.set_connect_state()
        sock_ssl.set_tlsext_host_name(hostname_idna)
        sock_ssl.do_handshake()
        cert = sock_ssl.get_peer_certificate()
        crypto_cert = cert.to_cryptography()
        sock_ssl.close()
        sock.close()

        return HostInfo(cert=crypto_cert,
                        peername=peer_name,
                        hostname=host_name)
Example #29
0
 def normalize(self, hostname):
     if not is_hostname(hostname):
         raise ObservableValidationError("Invalid Hostname (is_hostname={}): {}".format(is_hostname(hostname), hostname))
     if hostname.endswith('.'):
         hostname = hostname[:-1]
     self.idna = unicode(idna.encode(hostname.lower()))
     self.value = unicode(idna.decode(hostname.lower()))
def get_certificate_from_client(hostname, port, CERT_FILE):
    """

    :type hostname: str
    :type port: int
    :type CERT_FILE: str

    """
    hostname_idna = idna.encode(hostname)
    sock = socket.socket()

    sock.connect((hostname, int(port)))
    ctx = SSL.Context(SSL.SSLv23_METHOD)  # most compatible
    ctx.check_hostname = False
    ctx.verify_mode = SSL.VERIFY_NONE
    sock_ssl = SSL.Connection(ctx, sock)
    sock_ssl.set_connect_state()
    sock_ssl.set_tlsext_host_name(hostname_idna)
    sock_ssl.do_handshake()
    cert = sock_ssl.get_peer_certificate()
    sock_ssl.close()
    sock.close()
    _write_certificate(CERT_FILE, cert)
    subject_str = "".join(
        "/{0:s}={1:s}".format(name.decode(), value.decode())
        for name, value in cert.get_issuer().get_components())
    logging.debug(subject_str)
    return subject_str
    def __init__(self, val='', *, encoded=False, strict=None):
        if strict is not None:  # pragma: no cover
            warnings.warn("strict parameter is ignored")
        if isinstance(val, URL):
            self._val = val._val
            self._cache = val._cache
            return
        if isinstance(val, str):
            val = urlsplit(val)
        elif isinstance(val, SplitResult):
            if not encoded:
                raise ValueError("Cannot apply decoding to SplitResult")
        else:
            raise TypeError("Constructor parameter should be str")

        if not encoded:
            if not val[1]:  # netloc
                netloc = ''
            else:
                netloc = val.hostname
                if netloc is None:
                    raise ValueError(
                        "Invalid URL: host is required for abolute urls.")
                try:
                    netloc.encode('ascii')
                except UnicodeEncodeError:
                    netloc = idna.encode(netloc, uts46=True).decode('ascii')
                else:
                    try:
                        ip = ip_address(netloc)
                    except ValueError:
                        pass
                    else:
                        if ip.version == 6:
                            netloc = '[' + netloc + ']'
                if val.port:
                    netloc += ':{}'.format(val.port)
                if val.username:
                    user = _quote(val.username)
                else:
                    user = ''
                if val.password:
                    user += ':' + _quote(val.password)
                if user:
                    netloc = user + '@' + netloc

            path = _quote(val[2], safe='+@:', protected='/+')
            if netloc:
                path = _normalize_path(path)

            query = _quote(val[3],
                           safe='=+&?/:@',
                           protected=PROTECT_CHARS,
                           qs=True)
            fragment = _quote(val[4], safe='?/:@')
            val = SplitResult(val[0], netloc, path, query, fragment)

        self._val = val
        self._cache = {}
Example #32
0
 def runTest(self):
     if not self.fields:
         return
     try:
         types, source, to_unicode, to_ascii = (
             unicode_fixup(field) for field in self.fields[:4])
         if (unicode_fixup(u"\\uD804\\uDC39") in source
                 and sys.version_info[0] < 3):
             raise unittest.SkipTest(
                 "Python 2's Unicode support is too old for this test")
     except ValueError:
         raise unittest.SkipTest(
             "Test requires Python wide Unicode support")
     if source in _SKIP_TESTS:
         return
     if not to_unicode:
         to_unicode = source
     if not to_ascii:
         to_ascii = to_unicode
     nv8 = (len(self.fields) > 4 and self.fields[4])
     try:
         output = idna.decode(source, uts46=True, strict=True)
         if to_unicode[0] == u"[":
             self.fail(
                 "decode() did not emit required error {0} for {1}".format(
                     to_unicode, repr(source)))
         self.assertEqual(output, to_unicode, "unexpected decode() output")
     except (idna.IDNAError, UnicodeError, ValueError) as exc:
         if unicode(exc).startswith(u"Unknown"):
             raise unittest.SkipTest(
                 "Test requires support for a newer"
                 " version of Unicode than this Python supports")
         if to_unicode[0] != u"[" and not nv8:
             raise
     for transitional in {
             u"B": (True, False),
             u"T": (True, ),
             u"N": (False, ),
     }[types]:
         try:
             output = idna.encode(source,
                                  uts46=True,
                                  strict=True,
                                  transitional=transitional).decode("ascii")
             if to_ascii[0] == u"[":
                 self.fail(
                     "encode(transitional={0}) did not emit required error {1} for {2}"
                     .format(transitional, to_ascii, repr(source)))
             self.assertEqual(
                 output, to_ascii,
                 "unexpected encode(transitional={0}) output".format(
                     transitional))
         except (idna.IDNAError, UnicodeError, ValueError) as exc:
             if unicode(exc).startswith(u"Unknown"):
                 raise unittest.SkipTest(
                     "Test requires support for a newer"
                     " version of Unicode than this Python supports")
             if to_ascii[0] != u"[" and not nv8:
                 raise
Example #33
0
async def getaddrinfo(host, port, family=0, type=0, proto=0, flags=0):
    """Look up a numeric address given a name.

    Arguments and return values are identical to :func:`socket.getaddrinfo`,
    except that this version is async.

    Also, :func:`trio.socket.getaddrinfo` correctly uses IDNA 2008 to process
    non-ASCII domain names. (:func:`socket.getaddrinfo` uses IDNA 2003, which
    can give the wrong result in some cases and cause you to connect to a
    different host than the one you intended; see `bpo-17305
    <https://bugs.python.org/issue17305>`__.)

    This function's behavior can be customized using
    :func:`set_custom_hostname_resolver`.

    """

    # If host and port are numeric, then getaddrinfo doesn't block and we can
    # skip the whole thread thing, which seems worthwhile. So we try first
    # with the _NUMERIC_ONLY flags set, and then only spawn a thread if that
    # fails with EAI_NONAME:
    def numeric_only_failure(exc):
        return isinstance(exc, _stdlib_socket.gaierror) and \
            exc.errno == _stdlib_socket.EAI_NONAME

    async with _try_sync(numeric_only_failure):
        return _stdlib_socket.getaddrinfo(
            host, port, family, type, proto, flags | _NUMERIC_ONLY
        )
    # That failed; it's a real hostname. We better use a thread.
    #
    # Also, it might be a unicode hostname, in which case we want to do our
    # own encoding using the idna module, rather than letting Python do
    # it. (Python will use the old IDNA 2003 standard, and possibly get the
    # wrong answer - see bpo-17305). However, the idna module is picky, and
    # will refuse to process some valid hostname strings, like "::1". So if
    # it's already ascii, we pass it through; otherwise, we encode it to.
    if isinstance(host, str):
        try:
            host = host.encode("ascii")
        except UnicodeEncodeError:
            # UTS-46 defines various normalizations; in particular, by default
            # idna.encode will error out if the hostname has Capital Letters
            # in it; with uts46=True it will lowercase them instead.
            host = _idna.encode(host, uts46=True)
    hr = _resolver.get(None)
    if hr is not None:
        return await hr.getaddrinfo(host, port, family, type, proto, flags)
    else:
        return await run_sync_in_worker_thread(
            _stdlib_socket.getaddrinfo,
            host,
            port,
            family,
            type,
            proto,
            flags,
            cancellable=True
        )
Example #34
0
 def sendmail(self, subject, body):
     """ Send an email to the address.
     """
     from_address = "{0}@{1}".format(
         app.config['POSTMASTER'],
         idna.encode(app.config['DOMAIN']).decode('ascii'),
     )
     with smtplib.SMTP(app.config['HOST_AUTHSMTP'], port=10025) as smtp:
         to_address = "{0}@{1}".format(
             self.localpart,
             idna.encode(self.domain_name).decode('ascii'),
         )
         msg = text.MIMEText(body)
         msg['Subject'] = subject
         msg['From'] = from_address
         msg['To'] = to_address
         smtp.sendmail(from_address, [to_address], msg.as_string())
Example #35
0
    def _get_idna_encoded_host(host):
        import idna

        try:
            host = idna.encode(host, uts46=True).decode('utf-8')
        except idna.IDNAError:
            raise UnicodeError
        return host
Example #36
0
    def _idna_encode(self, value):
        # Import idna lazily becase it allocates a decent amoutn of memory, and
        # we're only using it in deprecated paths.
        import idna

        _, address = parseaddr(value)
        parts = address.split(u"@")
        return parts[0] + "@" + idna.encode(parts[1]).decode("ascii")
Example #37
0
    def __init__(self, domain):
        """Handle all the possible domain types coming in."""
        domain_unicode = self._try_parse_to_unicode_domain(domain)
        if domain_unicode is None:
            raise InvalidDomainException(f'Invalid domain: {repr(domain)}')

        self._domain_unicode = domain_unicode
        self._domain_ascii = idna.encode(domain_unicode).decode()
Example #38
0
 def normalize(self):
     self.value = refang(self.value.lower())
     try:
         self.idna = unicode(idna.encode(self.value))
     except idna.core.InvalidCodepoint:
         pass
     except Exception, e:
         raise ObservableValidationError(e.message)
Example #39
0
 def sendmail(self, subject, body):
     """ Send an email to the address.
     """
     from_address = "{0}@{1}".format(
         app.config['POSTMASTER'],
         idna.encode(app.config['DOMAIN']).decode('ascii'),
     )
     with smtplib.SMTP(app.config['HOST_AUTHSMTP'], port=10025) as smtp:
         to_address = "{0}@{1}".format(
             self.localpart,
             idna.encode(self.domain_name).decode('ascii'),
         )
         msg = text.MIMEText(body)
         msg['Subject'] = subject
         msg['From'] = from_address
         msg['To'] = to_address
         smtp.sendmail(from_address, [to_address], msg.as_string())
Example #40
0
async def getaddrinfo(host, port, family=0, type=0, proto=0, flags=0):
    """Look up a numeric address given a name.

    Arguments and return values are identical to :func:`socket.getaddrinfo`,
    except that this version is async.

    Also, :func:`trio.socket.getaddrinfo` correctly uses IDNA 2008 to process
    non-ASCII domain names. (:func:`socket.getaddrinfo` uses IDNA 2003, which
    can give the wrong result in some cases and cause you to connect to a
    different host than the one you intended; see `bpo-17305
    <https://bugs.python.org/issue17305>`__.)

    This function's behavior can be customized using
    :func:`set_custom_hostname_resolver`.

    """

    # If host and port are numeric, then getaddrinfo doesn't block and we can
    # skip the whole thread thing, which seems worthwhile. So we try first
    # with the _NUMERIC_ONLY flags set, and then only spawn a thread if that
    # fails with EAI_NONAME:
    def numeric_only_failure(exc):
        return isinstance(exc, gaierror) and exc.errno == EAI_NONAME

    async with _try_sync(numeric_only_failure):
        return _stdlib_socket.getaddrinfo(
            host, port, family, type, proto, flags | _NUMERIC_ONLY
        )
    # That failed; it's a real hostname. We better use a thread.
    #
    # Also, it might be a unicode hostname, in which case we want to do our
    # own encoding using the idna module, rather than letting Python do
    # it. (Python will use the old IDNA 2003 standard, and possibly get the
    # wrong answer - see bpo-17305). However, the idna module is picky, and
    # will refuse to process some valid hostname strings, like "::1". So if
    # it's already ascii, we pass it through; otherwise, we encode it to.
    if isinstance(host, str):
        try:
            host = host.encode("ascii")
        except UnicodeEncodeError:
            # UTS-46 defines various normalizations; in particular, by default
            # idna.encode will error out if the hostname has Capital Letters
            # in it; with uts46=True it will lowercase them instead.
            host = _idna.encode(host, uts46=True)
    hr = _resolver.get(None)
    if hr is not None:
        return await hr.getaddrinfo(host, port, family, type, proto, flags)
    else:
        return await run_sync_in_worker_thread(
            _stdlib_socket.getaddrinfo,
            host,
            port,
            family,
            type,
            proto,
            flags,
            cancellable=True
        )
Example #41
0
def validate_hostname(name):
    """ Validate a hostname. """

    if name.endswith("."):
        raise ValidationError("Name must not end with a punctuation mark.")
    # Assume we are not running a tld
    if '.' not in name:
        raise ValidationError("Name must include a tld.")
    # Any label in can be max 63 characters, after idna encoding
    for label in name.split("."):
        if label == '':
            raise ValidationError("Too many punctation marks")
        if label[0] == "-" or label[-1] == "-":
            raise ValidationError(
                "Can not start or end a label with a hyphen '{}'".format(
                    label))
        if "--" in label:
            raise ValidationError(
                "Label can not contain double hyphen '{}'".format(label))
        if len(label) > 63:
            raise ValidationError(
                "Label '{}' is {} characters long, maximum is 63".format(
                    label, len(label)))
        # convert to .isascii in python 3.7
        if all(ord(char) < 128 for char in label):
            if "*" in label:
                if len(label) > 1:
                    raise ValidationError("Wildcard must be standalone")
                continue
            label_regex = r"^_?[a-zA-Z0-9\-]+$"
            validator = RegexValidator(label_regex,
                                       message="Label '{}' is not valid. "
                                       "Must be within '{}'.".format(
                                           label, label_regex))
            validator(label)
        else:
            try:
                idna.encode(label)
            except idna.core.InvalidCodepoint as e:
                raise ValidationError("Invalid label '{}': {}".format(
                    label, e))
            except idna.core.IDNAError as e:
                raise ValidationError(
                    "Label '{}' could not be idna encoded: {}".format(
                        label, e))
Example #42
0
 def normalize(self, hostname):
     hostname = Hostname.check_type(hostname)
     if not hostname:
         raise ObservableValidationError("Invalid Hostname (check_type={}): {}".format(Hostname.check_type(hostname), hostname))
     self.value = unicode(hostname.lower())
     try:
         self.idna = unicode(idna.encode(hostname.lower()))
     except idna.core.InvalidCodepoint:
         pass
Example #43
0
    def handle_rf(self, order_item):
        assert order_item

        original_domain = order_item['domain']
        order_item['domain'] = idna.encode(order_item['domain'])

        logger.info("IDNA convert: %s -> %s" % (original_domain, order_item['domain']))

        return order_item
Example #44
0
 def idna_encoder(name):
     if any(ord(c) > 128 for c in name):
         try:
             return idna.encode(name.lower(),
                                strict=True,
                                std3_rules=True)
         except idna.IDNAError:
             raise exceptions.InvalidAuthority(self.authority)
     return name
def to_idna(domain):
    # If the domain is passed as a bytes object (alias for str in Python 2),
    # then assume it is already IDNA encoded and decode as if ASCII and work
    # with unicode (Py 2 unicode/Py 3 str) instances.
    if isinstance(domain, bytes):
        return domain.decode("ascii")

    # IDNA-encode, but get back a unicode instance.
    return idna.encode(domain).decode("ascii")
Example #46
0
 def process_bind_param(self, value, dialect):
     try:
         localpart, domain_name = value.split('@')
         return "{0}@{1}".format(
             localpart,
             idna.encode(domain_name).decode('ascii'),
         ).lower()
     except ValueError:
         pass
Example #47
0
    def build(self):
        ret = b""

        if self._ec_point_format is not None:
            ec_point_format_struct = construct.Container(
                ec_point_format_length=len(self._ec_point_format),
                ec_point_format=self._get_bytes_from_ec_point_format(
                    self._ec_point_format
                )
            )
            ret += hello_constructs.Extension.build(
                construct.Container(
                    extension_type=11,
                    extension_length=len(hello_constructs.ECPointFormat.build(
                        ec_point_format_struct
                    )),
                    extension_struct=ec_point_format_struct
                )
            )

        if self._ec_curves is not None:
            ec_curves_struct = construct.Container(
                ec_curves_length=len(self._ec_curves) * 2,
                named_curves=self._get_bytes_from_ec_curves(
                    self._ec_curves
                )
            )
            ret += hello_constructs.Extension.build(
                construct.Container(
                    extension_type=10,
                    extension_length=len(hello_constructs.ECCurves.build(
                        ec_curves_struct
                    )),
                    extension_struct=ec_curves_struct
                )
            )

        if self._hostname is not None:
            encoded_hostname = idna.encode(self._hostname)
            sni_struct = construct.Container(
                server_name_list_length=len(encoded_hostname) + 3,
                name_type=0,
                server_name_length=len(encoded_hostname),
                server_name=encoded_hostname
            )
            ret += hello_constructs.Extension.build(
                construct.Container(
                    extension_type=0,
                    extension_length=len(hello_constructs.ServerName.build(
                        sni_struct
                    )),
                    extension_struct=sni_struct
                )
            )

        return ret
Example #48
0
    def __init__(self, plugin=None, protocol=None, auth=None, domain=None,
                 port=None, path=None, query=None, fragment=None):
        self.plugin = plugin

        self.protocol = protocol
        self.auth = auth
        self.domain = idna.encode(to_unicode(domain))
        self.port = port
        self.path = path
        self.query = query
        self.fragment = fragment
Example #49
0
 def idna_encode(name):
     if name and any([ord(x) > 128 for x in name]):
         try:
             import idna
         except ImportError:
             raise LocationParseError("Unable to parse URL without the 'idna' module")
         try:
             return idna.encode(name.lower(), strict=True, std3_rules=True)
         except idna.IDNAError:
             raise LocationParseError(u"Name '%s' is not a valid IDNA label" % name)
     return name
Example #50
0
 def normalize(self):
     self.value = refang(self.value.lower())
     # Remove trailing dot if existing
     if self.value.endswith("."):
         self.value = self.value[:-1]
     try:
         self.idna = unicode(idna.encode(self.value))
     except idna.core.InvalidCodepoint:
         pass
     except Exception, e:
         raise ObservableValidationError(e.message)
Example #51
0
def _idnaBytes(text):
    """
    Convert some text typed by a human into some ASCII bytes. This is a
    copy of twisted.internet._idna._idnaBytes. For documentation, see the
    twisted documentation.
    """
    try:
        import idna
    except ImportError:
        return text.encode("idna")
    else:
        return idna.encode(text)
Example #52
0
def ishost(value):
    MAX_HOSTNAME_LEN = 253
    try:
        # returns bytestring, then encode to str
        value = to_str(idna.encode(value))
    except AttributeError:
        pass
    if value.endswith('.'):
        value = value[:-1]
    if not value or len(value) > MAX_HOSTNAME_LEN:
        return False
    return all(map(_RE_ISH.match, value.split('.')))
Example #53
0
    def _idna_encode(self, value):
        parsed = urllib_parse.urlparse(value)
        if parsed.port:
            netloc = (
                idna.encode(parsed.hostname) +
                ":{0}".format(parsed.port).encode("ascii")
            ).decode("ascii")
        else:
            netloc = idna.encode(parsed.hostname).decode("ascii")

        # Note that building a URL in this fashion means it should be
        # semantically indistinguishable from the original but is not
        # guaranteed to be exactly the same.
        return urllib_parse.urlunparse((
            parsed.scheme,
            netloc,
            parsed.path,
            parsed.params,
            parsed.query,
            parsed.fragment
        ))
Example #54
0
 def ace_address(self):
     if not is_pure_ascii(self.mailbox):
         raise ValueError('address {} has no ASCII-compatable encoding'
                          .format(self.address.encode('utf-8')))
     ace_hostname = self.hostname
     if not is_pure_ascii(self.hostname):
         try:
             ace_hostname = idna.encode(self.hostname)
         except idna.IDNAError:
             raise ValueError('address {} has no ASCII-compatable encoding'
                              .format(self.address.encode('utf-8')))
     return '{}@{}'.format(self.mailbox, ace_hostname)
Example #55
0
    def test_encode(self):

        self.assertEqual(idna.encode('xn--zckzah.xn--zckzah'), b'xn--zckzah.xn--zckzah')
        self.assertEqual(idna.encode(u'\u30c6\u30b9\u30c8.xn--zckzah'), b'xn--zckzah.xn--zckzah')
        self.assertEqual(idna.encode(u'\u30c6\u30b9\u30c8.\u30c6\u30b9\u30c8'), b'xn--zckzah.xn--zckzah')
        self.assertEqual(idna.encode('abc.abc'), b'abc.abc')
        self.assertEqual(idna.encode('xn--zckzah.abc'), b'xn--zckzah.abc')
        self.assertEqual(idna.encode(u'\u30c6\u30b9\u30c8.abc'), b'xn--zckzah.abc')
        self.assertEqual(idna.encode(u'\u0521\u0525\u0523-\u0523\u0523-----\u0521\u0523\u0523\u0523.aa'),
                                     b'xn---------90gglbagaar.aa')
        self.assertRaises(idna.IDNAError, idna.encode,
                          u'\u0521\u0524\u0523-\u0523\u0523-----\u0521\u0523\u0523\u0523.aa')
        self.assertEqual(idna.encode('a'*63), b'a'*63)
        self.assertRaises(idna.IDNAError, idna.encode, 'a'*64)
Example #56
0
 def test_encode(self):
     self.assertEqual(idna.encode('xn--zckzah.xn--zckzah'), b'xn--zckzah.xn--zckzah')
     self.assertEqual(idna.encode(u'\u30c6\u30b9\u30c8.xn--zckzah'), b'xn--zckzah.xn--zckzah')
     self.assertEqual(idna.encode(u'\u30c6\u30b9\u30c8.\u30c6\u30b9\u30c8'), b'xn--zckzah.xn--zckzah')
     self.assertEqual(idna.encode('abc.abc'), b'abc.abc')
     self.assertEqual(idna.encode('xn--zckzah.abc'), b'xn--zckzah.abc')
     self.assertEqual(idna.encode(u'\u30c6\u30b9\u30c8.abc'), b'xn--zckzah.abc')
 def runTest(self):
     if not self.fields:
         return
     try:
         types, source, to_unicode, to_ascii = (unicode_fixup(field)
             for field in self.fields[:4])
         if (unicode_fixup(u"\\uD804\\uDC39") in source and
                 sys.version_info[0] < 3):
             raise unittest.SkipTest(
                 "Python 2's Unicode support is too old for this test")
     except ValueError:
         raise unittest.SkipTest(
             "Test requires Python wide Unicode support")
     if source in _SKIP_TESTS:
         return
     if not to_unicode:
         to_unicode = source
     if not to_ascii:
         to_ascii = to_unicode
     nv8 = (len(self.fields) > 4 and self.fields[4])
     try:
         output = idna.decode(source, uts46=True, strict=True)
         if to_unicode[0] == u"[":
             self.fail("decode() did not emit required error {0} for {1}".format(to_unicode, repr(source)))
         self.assertEqual(output, to_unicode, "unexpected decode() output")
     except (idna.IDNAError, UnicodeError, ValueError) as exc:
         if unicode(exc).startswith(u"Unknown"):
             raise unittest.SkipTest("Test requires support for a newer"
                 " version of Unicode than this Python supports")
         if to_unicode[0] != u"[" and not nv8:
             raise
     for transitional in {
             u"B": (True, False),
             u"T": (True,),
             u"N": (False,),
             }[types]:
         try:
             output = idna.encode(source, uts46=True, strict=True,
                 transitional=transitional).decode("ascii")
             if to_ascii[0] == u"[":
                 self.fail(
                     "encode(transitional={0}) did not emit required error {1} for {2}".
                     format(transitional, to_ascii, repr(source)))
             self.assertEqual(output, to_ascii,
                 "unexpected encode(transitional={0}) output".
                 format(transitional))
         except (idna.IDNAError, UnicodeError, ValueError) as exc:
             if unicode(exc).startswith(u"Unknown"):
                 raise unittest.SkipTest("Test requires support for a newer"
                                         " version of Unicode than this Python supports")
             if to_ascii[0] != u"[" and not nv8:
                 raise
Example #58
0
File: _tls.py Project: hynek/tnw
 def handshake():
     sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     tls_ctx = SSL.Context(SSL.SSLv23_METHOD)
     conn = SSL.Connection(
         tls_ctx,
         sock
     )
     conn.connect((idna.encode(hostname).encode("ascii"), 443))
     conn.do_handshake()
     cert = conn.get_peer_certificate()
     conn.shutdown()
     conn.close()
     return cert
Example #59
0
def _dnsname_to_stdlib(name):
    """
    Converts a dNSName SubjectAlternativeName field to the form used by the
    standard library on the given Python version.

    Cryptography produces a dNSName as a unicode string that was idna-decoded
    from ASCII bytes. We need to idna-encode that string to get it back, and
    then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib
    uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8).
    """
    name = idna.encode(name)
    if sys.version_info >= (3, 0):
        name = name.decode('utf-8')
    return name
Example #60
0
def splitaddress(address):
    try:
        address = to_str(idna.encode(address))
    except (AttributeError, idna.IDNAError):
        pass
    sep = ']:' if address.split(':', 2)[2:] else ':'
    parts = address.rsplit(sep, 1)
    try:
        addr, port = parts
        port = int(port)
    except ValueError:
        addr = parts[0]
        port = None
    return addr, port