def _idna_encode(value): # Retain prefixes '*.' for common/alt names and '.' for name constraints for prefix in ['*.', '.']: if value.startswith(prefix): value = value[len(prefix):] return prefix + idna.encode(value).decode("ascii") return idna.encode(value).decode("ascii")
def __init__(self, value): if not isinstance(value, six.text_type): raise TypeError("value must be a unicode string") parsed = urllib_parse.urlparse(value) if not parsed.hostname: netloc = "" elif parsed.port: netloc = ( idna.encode(parsed.hostname) + ":{0}".format(parsed.port).encode("ascii") ).decode("ascii") else: netloc = idna.encode(parsed.hostname).decode("ascii") # Note that building a URL in this fashion means it should be # semantically indistinguishable from the original but is not # guaranteed to be exactly the same. uri = urllib_parse.urlunparse(( parsed.scheme, netloc, parsed.path, parsed.params, parsed.query, parsed.fragment )).encode("ascii") self._value = value self._encoded = uri
def idna_encode(name): """ Borrowed wholesale from the Python Cryptography Project. It turns out that we can't just safely call `idna.encode`: it can explode for wildcard names. This avoids that problem. """ for prefix in [u'*.', u'.']: if name.startswith(prefix): name = name[len(prefix):] return prefix.encode('ascii') + idna.encode(name) return idna.encode(name)
def requires_non_ascii(self): """ Can the address be converted to an ASCII compatible encoding? """ if not is_pure_ascii(self.mailbox): return True if not is_pure_ascii(self.hostname): try: idna.encode(self.hostname) except idna.IDNAError: return True return False
def get_object(self, **kwargs): qs = self.get_queryset(**kwargs) domain = self.kwargs['domain'] # support IDN domains, i.e. accept Unicode encoding too try: import idna if domain.startswith("*."): ace_domain = "*." + idna.encode(domain[2:]).decode("utf-8", "strict") else: ace_domain = idna.encode(domain).decode("utf-8", "strict") except: ace_domain = domain return get_object_or_404(qs, domain=ace_domain)
def __init__(self, raw_display_name=None, raw_addr_spec=None, _display_name=None, _mailbox=None, _hostname=None): raw_display_name = _to_parser_input(raw_display_name) raw_addr_spec = _to_parser_input(raw_addr_spec) if raw_display_name and raw_addr_spec: mailbox = addr_spec_parser.parse(raw_addr_spec, lexer.clone()) self._display_name = _to_text(raw_display_name) self._mailbox = _to_text(mailbox.local_part) self._hostname = _to_text(mailbox.domain) elif raw_display_name: mailbox = mailbox_parser.parse(raw_display_name, lexer.clone()) self._display_name = _to_text(mailbox.display_name) self._mailbox = _to_text(mailbox.local_part) self._hostname = _to_text(mailbox.domain) elif raw_addr_spec: mailbox = addr_spec_parser.parse(raw_addr_spec, lexer.clone()) self._display_name = u'' self._mailbox = _to_text(mailbox.local_part) self._hostname = _to_text(mailbox.domain) elif _mailbox and _hostname: self._display_name = _display_name or u'' self._mailbox = _mailbox self._hostname = _hostname else: raise SyntaxError('failed to create EmailAddress: bad parameters') # Convert display name to decoded unicode string. if (self._display_name.startswith('=?') and self._display_name.endswith('?=')): self._display_name = mime_to_unicode(self._display_name) if (self._display_name.startswith('"') and self._display_name.endswith('"') and len(self._display_name) > 2): self._display_name = smart_unquote(self._display_name) # Convert hostname to lowercase unicode string. self._hostname = self._hostname.lower() if self._hostname.startswith('xn--') or '.xn--' in self._hostname: self._hostname = idna.decode(self._hostname) if not is_pure_ascii(self._hostname): idna.encode(self._hostname) assert isinstance(self._display_name, six.text_type) assert isinstance(self._mailbox, six.text_type) assert isinstance(self._hostname, six.text_type)
def full_spec(self): """ Returns an ASCII-compatable encoding of an email address or raises a ValueError. Display name and domain parts will be converted to ASCII-compatable encoding. The transformed address will be ASCII-only and RFC-2822 compliant. >>> EmailAddress("Ev K", "*****@*****.**").full_spec() 'Ev K <*****@*****.**>' >>> EmailAddress("Жека", "*****@*****.**").full_spec() '=?utf-8?b?0JbQtdC60LA=?= <*****@*****.**>' """ if not is_pure_ascii(self.mailbox): raise ValueError( 'address {} has no ASCII-compatable encoding'.format( self.address.encode('utf-8'))) if not is_pure_ascii(self.hostname): try: ace_hostname = idna.encode(self.hostname) except idna.IDNAError: raise ValueError( 'address {} has no ASCII-compatable encoding'.format( self.address.encode('utf-8'))) else: ace_hostname = self.hostname if self.display_name: ace_display_name = smart_quote( encode_string(None, self.display_name, maxlinelen=MAX_ADDRESS_LENGTH)) return u'{} <{}@{}>'.format(ace_display_name, self.mailbox, ace_hostname) return u'{}@{}'.format(self.mailbox, ace_hostname)
def get_certificate(hostname, port): hostname_idna = idna.encode(hostname) sock = socket() sock.settimeout(5) sock.connect((hostname, 443)) sock.setblocking(1) peername = sock.getpeername() ctx = SSL.Context(SSL.SSLv23_METHOD) # most compatible ctx.check_hostname = False ctx.verify_mode = SSL.VERIFY_NONE sock_ssl = SSL.Connection(ctx, sock) sock_ssl.set_connect_state() sock_ssl.set_tlsext_host_name(hostname_idna) sock_ssl.do_handshake() cert = sock_ssl.get_peer_certificate() crypto_cert = cert.to_cryptography() import pprint pprint.pprint(crypto_cert) sock_ssl.close() sock.close() return HostInfo(cert=crypto_cert, peername=peername, hostname=hostname)
def gen(self): full_url = self.full_url for char in self.domain: if char in self.glyphs: full_url = full_url.replace( char, "[{} x {}]".format(char, len(self.glyphs[char]))) special_chars = self.glyphs[char] for special_char in special_chars: full_domain = "{}.{}".format( idna.encode(self.domain.replace( char, "{}".format(special_char)), uts46=True).decode('utf8'), self.url_extract.suffix) ip = domains_tool.Domain.check_ip(full_domain) if ip: hopp = domains_tool.Domain.check_hopp(full_domain) else: hopp = '?' if self.show_all: print("sign: {}, {}.{} => {} [{}], {}".format( char, self.domain.replace(char, "{}".format(special_char)), self.url_extract.suffix, full_domain, ip, hopp)) elif ip: print("sign: {}, {}.{} => {} [{}], {}".format( char, self.domain.replace(char, "{}".format(special_char)), self.url_extract.suffix, full_domain, ip, hopp))
def patched_match_hostname(cert, hostname): try: hostname = idna.encode(hostname, uts46=True).decode('ascii') except UnicodeError: hostname = hostname.encode('idna').decode('ascii') return real_match_hostname(cert, hostname)
def normalise_domain(domain): domain = domain.lower() # Handle users copying domains with the scheme attached. # Only allow these two schemes - GPC is for HTTP(s). if domain.startswith('https://'): domain = domain[8:] elif domain.startswith('http://'): domain = domain[7:] # Similar to handling schemes, handle one slash at the end of the domain. if domain.endswith('/'): domain = domain[:-1] # Strip any optional trailing period from the domain. if domain.endswith('.'): domain = domain[:-1] try: # Convert to and from IDNA encoding with compatibility mapping enabled to normalise. domain = idna.decode(idna.encode(domain, uts46=True)) except idna.IDNAError: # Ignore IDNA errors and return the domain without IDNA normalisation. # Any IDNA error will cause check_domain() to fail anyway. pass return domain
def dmarc_fetch_public_suffix_list(): """ Fetch the list from the configured URL and parse it leaving out comments, empty lines and invalid lines. """ public_suffix_list = [] r = http_get(settings.PUBLIC_SUFFIX_LIST_URL) if not r: return public_suffix_list lines = r.text.split("\n") for line in lines: labels = [] line = line.rstrip() if line and not line.startswith(("//", " ")): exception = False if line.startswith("!"): exception = True line = line[1:] # Convert to punnycode. # This is how we are going to compare domain names later. try: for label in line.split('.'): if label == '*': labels.append(label) else: labels.append(idna.encode(label).decode('ascii')) public_suffix_list.append((labels[::-1], exception)) except (UnicodeError, ValueError, idna.IDNAError): pass return public_suffix_list
def normalize_url(url): purl = rfc3986.urlparse(url) if purl.scheme is None and purl.host is None and purl.path is not None: # no protocol, no // : it is a path according to the rfc3986 # but we know it is a host purl = rfc3986.urlparse('//' + url) if purl.scheme is None: # The url starts with // # Add https (or http for .onion or i2p TLD) if model.host_use_http(purl.host): purl = purl.copy_with(scheme='http') else: purl = purl.copy_with(scheme='https') # first normalization # * idna encoding to avoid misleading host # * remove query and fragment # * remove empty path purl = purl.copy_with(scheme=purl.scheme.lower(), host=idna.encode(purl.host).decode('utf-8').lower(), path='' if purl.path == '/' else purl.path, query=None, fragment=None) # only https (exception: http for .onion and .i2p TLD) if (purl.scheme == 'https' and not model.host_use_http(purl.host)) or\ (purl.scheme == 'http' and model.host_use_http(purl.host)): # normalize the URL return rfc3986.normalize_uri(purl.geturl()) # return None
def request_certificate(): try: domain = request.form.get('domain', None) if domain is not None: domain = domain.strip() is_ok = Domain.query.filter_by(name=domain).filter_by(belongs=session['username']).first() if is_ok is None: return jsonify({'error': 'domain non exist!'}), 404 if not is_ok.validated: return jsonify({'error': 'domain not validated'}), 404 latest_cert = Certificates.query.filter_by(domain=domain).order_by(desc(Certificates.create_time)).first() if latest_cert is not None and datetime.now() - latest_cert.create_time < timedelta(days=60): return jsonify({'error': 'no reason to issue new certificate, validity over 60 days'}), 429 privkey = request.form.get('privkey', None) # PEM encoded if latest_cert is not None and privkey is None: privkey = latest_cert.certificate_key client = Client(Account(KEY1, '*****@*****.**')) client.account_register() # IDN encode domain = idna.encode(domain).decode() cert, cert_key = client.obtain_certificate([domain, "www." + domain], privkey) cert_now = Certificates(domain, cert, cert_key) db.session.add(cert_now) db.session.commit() return jsonify({}), 200 except ValueError as e: return jsonify({'error': str(e)}), 403
def fqdn_check(match: regex.Match): mnfo = match.groupdict() valu = mnfo.get('valu') nval = unicodedata.normalize('NFKC', valu) nval = regex.sub(udots, '.', nval) nval = nval.strip().strip('.') try: idna.encode(nval, uts46=True).decode('utf8') except idna.IDNAError: try: nval.encode('idna').decode('utf8').lower() except UnicodeError: return None, {} return valu, {}
def get_expire_date(hostname): ## Encode in domains with IDN compilance hostname_idna = idna.encode(hostname) ## Open and connect socket sock = socket() sock.connect((hostname, 443)) ## Add context contex = SSL.Context(SSL.SSLv23_METHOD) contex.check_hostname = False contex.verify_mode = SSL.VERIFY_NONE ## HTTP request with SSL sock_ssl = SSL.Connection(contex, sock) sock_ssl.set_connect_state() sock_ssl.set_tlsext_host_name(hostname_idna) sock_ssl.do_handshake() ## Get certificate infos cert = sock_ssl.get_peer_certificate() crypto_cert = cert.to_cryptography() ## Close conection SSL sock_ssl.close() ## Close socket sock.close() ## Get expire date from certificate return crypto_cert.not_valid_after
def get_surt_host(url): try: host = urlparse(url).hostname except: # self.get_logger().debug("Failed to parse URL {}: {}".format(url, e)) return None if host is None or ExtractHostLinksJob.ip_pattern.match(host): return None host = host.strip().lower() if len(host) < 1 or len(host) > 253: return None parts = host.split('.') if parts[-1] == '': # trailing dot is allowed, strip it parts = parts[0:-1] if parts[0] == 'www' and len(parts) > 1: # strip leading 'www' to reduce number of "duplicate" hosts parts = parts[1:] for i in range(0, len(parts)): part = parts[i] if not ExtractHostLinksJob.host_part_pattern.match(part): try: idn = idna.encode(part).decode('ascii') except (idna.IDNAError, UnicodeDecodeError, IndexError, UnicodeEncodeError, Exception): # self.get_logger().debug("Invalid host name: {}".format(url)) return None if ExtractHostLinksJob.host_part_pattern.match(idn): parts[i] = idn else: # self.get_logger().debug("Invalid host name: {}".format(url)) return None parts.reverse() return '.'.join(parts)
def _idna_encode(self, value): idna = _lazy_import_idna() parsed = urllib_parse.urlparse(value) if parsed.port: netloc = ( idna.encode(parsed.hostname) + ":{0}".format(parsed.port).encode("ascii")).decode("ascii") else: netloc = idna.encode(parsed.hostname).decode("ascii") # Note that building a URL in this fashion means it should be # semantically indistinguishable from the original but is not # guaranteed to be exactly the same. return urllib_parse.urlunparse( (parsed.scheme, netloc, parsed.path, parsed.params, parsed.query, parsed.fragment))
def make_query_args(address, exact_extension=True, wildcard=None, domain_search=False): assert isinstance(address, six.text_type),\ "address should be of type %s" % six.text_type.__name__ conf = dict(param_tools.get_global_parameters("modoboa_amavis")) local_part, domain = split_address(address) if not conf["localpart_is_case_sensitive"]: local_part = local_part.lower() if domain: domain = domain.lstrip("@").rstrip(".") domain = domain.lower() orig_domain = domain domain = idna.encode(domain, uts46=True).decode("ascii") delimiter = conf["recipient_delimiter"] local_part, extension = split_local_part(local_part, delimiter=delimiter) query_args = [] if (conf["localpart_is_case_sensitive"] or (domain and domain != orig_domain)): query_args.append(address) if extension: query_args.append("%s%s%s@%s" % (local_part, delimiter, extension, domain)) if delimiter and not exact_extension and wildcard: query_args.append("%s%s%s@%s" % (local_part, delimiter, wildcard, domain)) query_args.append("%s@%s" % (local_part, domain)) if domain_search: query_args.append("@%s" % domain) query_args.append("@.") return query_args
def normalize(self, hostname): hostname = Hostname.check_type(hostname) if not hostname: raise ObservableValidationError("Invalid Hostname (check_type={}): {}".format(Hostname.check_type(hostname), hostname)) self.idna = unicode(idna.encode(hostname.lower())) self.value = unicode(idna.decode(hostname.lower()))
def get_certificate(self): try: hostname_idna = idna.encode(self.domain) sock = socket() sock.settimeout(10) sock.connect((self.domain, 443)) sock.settimeout(None) ctx = SSL.Context(SSL.SSLv23_METHOD) # most compatible ctx.check_hostname = False ctx.verify_mode = SSL.VERIFY_NONE sock_ssl = SSL.Connection(ctx, sock) sock_ssl.set_connect_state() sock_ssl.set_tlsext_host_name(hostname_idna) sock_ssl.do_handshake() cert = sock_ssl.get_peer_certificate() self.crypto_cert = cert.to_cryptography() sock_ssl.close() sock.close() self.get_issuer() except: pass
def with_host(self, host): """Return a new URL with host replaced. Autoencode host if needed. Changing host for relative URLs is not allowed, use .join() instead. """ # N.B. doesn't cleanup query/fragment if not isinstance(host, str): raise TypeError("Invalid host type") if not self.is_absolute(): raise ValueError("host replacement is not allowed " "for relative URLs") if not host: raise ValueError("host removing is not allowed") try: ip = ip_address(host) except ValueError: host = idna.encode(host, uts46=True).decode('ascii') else: if ip.version == 6: host = '[' + host + ']' val = self._val return URL(self._val._replace(netloc=self._make_netloc( val.username, val.password, host, val.port)), encoded=True)
def idna_encode(name): """ Borrowed wholesale from the Python Cryptography Project. It turns out that we can't just safely call `idna.encode`: it can explode for wildcard names. This avoids that problem. """ import idna try: for prefix in [u"*.", u"."]: if name.startswith(prefix): name = name[len(prefix) :] return prefix.encode("ascii") + idna.encode(name) return idna.encode(name) except idna.core.IDNAError: return None
def get_certificate(hostname, port): hostname_idna = idna.encode(hostname) sock = socket() sock.connect((hostname, port)) peername = sock.getpeername() ctx = SSL.Context(SSL.SSLv23_METHOD) # most compatible ctx.check_hostname = False ctx.verify_mode = SSL.VERIFY_NONE sock_ssl = SSL.Connection(ctx, sock) sock_ssl.set_connect_state() sock_ssl.set_tlsext_host_name(hostname_idna) sock_ssl.do_handshake() cert = sock_ssl.get_peer_certificate() crypto_cert = cert.to_cryptography() certs = sock_ssl.get_peer_cert_chain() for pos, cert in enumerate(certs): print("====SSL session certs[" + str(pos) + "]===") fd = os.open("cert" + str(pos) + ".pem", os.O_RDWR | os.O_CREAT) os.write(fd, crypto.dump_certificate(crypto.FILETYPE_PEM, cert)) os.close(fd) print_cert_info(cert.to_cryptography()) sock_ssl.close() sock.close() return HostInfo(cert=crypto_cert, peername=peername, hostname=hostname)
def _get_object(self, handle: str) -> Any: objects = {} with suppress(OBJECT_NOT_FOUND, INVALID_HANDLE): objects['contact'] = WHOIS.get_contact_by_handle(handle) with suppress(OBJECT_NOT_FOUND, INVALID_HANDLE): objects['nsset'] = WHOIS.get_nsset_by_handle(handle) with suppress(OBJECT_NOT_FOUND, INVALID_HANDLE): objects['keyset'] = WHOIS.get_keyset_by_handle(handle) with suppress(OBJECT_NOT_FOUND, INVALID_HANDLE): objects['registrar'] = WHOIS.get_registrar_by_handle(handle) if not handle.startswith("."): with suppress(OBJECT_NOT_FOUND, UNMANAGED_ZONE, INVALID_LABEL, TOO_MANY_LABELS, idna.IDNAError): idna_handle = idna.encode(handle).decode() try: objects['domain'] = WHOIS.get_domain_by_handle(idna_handle) except OBJECT_DELETE_CANDIDATE: objects['domain'] = None if not objects: raise WebwhoisError( code="OBJECT_NOT_FOUND", title=_("Record not found"), message=self.message_with_handle_in_html( _("%s does not match any record."), handle), object_not_found=True, ) return objects
def execute(self, domain_name): idns = [] oldidns = [] # Walk through all characters in the domain name domain = domain_name.rsplit('.', 1)[0] # Initially, replace single instances of homoglyphs idns = idns + self.replace_homoglyphs([domain]) # Now iterate over this list again, to replace additional homoglyphs and always remove duplicates while not set(oldidns) == set(idns): oldidns = idns idns = list(set(idns + self.replace_homoglyphs(idns))) domains = [] for idn in idns: try: clear = (idn + '.' + domain_name.rsplit('.', 1)[1]).encode('utf-8') puni = idna.encode(clear.decode('utf-8')) print clear, puni domains.append(puni) except: pass
def get_certificate(host_name, port, proxy_ip='', proxy_port=8080): sock = socks.socksocket( ) # Same API as socket.socket in the standard lib if proxy_ip: sock.set_proxy(socks.HTTP, proxy_ip, proxy_port) hostname_idna = idna.encode(host_name) # sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # sock = socket.socket() sock.connect((host_name, port)) peer_name = sock.getpeername() ctx = SSL.Context(SSL.SSLv23_METHOD) # most compatible ctx.check_hostname = False ctx.verify_mode = SSL.VERIFY_NONE sock_ssl = SSL.Connection(ctx, sock) sock_ssl.set_connect_state() sock_ssl.set_tlsext_host_name(hostname_idna) sock_ssl.do_handshake() cert = sock_ssl.get_peer_certificate() crypto_cert = cert.to_cryptography() sock_ssl.close() sock.close() return HostInfo(cert=crypto_cert, peername=peer_name, hostname=host_name)
def normalize(self, hostname): if not is_hostname(hostname): raise ObservableValidationError("Invalid Hostname (is_hostname={}): {}".format(is_hostname(hostname), hostname)) if hostname.endswith('.'): hostname = hostname[:-1] self.idna = unicode(idna.encode(hostname.lower())) self.value = unicode(idna.decode(hostname.lower()))
def get_certificate_from_client(hostname, port, CERT_FILE): """ :type hostname: str :type port: int :type CERT_FILE: str """ hostname_idna = idna.encode(hostname) sock = socket.socket() sock.connect((hostname, int(port))) ctx = SSL.Context(SSL.SSLv23_METHOD) # most compatible ctx.check_hostname = False ctx.verify_mode = SSL.VERIFY_NONE sock_ssl = SSL.Connection(ctx, sock) sock_ssl.set_connect_state() sock_ssl.set_tlsext_host_name(hostname_idna) sock_ssl.do_handshake() cert = sock_ssl.get_peer_certificate() sock_ssl.close() sock.close() _write_certificate(CERT_FILE, cert) subject_str = "".join( "/{0:s}={1:s}".format(name.decode(), value.decode()) for name, value in cert.get_issuer().get_components()) logging.debug(subject_str) return subject_str
def __init__(self, val='', *, encoded=False, strict=None): if strict is not None: # pragma: no cover warnings.warn("strict parameter is ignored") if isinstance(val, URL): self._val = val._val self._cache = val._cache return if isinstance(val, str): val = urlsplit(val) elif isinstance(val, SplitResult): if not encoded: raise ValueError("Cannot apply decoding to SplitResult") else: raise TypeError("Constructor parameter should be str") if not encoded: if not val[1]: # netloc netloc = '' else: netloc = val.hostname if netloc is None: raise ValueError( "Invalid URL: host is required for abolute urls.") try: netloc.encode('ascii') except UnicodeEncodeError: netloc = idna.encode(netloc, uts46=True).decode('ascii') else: try: ip = ip_address(netloc) except ValueError: pass else: if ip.version == 6: netloc = '[' + netloc + ']' if val.port: netloc += ':{}'.format(val.port) if val.username: user = _quote(val.username) else: user = '' if val.password: user += ':' + _quote(val.password) if user: netloc = user + '@' + netloc path = _quote(val[2], safe='+@:', protected='/+') if netloc: path = _normalize_path(path) query = _quote(val[3], safe='=+&?/:@', protected=PROTECT_CHARS, qs=True) fragment = _quote(val[4], safe='?/:@') val = SplitResult(val[0], netloc, path, query, fragment) self._val = val self._cache = {}
def runTest(self): if not self.fields: return try: types, source, to_unicode, to_ascii = ( unicode_fixup(field) for field in self.fields[:4]) if (unicode_fixup(u"\\uD804\\uDC39") in source and sys.version_info[0] < 3): raise unittest.SkipTest( "Python 2's Unicode support is too old for this test") except ValueError: raise unittest.SkipTest( "Test requires Python wide Unicode support") if source in _SKIP_TESTS: return if not to_unicode: to_unicode = source if not to_ascii: to_ascii = to_unicode nv8 = (len(self.fields) > 4 and self.fields[4]) try: output = idna.decode(source, uts46=True, strict=True) if to_unicode[0] == u"[": self.fail( "decode() did not emit required error {0} for {1}".format( to_unicode, repr(source))) self.assertEqual(output, to_unicode, "unexpected decode() output") except (idna.IDNAError, UnicodeError, ValueError) as exc: if unicode(exc).startswith(u"Unknown"): raise unittest.SkipTest( "Test requires support for a newer" " version of Unicode than this Python supports") if to_unicode[0] != u"[" and not nv8: raise for transitional in { u"B": (True, False), u"T": (True, ), u"N": (False, ), }[types]: try: output = idna.encode(source, uts46=True, strict=True, transitional=transitional).decode("ascii") if to_ascii[0] == u"[": self.fail( "encode(transitional={0}) did not emit required error {1} for {2}" .format(transitional, to_ascii, repr(source))) self.assertEqual( output, to_ascii, "unexpected encode(transitional={0}) output".format( transitional)) except (idna.IDNAError, UnicodeError, ValueError) as exc: if unicode(exc).startswith(u"Unknown"): raise unittest.SkipTest( "Test requires support for a newer" " version of Unicode than this Python supports") if to_ascii[0] != u"[" and not nv8: raise
async def getaddrinfo(host, port, family=0, type=0, proto=0, flags=0): """Look up a numeric address given a name. Arguments and return values are identical to :func:`socket.getaddrinfo`, except that this version is async. Also, :func:`trio.socket.getaddrinfo` correctly uses IDNA 2008 to process non-ASCII domain names. (:func:`socket.getaddrinfo` uses IDNA 2003, which can give the wrong result in some cases and cause you to connect to a different host than the one you intended; see `bpo-17305 <https://bugs.python.org/issue17305>`__.) This function's behavior can be customized using :func:`set_custom_hostname_resolver`. """ # If host and port are numeric, then getaddrinfo doesn't block and we can # skip the whole thread thing, which seems worthwhile. So we try first # with the _NUMERIC_ONLY flags set, and then only spawn a thread if that # fails with EAI_NONAME: def numeric_only_failure(exc): return isinstance(exc, _stdlib_socket.gaierror) and \ exc.errno == _stdlib_socket.EAI_NONAME async with _try_sync(numeric_only_failure): return _stdlib_socket.getaddrinfo( host, port, family, type, proto, flags | _NUMERIC_ONLY ) # That failed; it's a real hostname. We better use a thread. # # Also, it might be a unicode hostname, in which case we want to do our # own encoding using the idna module, rather than letting Python do # it. (Python will use the old IDNA 2003 standard, and possibly get the # wrong answer - see bpo-17305). However, the idna module is picky, and # will refuse to process some valid hostname strings, like "::1". So if # it's already ascii, we pass it through; otherwise, we encode it to. if isinstance(host, str): try: host = host.encode("ascii") except UnicodeEncodeError: # UTS-46 defines various normalizations; in particular, by default # idna.encode will error out if the hostname has Capital Letters # in it; with uts46=True it will lowercase them instead. host = _idna.encode(host, uts46=True) hr = _resolver.get(None) if hr is not None: return await hr.getaddrinfo(host, port, family, type, proto, flags) else: return await run_sync_in_worker_thread( _stdlib_socket.getaddrinfo, host, port, family, type, proto, flags, cancellable=True )
def sendmail(self, subject, body): """ Send an email to the address. """ from_address = "{0}@{1}".format( app.config['POSTMASTER'], idna.encode(app.config['DOMAIN']).decode('ascii'), ) with smtplib.SMTP(app.config['HOST_AUTHSMTP'], port=10025) as smtp: to_address = "{0}@{1}".format( self.localpart, idna.encode(self.domain_name).decode('ascii'), ) msg = text.MIMEText(body) msg['Subject'] = subject msg['From'] = from_address msg['To'] = to_address smtp.sendmail(from_address, [to_address], msg.as_string())
def _get_idna_encoded_host(host): import idna try: host = idna.encode(host, uts46=True).decode('utf-8') except idna.IDNAError: raise UnicodeError return host
def _idna_encode(self, value): # Import idna lazily becase it allocates a decent amoutn of memory, and # we're only using it in deprecated paths. import idna _, address = parseaddr(value) parts = address.split(u"@") return parts[0] + "@" + idna.encode(parts[1]).decode("ascii")
def __init__(self, domain): """Handle all the possible domain types coming in.""" domain_unicode = self._try_parse_to_unicode_domain(domain) if domain_unicode is None: raise InvalidDomainException(f'Invalid domain: {repr(domain)}') self._domain_unicode = domain_unicode self._domain_ascii = idna.encode(domain_unicode).decode()
def normalize(self): self.value = refang(self.value.lower()) try: self.idna = unicode(idna.encode(self.value)) except idna.core.InvalidCodepoint: pass except Exception, e: raise ObservableValidationError(e.message)
async def getaddrinfo(host, port, family=0, type=0, proto=0, flags=0): """Look up a numeric address given a name. Arguments and return values are identical to :func:`socket.getaddrinfo`, except that this version is async. Also, :func:`trio.socket.getaddrinfo` correctly uses IDNA 2008 to process non-ASCII domain names. (:func:`socket.getaddrinfo` uses IDNA 2003, which can give the wrong result in some cases and cause you to connect to a different host than the one you intended; see `bpo-17305 <https://bugs.python.org/issue17305>`__.) This function's behavior can be customized using :func:`set_custom_hostname_resolver`. """ # If host and port are numeric, then getaddrinfo doesn't block and we can # skip the whole thread thing, which seems worthwhile. So we try first # with the _NUMERIC_ONLY flags set, and then only spawn a thread if that # fails with EAI_NONAME: def numeric_only_failure(exc): return isinstance(exc, gaierror) and exc.errno == EAI_NONAME async with _try_sync(numeric_only_failure): return _stdlib_socket.getaddrinfo( host, port, family, type, proto, flags | _NUMERIC_ONLY ) # That failed; it's a real hostname. We better use a thread. # # Also, it might be a unicode hostname, in which case we want to do our # own encoding using the idna module, rather than letting Python do # it. (Python will use the old IDNA 2003 standard, and possibly get the # wrong answer - see bpo-17305). However, the idna module is picky, and # will refuse to process some valid hostname strings, like "::1". So if # it's already ascii, we pass it through; otherwise, we encode it to. if isinstance(host, str): try: host = host.encode("ascii") except UnicodeEncodeError: # UTS-46 defines various normalizations; in particular, by default # idna.encode will error out if the hostname has Capital Letters # in it; with uts46=True it will lowercase them instead. host = _idna.encode(host, uts46=True) hr = _resolver.get(None) if hr is not None: return await hr.getaddrinfo(host, port, family, type, proto, flags) else: return await run_sync_in_worker_thread( _stdlib_socket.getaddrinfo, host, port, family, type, proto, flags, cancellable=True )
def validate_hostname(name): """ Validate a hostname. """ if name.endswith("."): raise ValidationError("Name must not end with a punctuation mark.") # Assume we are not running a tld if '.' not in name: raise ValidationError("Name must include a tld.") # Any label in can be max 63 characters, after idna encoding for label in name.split("."): if label == '': raise ValidationError("Too many punctation marks") if label[0] == "-" or label[-1] == "-": raise ValidationError( "Can not start or end a label with a hyphen '{}'".format( label)) if "--" in label: raise ValidationError( "Label can not contain double hyphen '{}'".format(label)) if len(label) > 63: raise ValidationError( "Label '{}' is {} characters long, maximum is 63".format( label, len(label))) # convert to .isascii in python 3.7 if all(ord(char) < 128 for char in label): if "*" in label: if len(label) > 1: raise ValidationError("Wildcard must be standalone") continue label_regex = r"^_?[a-zA-Z0-9\-]+$" validator = RegexValidator(label_regex, message="Label '{}' is not valid. " "Must be within '{}'.".format( label, label_regex)) validator(label) else: try: idna.encode(label) except idna.core.InvalidCodepoint as e: raise ValidationError("Invalid label '{}': {}".format( label, e)) except idna.core.IDNAError as e: raise ValidationError( "Label '{}' could not be idna encoded: {}".format( label, e))
def normalize(self, hostname): hostname = Hostname.check_type(hostname) if not hostname: raise ObservableValidationError("Invalid Hostname (check_type={}): {}".format(Hostname.check_type(hostname), hostname)) self.value = unicode(hostname.lower()) try: self.idna = unicode(idna.encode(hostname.lower())) except idna.core.InvalidCodepoint: pass
def handle_rf(self, order_item): assert order_item original_domain = order_item['domain'] order_item['domain'] = idna.encode(order_item['domain']) logger.info("IDNA convert: %s -> %s" % (original_domain, order_item['domain'])) return order_item
def idna_encoder(name): if any(ord(c) > 128 for c in name): try: return idna.encode(name.lower(), strict=True, std3_rules=True) except idna.IDNAError: raise exceptions.InvalidAuthority(self.authority) return name
def to_idna(domain): # If the domain is passed as a bytes object (alias for str in Python 2), # then assume it is already IDNA encoded and decode as if ASCII and work # with unicode (Py 2 unicode/Py 3 str) instances. if isinstance(domain, bytes): return domain.decode("ascii") # IDNA-encode, but get back a unicode instance. return idna.encode(domain).decode("ascii")
def process_bind_param(self, value, dialect): try: localpart, domain_name = value.split('@') return "{0}@{1}".format( localpart, idna.encode(domain_name).decode('ascii'), ).lower() except ValueError: pass
def build(self): ret = b"" if self._ec_point_format is not None: ec_point_format_struct = construct.Container( ec_point_format_length=len(self._ec_point_format), ec_point_format=self._get_bytes_from_ec_point_format( self._ec_point_format ) ) ret += hello_constructs.Extension.build( construct.Container( extension_type=11, extension_length=len(hello_constructs.ECPointFormat.build( ec_point_format_struct )), extension_struct=ec_point_format_struct ) ) if self._ec_curves is not None: ec_curves_struct = construct.Container( ec_curves_length=len(self._ec_curves) * 2, named_curves=self._get_bytes_from_ec_curves( self._ec_curves ) ) ret += hello_constructs.Extension.build( construct.Container( extension_type=10, extension_length=len(hello_constructs.ECCurves.build( ec_curves_struct )), extension_struct=ec_curves_struct ) ) if self._hostname is not None: encoded_hostname = idna.encode(self._hostname) sni_struct = construct.Container( server_name_list_length=len(encoded_hostname) + 3, name_type=0, server_name_length=len(encoded_hostname), server_name=encoded_hostname ) ret += hello_constructs.Extension.build( construct.Container( extension_type=0, extension_length=len(hello_constructs.ServerName.build( sni_struct )), extension_struct=sni_struct ) ) return ret
def __init__(self, plugin=None, protocol=None, auth=None, domain=None, port=None, path=None, query=None, fragment=None): self.plugin = plugin self.protocol = protocol self.auth = auth self.domain = idna.encode(to_unicode(domain)) self.port = port self.path = path self.query = query self.fragment = fragment
def idna_encode(name): if name and any([ord(x) > 128 for x in name]): try: import idna except ImportError: raise LocationParseError("Unable to parse URL without the 'idna' module") try: return idna.encode(name.lower(), strict=True, std3_rules=True) except idna.IDNAError: raise LocationParseError(u"Name '%s' is not a valid IDNA label" % name) return name
def normalize(self): self.value = refang(self.value.lower()) # Remove trailing dot if existing if self.value.endswith("."): self.value = self.value[:-1] try: self.idna = unicode(idna.encode(self.value)) except idna.core.InvalidCodepoint: pass except Exception, e: raise ObservableValidationError(e.message)
def _idnaBytes(text): """ Convert some text typed by a human into some ASCII bytes. This is a copy of twisted.internet._idna._idnaBytes. For documentation, see the twisted documentation. """ try: import idna except ImportError: return text.encode("idna") else: return idna.encode(text)
def ishost(value): MAX_HOSTNAME_LEN = 253 try: # returns bytestring, then encode to str value = to_str(idna.encode(value)) except AttributeError: pass if value.endswith('.'): value = value[:-1] if not value or len(value) > MAX_HOSTNAME_LEN: return False return all(map(_RE_ISH.match, value.split('.')))
def _idna_encode(self, value): parsed = urllib_parse.urlparse(value) if parsed.port: netloc = ( idna.encode(parsed.hostname) + ":{0}".format(parsed.port).encode("ascii") ).decode("ascii") else: netloc = idna.encode(parsed.hostname).decode("ascii") # Note that building a URL in this fashion means it should be # semantically indistinguishable from the original but is not # guaranteed to be exactly the same. return urllib_parse.urlunparse(( parsed.scheme, netloc, parsed.path, parsed.params, parsed.query, parsed.fragment ))
def ace_address(self): if not is_pure_ascii(self.mailbox): raise ValueError('address {} has no ASCII-compatable encoding' .format(self.address.encode('utf-8'))) ace_hostname = self.hostname if not is_pure_ascii(self.hostname): try: ace_hostname = idna.encode(self.hostname) except idna.IDNAError: raise ValueError('address {} has no ASCII-compatable encoding' .format(self.address.encode('utf-8'))) return '{}@{}'.format(self.mailbox, ace_hostname)
def test_encode(self): self.assertEqual(idna.encode('xn--zckzah.xn--zckzah'), b'xn--zckzah.xn--zckzah') self.assertEqual(idna.encode(u'\u30c6\u30b9\u30c8.xn--zckzah'), b'xn--zckzah.xn--zckzah') self.assertEqual(idna.encode(u'\u30c6\u30b9\u30c8.\u30c6\u30b9\u30c8'), b'xn--zckzah.xn--zckzah') self.assertEqual(idna.encode('abc.abc'), b'abc.abc') self.assertEqual(idna.encode('xn--zckzah.abc'), b'xn--zckzah.abc') self.assertEqual(idna.encode(u'\u30c6\u30b9\u30c8.abc'), b'xn--zckzah.abc') self.assertEqual(idna.encode(u'\u0521\u0525\u0523-\u0523\u0523-----\u0521\u0523\u0523\u0523.aa'), b'xn---------90gglbagaar.aa') self.assertRaises(idna.IDNAError, idna.encode, u'\u0521\u0524\u0523-\u0523\u0523-----\u0521\u0523\u0523\u0523.aa') self.assertEqual(idna.encode('a'*63), b'a'*63) self.assertRaises(idna.IDNAError, idna.encode, 'a'*64)
def test_encode(self): self.assertEqual(idna.encode('xn--zckzah.xn--zckzah'), b'xn--zckzah.xn--zckzah') self.assertEqual(idna.encode(u'\u30c6\u30b9\u30c8.xn--zckzah'), b'xn--zckzah.xn--zckzah') self.assertEqual(idna.encode(u'\u30c6\u30b9\u30c8.\u30c6\u30b9\u30c8'), b'xn--zckzah.xn--zckzah') self.assertEqual(idna.encode('abc.abc'), b'abc.abc') self.assertEqual(idna.encode('xn--zckzah.abc'), b'xn--zckzah.abc') self.assertEqual(idna.encode(u'\u30c6\u30b9\u30c8.abc'), b'xn--zckzah.abc')
def runTest(self): if not self.fields: return try: types, source, to_unicode, to_ascii = (unicode_fixup(field) for field in self.fields[:4]) if (unicode_fixup(u"\\uD804\\uDC39") in source and sys.version_info[0] < 3): raise unittest.SkipTest( "Python 2's Unicode support is too old for this test") except ValueError: raise unittest.SkipTest( "Test requires Python wide Unicode support") if source in _SKIP_TESTS: return if not to_unicode: to_unicode = source if not to_ascii: to_ascii = to_unicode nv8 = (len(self.fields) > 4 and self.fields[4]) try: output = idna.decode(source, uts46=True, strict=True) if to_unicode[0] == u"[": self.fail("decode() did not emit required error {0} for {1}".format(to_unicode, repr(source))) self.assertEqual(output, to_unicode, "unexpected decode() output") except (idna.IDNAError, UnicodeError, ValueError) as exc: if unicode(exc).startswith(u"Unknown"): raise unittest.SkipTest("Test requires support for a newer" " version of Unicode than this Python supports") if to_unicode[0] != u"[" and not nv8: raise for transitional in { u"B": (True, False), u"T": (True,), u"N": (False,), }[types]: try: output = idna.encode(source, uts46=True, strict=True, transitional=transitional).decode("ascii") if to_ascii[0] == u"[": self.fail( "encode(transitional={0}) did not emit required error {1} for {2}". format(transitional, to_ascii, repr(source))) self.assertEqual(output, to_ascii, "unexpected encode(transitional={0}) output". format(transitional)) except (idna.IDNAError, UnicodeError, ValueError) as exc: if unicode(exc).startswith(u"Unknown"): raise unittest.SkipTest("Test requires support for a newer" " version of Unicode than this Python supports") if to_ascii[0] != u"[" and not nv8: raise
def handshake(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) tls_ctx = SSL.Context(SSL.SSLv23_METHOD) conn = SSL.Connection( tls_ctx, sock ) conn.connect((idna.encode(hostname).encode("ascii"), 443)) conn.do_handshake() cert = conn.get_peer_certificate() conn.shutdown() conn.close() return cert
def _dnsname_to_stdlib(name): """ Converts a dNSName SubjectAlternativeName field to the form used by the standard library on the given Python version. Cryptography produces a dNSName as a unicode string that was idna-decoded from ASCII bytes. We need to idna-encode that string to get it back, and then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8). """ name = idna.encode(name) if sys.version_info >= (3, 0): name = name.decode('utf-8') return name
def splitaddress(address): try: address = to_str(idna.encode(address)) except (AttributeError, idna.IDNAError): pass sep = ']:' if address.split(':', 2)[2:] else ':' parts = address.rsplit(sep, 1) try: addr, port = parts port = int(port) except ValueError: addr = parts[0] port = None return addr, port