def __init__(self, host, port, bindAddress, connector, reactor=None): # BaseClient.__init__ is invoked later self.connector = connector self.addr = (host, port) whenDone = self.resolveAddress err = None skt = None if abstract.isIPAddress(host): self._requiresResolution = False elif abstract.isIPv6Address(host): self._requiresResolution = False self.addr = _resolveIPv6(host, port) self.addressFamily = socket.AF_INET6 self._addressType = address.IPv6Address else: self._requiresResolution = True try: skt = self.createInternetSocket() except socket.error as se: err = error.ConnectBindError(se.args[0], se.args[1]) whenDone = None if whenDone and bindAddress is not None: try: if abstract.isIPv6Address(bindAddress[0]): bindinfo = _resolveIPv6(*bindAddress) else: bindinfo = bindAddress skt.bind(bindinfo) except socket.error as se: err = error.ConnectBindError(se.args[0], se.args[1]) whenDone = None self._finishInit(whenDone, skt, err, reactor)
def write(self, datagram, addr=None): """ Write a datagram. @param addr: should be a tuple (ip, port), can be None in connected mode. """ if self._connectedAddr: assert addr in (None, self._connectedAddr) try: return self.socket.send(datagram) except OSError as se: no = se.args[0] if no == errno.WSAEINTR: return self.write(datagram) elif no == errno.WSAEMSGSIZE: raise error.MessageLengthError("message too long") elif no in ( errno.WSAECONNREFUSED, errno.WSAECONNRESET, ERROR_CONNECTION_REFUSED, ERROR_PORT_UNREACHABLE, ): self.protocol.connectionRefused() else: raise else: assert addr != None if (not isIPAddress(addr[0]) and not isIPv6Address(addr[0]) and addr[0] != "<broadcast>"): raise error.InvalidAddressError( addr[0], "write() only accepts IP addresses, not hostnames") if isIPAddress(addr[0]) and self.addressFamily == socket.AF_INET6: raise error.InvalidAddressError( addr[0], "IPv6 port write() called with IPv4 address") if isIPv6Address(addr[0]) and self.addressFamily == socket.AF_INET: raise error.InvalidAddressError( addr[0], "IPv4 port write() called with IPv6 address") try: return self.socket.sendto(datagram, addr) except OSError as se: no = se.args[0] if no == errno.WSAEINTR: return self.write(datagram, addr) elif no == errno.WSAEMSGSIZE: raise error.MessageLengthError("message too long") elif no in ( errno.WSAECONNREFUSED, errno.WSAECONNRESET, ERROR_CONNECTION_REFUSED, ERROR_PORT_UNREACHABLE, ): # in non-connected UDP ECONNREFUSED is platform dependent, # I think and the info is not necessarily useful. # Nevertheless maybe we should call connectionRefused? XXX return else: raise
def test_invalidWithScopeID(self): """ An otherwise invalid IPv6 address literal is still invalid with a trailing scope identifier. """ self.assertFalse(isIPv6Address("%eth0")) self.assertFalse(isIPv6Address(":%eth0")) self.assertFalse(isIPv6Address("hello%eth0"))
def test_scopeID(self): """ An otherwise valid IPv6 address literal may also include a C{"%"} followed by an arbitrary scope identifier. """ self.assertTrue(isIPv6Address("fe80::1%eth0")) self.assertTrue(isIPv6Address("fe80::2%1")) self.assertTrue(isIPv6Address("fe80::3%en2"))
def write(self, datagram, addr=None): """ Write a datagram. @type datagram: L{bytes} @param datagram: The datagram to be sent. @type addr: L{tuple} containing L{str} as first element and L{int} as second element, or L{None} @param addr: A tuple of (I{stringified IPv4 or IPv6 address}, I{integer port number}); can be L{None} in connected mode. """ if self._connectedAddr: assert addr in (None, self._connectedAddr) try: return self.socket.send(datagram) except socket.error as se: no = se.args[0] if no == EINTR: return self.write(datagram) elif no == EMSGSIZE: raise error.MessageLengthError("message too long") elif no == ECONNREFUSED: self.protocol.connectionRefused() else: raise else: assert addr != None if (not abstract.isIPAddress(addr[0]) and not abstract.isIPv6Address(addr[0]) and addr[0] != "<broadcast>"): raise error.InvalidAddressError( addr[0], "write() only accepts IP addresses, not hostnames") if ((abstract.isIPAddress(addr[0]) or addr[0] == "<broadcast>") and self.addressFamily == socket.AF_INET6): raise error.InvalidAddressError( addr[0], "IPv6 port write() called with IPv4 or broadcast address") if (abstract.isIPv6Address(addr[0]) and self.addressFamily == socket.AF_INET): raise error.InvalidAddressError( addr[0], "IPv4 port write() called with IPv6 address") try: return self.socket.sendto(datagram, addr) except socket.error as se: no = se.args[0] if no == EINTR: return self.write(datagram, addr) elif no == EMSGSIZE: raise error.MessageLengthError("message too long") elif no == ECONNREFUSED: # in non-connected UDP ECONNREFUSED is platform dependent, I # think and the info is not necessarily useful. Nevertheless # maybe we should call connectionRefused? XXX return else: raise
def write(self, datagram, addr=None): """ Write a datagram. @type datagram: C{str} @param datagram: The datagram to be sent. @type addr: C{tuple} containing C{str} as first element and C{int} as second element, or L{None} @param addr: A tuple of (I{stringified IPv4 or IPv6 address}, I{integer port number}); can be L{None} in connected mode. """ if self._connectedAddr: assert addr in (None, self._connectedAddr) try: return self.socket.send(datagram) except socket.error as se: no = se.args[0] if no == EINTR: return self.write(datagram) elif no == EMSGSIZE: raise error.MessageLengthError("message too long") elif no == ECONNREFUSED: self.protocol.connectionRefused() else: raise else: assert addr != None if (not abstract.isIPAddress(addr[0]) and not abstract.isIPv6Address(addr[0]) and addr[0] != "<broadcast>"): raise error.InvalidAddressError( addr[0], "write() only accepts IP addresses, not hostnames") if ((abstract.isIPAddress(addr[0]) or addr[0] == "<broadcast>") and self.addressFamily == socket.AF_INET6): raise error.InvalidAddressError( addr[0], "IPv6 port write() called with IPv4 or broadcast address") if (abstract.isIPv6Address(addr[0]) and self.addressFamily == socket.AF_INET): raise error.InvalidAddressError( addr[0], "IPv4 port write() called with IPv6 address") try: return self.socket.sendto(datagram, addr) except socket.error as se: no = se.args[0] if no == EINTR: return self.write(datagram, addr) elif no == EMSGSIZE: raise error.MessageLengthError("message too long") elif no == ECONNREFUSED: # in non-connected UDP ECONNREFUSED is platform dependent, I # think and the info is not necessarily useful. Nevertheless # maybe we should call connectionRefused? XXX return else: raise
def test_unicodeAndBytes(self): """ L{isIPv6Address} evaluates ASCII-encoded bytes as well as text. """ self.assertTrue(isIPv6Address(b"fe80::2%1")) self.assertTrue(isIPv6Address(u"fe80::2%1")) self.assertFalse(isIPv6Address(u"\u4321")) self.assertFalse(isIPv6Address(u"hello%eth0")) self.assertFalse(isIPv6Address(b"hello%eth0"))
def write(self, datagram, addr=None): """ Write a datagram. @param addr: should be a tuple (ip, port), can be None in connected mode. """ if self._connectedAddr: assert addr in (None, self._connectedAddr) try: return self.socket.send(datagram) except socket.error as se: no = se.args[0] if no == errno.WSAEINTR: return self.write(datagram) elif no == errno.WSAEMSGSIZE: raise error.MessageLengthError("message too long") elif no in (errno.WSAECONNREFUSED, errno.WSAECONNRESET, ERROR_CONNECTION_REFUSED, ERROR_PORT_UNREACHABLE): self.protocol.connectionRefused() else: raise else: assert addr != None if (not isIPAddress(addr[0]) and not isIPv6Address(addr[0]) and addr[0] != "<broadcast>"): raise error.InvalidAddressError( addr[0], "write() only accepts IP addresses, not hostnames") if isIPAddress(addr[0]) and self.addressFamily == socket.AF_INET6: raise error.InvalidAddressError( addr[0], "IPv6 port write() called with IPv4 address") if isIPv6Address(addr[0]) and self.addressFamily == socket.AF_INET: raise error.InvalidAddressError( addr[0], "IPv4 port write() called with IPv6 address") try: return self.socket.sendto(datagram, addr) except socket.error as se: no = se.args[0] if no == errno.WSAEINTR: return self.write(datagram, addr) elif no == errno.WSAEMSGSIZE: raise error.MessageLengthError("message too long") elif no in (errno.WSAECONNREFUSED, errno.WSAECONNRESET, ERROR_CONNECTION_REFUSED, ERROR_PORT_UNREACHABLE): # in non-connected UDP ECONNREFUSED is platform dependent, # I think and the info is not necessarily useful. # Nevertheless maybe we should call connectionRefused? XXX return else: raise
def preprocess(self, request): request.headers = request.getAllHeaders() request.hostname = request.getRequestHostname().split(':')[0] request.port = request.getHost().port if (request.hostname == 'localhost' or isIPAddress(request.hostname) or isIPv6Address(request.hostname)): request.tid = 1 else: request.tid = State.tenant_hostname_id_map.get(request.hostname, 1) request.client_ip = request.headers.get('gl-forwarded-for') request.client_proto = 'https' if request.client_ip is None: request.client_ip = request.getClientIP() request.client_proto = 'http' request.client_using_tor = request.client_ip in State.tor_exit_set or \ request.port == 8083 if 'x-tor2web' in request.headers: request.client_using_tor = False request.language = unicode(self.detect_language(request)) if 'multilang' in request.args: request.language = None
def is_valid_matrix_server_name(string: str) -> bool: """Validate that the given string is a valid Matrix server name. A string is a valid Matrix server name if it is one of the following, plus an optional port: a. IPv4 address b. IPv6 literal (`[IPV6_ADDRESS]`) c. A valid hostname :param string: The string to validate :return: Whether the input is a valid Matrix server name """ try: host, port = parse_server_name(string) except ValueError: return False valid_ipv4_addr = isIPAddress(host) valid_ipv6_literal = (host[0] == "[" and host[-1] == "]" and isIPv6Address(host[1:-1])) return valid_ipv4_addr or valid_ipv6_literal or is_valid_hostname(host)
def resolveAddress(self): if abstract.isIPAddress(self.addr[0]) or abstract.isIPv6Address( self.addr[0]): self._setRealAddress(self.addr[0]) else: d = self.reactor.resolve(self.addr[0]) d.addCallbacks(self._setRealAddress, self.failIfNotConnected)
def noticed(self, user, channel, message): # We only care about notices from the server, not from users. # Users have a hostmask as "user", servers do not. if '!' in user: return if self.factory.connregex is None: return match = self.factory.connregex.search(message) if match is None: return d = match.groupdict() nick = d['nick'] user = d['user'] ip = d['ip'] host = d.get("host", None) if not isIPAddress(d['ip']) and not isIPv6Address(d['ip']): return hostmask = f'{nick}!{user}@{host or ip}' scansets = set() for mask, pattern, sets in self.factory.masks: if pattern.match(hostmask) is not None: scansets.update(sets) if ip in self.immune_cache: log.msg(f'Immunity given to {hostmask} (for IP {ip})') result = None elif ip in self.ip_cache: result = self.ip_cache.get(ip) log.msg(f'Cache hit for {hostmask}: {result}') else: log.msg(f'Scanning {hostmask} on scanners {" ".join(scansets)}') result = yield self.factory.scanner.scan(ip, scansets) self.ip_cache.set(ip, result, self.factory.cache_time) if result is not None: scanset, result = result formats = { 'NICK': nick, 'USER': user, 'IP': ip, 'MASK': hostmask, 'DESC': result, 'CHAN': self.factory.channel } formats['UREAS'] = scanset.user_reason.format(**formats) formats['OREAS'] = scanset.oper_reason.format(**formats) for action in self.factory.actions: self.sendLine(action.format(**formats)) log.msg('KILL {MASK} for {OREAS}'.format(**formats)) else: log.msg(f'GOOD {hostmask}')
def apply(self, actionType, user, param, settingUser, uid, adding, *params, **kw): if adding: userHost = user.realHost if isIPv6Address(userHost): user.changeHost("cloak", self.applyIPv6Cloak(userHost)) elif isIPAddress(userHost): user.changeHost("cloak", self.applyIPv4Cloak(userHost)) else: if "." in userHost: user.changeHost("cloak", self.applyHostCloak(userHost, user.ip)) else: if isIPv6Address(user.ip): return self.applyIPv6Cloak(user.ip) else: return self.applyIPv4Cloak(user.ip) else: user.resetHost("cloak")
class Server(object): """ Represents a server. """ def __init__(self, id=None, uri=None, sharedSecret=None, thisServer=False): self.id = id self.uri = uri self.thisServer = thisServer self.ips = set() self.allowed_from_ips = set() self.shared_secret = sharedSecret self.v5 = False # Needs old style urn:uuid cu-address def details(self): if not hasattr(self, "ssl"): self._parseDetails() return (self.ssl, self.host, self.port, self.path,) def check(self, ignoreIPLookupFailures=False): # Check whether this matches the current server parsed_uri = urlparse.urlparse(self.uri) if parsed_uri.hostname == config.ServerHostName: if parsed_uri.scheme == "http": if config.HTTPPort: self.thisServer = parsed_uri.port in (config.HTTPPort,) + tuple(config.BindHTTPPorts) elif parsed_uri.scheme == "https": if config.SSLPort: self.thisServer = parsed_uri.port in (config.SSLPort,) + tuple(config.BindSSLPorts) # Need to cache IP addresses try: ips = getIPsFromHost(parsed_uri.hostname) except socket.gaierror, e: msg = "Unable to lookup ip-addr for server '{}': {}".format(parsed_uri.hostname, str(e)) log.error(msg) if ignoreIPLookupFailures: ips = () else: raise ValueError(msg) self.ips = set(ips) actual_ips = set() for item in self.allowed_from_ips: if not isIPAddress(item) and not isIPv6Address(item): try: ips = getIPsFromHost(item) except socket.gaierror, e: msg = "Unable to lookup ip-addr for allowed-from '{}': {}".format(item, str(e)) log.error(msg) if not ignoreIPLookupFailures: raise ValueError(msg) else: actual_ips.update(ips) else: actual_ips.add(item)
def __init__(self, skt, addr, factory, reactor, interface=''): self.skt = skt self.addr = addr self.factory = factory self.reactor = reactor self._realPortNumber = skt.getsockname()[1] self._addressType = address.IPv4Address if abstract.isIPv6Address(interface): self._addressType = address.IPv6Address
def __init__(self, port, factory, backlog=50, interface="", reactor=None): self.port = port self.factory = factory self.backlog = backlog self.interface = interface self.reactor = reactor if isIPv6Address(interface): self.addressFamily = socket.AF_INET6 self._addressType = address.IPv6Address
def __init__(self, port, factory, backlog=50, interface='', reactor=None): self.port = port self.factory = factory self.backlog = backlog self.interface = interface self.reactor = reactor if isIPv6Address(interface): self.addressFamily = socket.AF_INET6 self._addressType = address.IPv6Address
def _setAddressFamily(self): """ Resolve address family for the socket. """ if abstract.isIPv6Address(self.interface): self.addressFamily = socket.AF_INET6 elif abstract.isIPAddress(self.interface): self.addressFamily = socket.AF_INET elif self.interface: raise error.InvalidAddressError(self.interface, "not an IPv4 or IPv6 address.")
def connect(self, host, port): """ 'Connect' to remote server. """ if self._connectedAddr: raise RuntimeError("already connected, reconnecting is not currently supported") if not abstract.isIPAddress(host) and not abstract.isIPv6Address(host): raise error.InvalidAddressError(host, "not an IPv4 or IPv6 address.") self._connectedAddr = (host, port) self.socket.connect((host, port))
def _aaaaRecords(self, name): """ Return a tuple of L{dns.RRHeader} instances for all of the IPv6 addresses in the hosts file. """ return tuple( dns.RRHeader(name, dns.AAAA, dns.IN, self.ttl, dns.Record_AAAA(addr, self.ttl)) for addr in searchFileForAll(FilePath(self.file), name) if isIPv6Address(addr))
def connect(self, host, port): """ 'Connect' to remote server. """ if self._connectedAddr: raise RuntimeError("already connected, reconnecting is not currently supported") if not abstract.isIPAddress(host) and not abstract.isIPv6Address(host): raise error.InvalidAddressError( host, 'not an IPv4 or IPv6 address.') self._connectedAddr = (host, port) self.socket.connect((host, port))
def _setAddressFamily(self): """ Resolve address family for the socket. """ if abstract.isIPv6Address(self.interface): self.addressFamily = socket.AF_INET6 elif abstract.isIPAddress(self.interface): self.addressFamily = socket.AF_INET elif self.interface: raise error.InvalidAddressError(self.interface, 'not an IPv4 or IPv6 address.')
def open_socket_listen(ip, port): if isIPv6Address(ip): s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) else: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.setblocking(False) s.bind((ip, port)) s.listen(1024) return s
def listenTCP(self, port, factory, backlog=50, interface=''): """ Fake L{IReactorTCP.listenTCP}, that logs the call and returns an L{IListeningPort}. """ self.tcpServers.append((port, factory, backlog, interface)) if isIPv6Address(interface): address = IPv6Address('TCP', interface, port) else: address = IPv4Address('TCP', '0.0.0.0', port) return _FakePort(address)
def __init__(self, host, port, factory, timeout, bindAddress, reactor=None): if isinstance(port, _portNameType): try: port = socket.getservbyname(port, 'tcp') except socket.error as e: raise error.ServiceNameUnknownError(string="%s (%r)" % (e, port)) self.host, self.port = host, port if abstract.isIPv6Address(host): self._addressType = address.IPv6Address self.bindAddress = bindAddress base.BaseConnector.__init__(self, factory, timeout, reactor)
def __init__(self, port, factory, backlog=50, interface='', reactor=None): """Initialize with a numeric port to listen on. """ base.BasePort.__init__(self, reactor=reactor) self.port = port self.factory = factory self.backlog = backlog if abstract.isIPv6Address(interface): self.addressFamily = socket.AF_INET6 self._addressType = address.IPv6Address self.interface = interface
def __init__(self, hostname, ctx): self._ctx = ctx if isIPAddress(hostname) or isIPv6Address(hostname): self._hostnameBytes = hostname.encode('ascii') self._sendSNI = False else: self._hostnameBytes = _idnaBytes(hostname) self._sendSNI = True ctx.set_info_callback(_tolerateErrors(self._identityVerifyingInfoCallback))
def __init__(self, hostname: bytes, verify_certs): self._verify_certs = verify_certs _decoded = hostname.decode("ascii") if isIPAddress(_decoded) or isIPv6Address(_decoded): self._is_ip_address = True else: self._is_ip_address = False self._hostnameBytes = hostname self._hostnameASCII = self._hostnameBytes.decode("ascii")
def listenTCP(self, port, factory, backlog=50, interface=''): """ Fake L{reactor.listenTCP}, that logs the call and returns an L{IListeningPort}. """ self.tcpServers.append((port, factory, backlog, interface)) if isIPv6Address(interface): address = IPv6Address('TCP', interface, port) else: address = IPv4Address('TCP', '0.0.0.0', port) return _FakePort(address)
def connectTCP(self, host, port, factory, timeout=30, bindAddress=None): """ Fake L{reactor.connectTCP}, that logs the call and returns an L{IConnector}. """ self.tcpClients.append((host, port, factory, timeout, bindAddress)) if isIPv6Address(host): conn = _FakeConnector(IPv6Address('TCP', host, port)) else: conn = _FakeConnector(IPv4Address('TCP', host, port)) factory.startedConnecting(conn) return conn
def preprocess(self, request): request.headers = request.getAllHeaders() # Twisted annoyingly different between Py2/Py3 # which requires us to handle this specially in each # case. if sys.version[0] == '2': request.hostname = request.getRequestHostname().decode('utf-8') else: request.hostname = request.getRequestHostname() request.port = request.getHost().port if (request.hostname == b'localhost' or isIPAddress(request.hostname) or isIPv6Address(request.hostname)): request.tid = 1 else: request.tid = State.tenant_hostname_id_map.get( request.hostname, None) if request.tid == 1: match = re.match(b'^/t/([0-9]+)(/.*)', request.path) else: match = re.match(b'^/t/(1)(/.*)', request.path) if match is not None: groups = match.groups() tid = int(groups[0]) if tid in State.tenant_cache: request.tid, request.path = tid, groups[1] request.client_ip = request.getClientIP() request.client_proto = b'https' if request.port in [443, 8443 ] else b'http' request.client_using_tor = request.client_ip in State.tor_exit_set or \ request.port == 8083 if isinstance(request.client_ip, binary_type): request.client_ip = request.client_ip.decode('utf-8') if 'x-tor2web' in request.headers: request.client_using_tor = False request.client_ua = request.headers.get(b'user-agent', u'') request.client_mobile = re.match(b'Mobi|Android', request.client_ua, re.IGNORECASE) is not None request.language = text_type(self.detect_language(request)) if b'multilang' in request.args: request.language = None
def connect(self, protocolFactory): """ Implement L{IStreamClientEndpoint.connect} to connect via TCP, once the hostname resolution is done. """ if isIPv6Address(self._host): d = self._resolvedHostConnect(self._host, protocolFactory) else: d = self._nameResolution(self._host) d.addCallback(lambda result: result[0][self._GAI_ADDRESS][self._GAI_ADDRESS_HOST]) d.addCallback(self._resolvedHostConnect, protocolFactory) return d
def _setRealAddress(self, addr): """ Set the real IP address for this client. Once the IP address is set, the socket is created using the correct address family. """ if abstract.isIPv6Address(addr): self.addressFamily = socket.AF_INET6 self._addressType = address.IPv6Address self.realAddress = (addr, self.addr[1]) # create the socket and wait finish init after that self.initConnection()
def applyHostCloak(self, host, ip): # Find the last segments of the hostname. index = len(host[::-1].split(".", 3)[-1]) # Cloak the first part of the host and leave the last segments alone. hostmask = "{}-{}{}".format(self.ircd.config.get("cloaking_prefix", "txircd"), sha256(self.ircd.config.get("cloaking_salt", "") + host[:index]).hexdigest()[:8], host[index:]) # This is very rare since we only leave up to 3 segments uncloaked, but make sure the end result isn't too long. if len(hostmask) > self.ircd.config.get("hostname_length", 64): if isIPv6Address(ip): return self.applyIPv6Cloak(ip) else: return self.applyIPv4Cloak(ip) else: return hostmask
def connect(self, protocolFactory): """ Implement L{IStreamClientEndpoint.connect} to connect via TCP, once the hostname resolution is done. """ if isIPv6Address(self._host): d = self._resolvedHostConnect(self._host, protocolFactory) else: d = self._nameResolution(self._host) d.addCallback(lambda result: result[0][self._GAI_ADDRESS] [self._GAI_ADDRESS_HOST]) d.addCallback(self._resolvedHostConnect, protocolFactory) return d
def connect(self, host, port): """ 'Connect' to remote server. """ if self._connectedAddr: raise RuntimeError( "already connected, reconnecting is not currently supported " "(talk to itamar if you want this)") if not isIPAddress(host) and not isIPv6Address(host): raise error.InvalidAddressError(host, "not an IPv4 or IPv6 address.") self._connectedAddr = (host, port) self.socket.connect((host, port))
def connect(self, host, port): """ 'Connect' to remote server. """ if self._connectedAddr: raise RuntimeError( "already connected, reconnecting is not currently supported " "(talk to itamar if you want this)") if not isIPAddress(host) and not isIPv6Address(host): raise error.InvalidAddressError( host, 'not an IPv4 or IPv6 address.') self._connectedAddr = (host, port) self.socket.connect((host, port))
def __init__(self, hostname): if isIPAddress(hostname) or isIPv6Address(hostname): self._hostnameBytes = hostname.encode("ascii") self._is_ip_address = True else: # twisted's ClientTLSOptions falls back to the stdlib impl here if # idna is not installed, but points out that lacks support for # IDNA2008 (http://bugs.python.org/issue17305). # # We can rely on having idna. self._hostnameBytes = idna.encode(hostname) self._is_ip_address = False self._hostnameASCII = self._hostnameBytes.decode("ascii")
def preprocess(self, request): request.cors = False request.headers = request.getAllHeaders() request.hostname = request.getRequestHostname() request.port = request.getHost().port if (not State.tenant_cache[1].wizard_done or request.hostname == b'localhost' or isIPAddress(request.hostname) or isIPv6Address(request.hostname)): request.tid = 1 else: request.tid = State.tenant_hostname_id_map.get( request.hostname, None) if request.tid == 1: match = re.match(b'^/t/([0-9]+)(/.*)', request.path) else: match = re.match(b'^/t/(1)(/.*)', request.path) if match is not None: groups = match.groups() tid = int(groups[0]) if tid in State.tenant_cache: request.tid, request.path = tid, groups[1] request.client_ip = request.getClientIP() if isinstance(request.client_ip, bytes): request.client_ip = request.client_ip.decode() # Handle IPv4 mapping on IPv6 if request.client_ip.startswith('::ffff:'): request.client_ip = request.client_ip[7:] request.client_using_tor = request.client_ip in State.tor_exit_set or \ request.port == 8083 if 'x-tor2web' in request.headers: request.client_using_tor = False request.client_ua = request.headers.get(b'user-agent', b'') request.client_mobile = re.match(b'Mobi|Android', request.client_ua, re.IGNORECASE) is not None request.language = self.detect_language(request) if b'multilang' in request.args: request.language = None
def connectionMade(self): twunnel.logger.log(3, "trace: HTTPSTunnelOutputProtocol.connectionMade") request = "CONNECT " if isIPv6Address(self.factory.address) == True: request = request + "[" + str(self.factory.address) + "]:" + str(self.factory.port) else: request = request + str(self.factory.address) + ":" + str(self.factory.port) request = request + " HTTP/1.1\r\n" if self.factory.configuration["PROXY_SERVER"]["ACCOUNT"]["NAME"] != "": request = request + "Proxy-Authorization: Basic " + base64.standard_b64encode(self.factory.configuration["PROXY_SERVER"]["ACCOUNT"]["NAME"] + ":" + self.factory.configuration["PROXY_SERVER"]["ACCOUNT"]["PASSWORD"]) + "\r\n" request = request + "\r\n" self.transport.write(request)
def preprocess(self, request): request.headers = request.getAllHeaders() # Twisted annoyingly different between Py2/Py3 # which requires us to handle this specially in each # case. if sys.version[0] == '2': request.hostname = request.getRequestHostname().decode('utf-8') else: request.hostname = request.getRequestHostname() request.hostname = request.hostname.split(b':')[0] request.port = request.getHost().port if (request.hostname == b'localhost' or isIPAddress(request.hostname) or isIPv6Address(request.hostname)): request.tid = 1 else: request.tid = State.tenant_hostname_id_map.get(request.hostname, 1) request.client_ip = request.headers.get(b'gl-forwarded-for') request.client_proto = b'https' if request.client_ip is None: request.client_ip = request.getClientIP() request.client_proto = b'http' request.client_using_tor = request.client_ip in State.tor_exit_set or \ request.port == 8083 if isinstance(request.client_ip, binary_type): request.client_ip = request.client_ip.decode('utf-8') if 'x-tor2web' in request.headers: request.client_using_tor = False request.client_ua = request.headers.get(b'user-agent', u'') request.language = text_type(self.detect_language(request)) if b'multilang' in request.args: request.language = None
def processDataState2(self): twunnel.logger.log(3, "trace: SOCKS5TunnelOutputProtocol.processDataState2") addressType = 0x03 if isIPAddress(self.factory.address) == True: addressType = 0x01 else: if isIPv6Address(self.factory.address) == True: addressType = 0x04 request = struct.pack("!BBB", 0x05, 0x01, 0x00) if addressType == 0x01: address = self.factory.address address = socket.inet_pton(socket.AF_INET, address) address, = struct.unpack("!I", address) request = request + struct.pack("!BI", 0x01, address) else: if addressType == 0x03: address = self.factory.address addressLength = len(address) request = request + struct.pack("!BB%ds" % addressLength, 0x03, addressLength, address) else: if addressType == 0x04: address = self.factory.address address = socket.inet_pton(socket.AF_INET6, address) address1, address2, address3, address4 = struct.unpack("!IIII", address) request = request + struct.pack("!BIIII", 0x04, address1, address2, address3, address4) port = self.factory.port request = request + struct.pack("!H", port) self.transport.write(request) self.dataState = 3 return True
def _query(self, *args): """ Get a new L{DNSDatagramProtocol} instance from L{_connectedProtocol}, issue a query to it using C{*args}, and arrange for it to be disconnected from its transport after the query completes. @param *args: Positional arguments to be passed to L{DNSDatagramProtocol.query}. @return: A L{Deferred} which will be called back with the result of the query. """ if isIPv6Address(args[0][0]): protocol = self._connectedProtocol(interface='::') else: protocol = self._connectedProtocol() d = protocol.query(*args) def cbQueried(result): protocol.transport.stopListening() return result d.addBoth(cbQueried) return d
def processDataState0(self): logger.debug("SOCKS5TunnelOutputProtocol.processDataState0") if len(self.data) < 2: return if ord(self.data[0]) != 0x05: self.transport.loseConnection() return addressType = 0x03 if isIPAddress(self.factory.address) == True: addressType = 0x01 else: if isIPv6Address(self.factory.address) == True: addressType = 0x04 request = struct.pack("!BBB", 0x05, 0x01, 0x00) if addressType == 0x01: address = struct.unpack("!I", socket.inet_aton(self.factory.address))[0] request = request + struct.pack("!BI", 0x01, address) else: if addressType == 0x03: address = str(self.factory.address) addressLength = len(address) request = request + struct.pack("!BB%ds" % addressLength, 0x03, addressLength, address) else: self.transport.loseConnection() return request = request + struct.pack("!H", self.factory.port) self.transport.write(request) self.data = "" self.dataState = 1
try: return self.socket.send(datagram) except socket.error, se: no = se.args[0] if no == errno.WSAEINTR: return self.write(datagram) elif no == errno.WSAEMSGSIZE: raise error.MessageLengthError("message too long") elif no in (errno.WSAECONNREFUSED, errno.WSAECONNRESET, ERROR_CONNECTION_REFUSED, ERROR_PORT_UNREACHABLE): self.protocol.connectionRefused() else: raise else: assert addr != None if (not isIPAddress(addr[0]) and not isIPv6Address(addr[0]) and addr[0] != "<broadcast>"): raise error.InvalidAddressError( addr[0], "write() only accepts IP addresses, not hostnames") if isIPAddress(addr[0]) and self.addressFamily == socket.AF_INET6: raise error.InvalidAddressError( addr[0], "IPv6 port write() called with IPv4 address") if isIPv6Address(addr[0]) and self.addressFamily == socket.AF_INET: raise error.InvalidAddressError( addr[0], "IPv4 port write() called with IPv6 address") try: return self.socket.sendto(datagram, addr) except socket.error, se: no = se.args[0] if no == errno.WSAEINTR:
def test_colon(self): """ A single C{":"} is not an IPv6 address literal. """ self.assertFalse(isIPv6Address(":"))
def test_loopback(self): """ C{"::1"} is the IPv6 loopback address literal. """ self.assertTrue(isIPv6Address("::1"))
def verify_resource_is_local(host, uri, path): return isIPAddress(host) or isIPv6Address(host) or uri.startswith(path)
def test_empty(self): """ The empty string is not an IPv6 address literal. """ self.assertFalse(isIPv6Address(""))
def process(self): request = Storage() request.headers = self.requestHeaders request.host = self.getRequestHostname() request.uri = self.uri content_length = self.getHeader(b'content-length') transfer_encoding = self.getHeader(b'transfer-encoding') staticpath = request.uri staticpath = re.sub('/$', '/index.html', staticpath) staticpath = re.sub('^(/antanistaticmap/)?', '', staticpath) staticpath = re.sub('^/', '', staticpath) resource_is_local = (config.mode != "TRANSLATION" and (request.host == config.basehost or request.host == 'www.' + config.basehost)) or \ isIPAddress(request.host) or \ isIPv6Address(request.host) or \ (config.overriderobotstxt and request.uri == '/robots.txt') or \ request.uri.startswith('/antanistaticmap/') if content_length is not None: self.bodyProducer.length = int(content_length) producer = self.bodyProducer request.headers.removeHeader(b'content-length') elif transfer_encoding is not None: producer = self.bodyProducer request.headers.removeHeader(b'transfer-encoding') else: producer = None if config.mirror is not None: if config.basehost in config.mirror: config.mirror.remove(config.basehost) if len(config.mirror) > 1: self.var['mirror'] = choice(config.mirror) elif len(config.mirror) == 1: self.var['mirror'] = config.mirror[0] # we serve contents only over https if not self.isSecure() and (config.transport != 'HTTP'): self.redirect("https://" + request.host + request.uri) self.finish() defer.returnValue(None) # 0: Request admission control stage # we try to deny some ua/crawlers regardless the request is (valid or not) / (local or not) # we deny EVERY request to known user agents reconized with pattern matching if config.blockcrawl and request.headers.getRawHeaders(b'user-agent') is not None: for ua in blocked_ua_list: if re.match(ua, request.headers.getRawHeaders(b'user-agent')[0].lower()): self.sendError(403, "error_blocked_ua.tpl") defer.returnValue(NOT_DONE_YET) # 1: Client capability assessment stage if request.headers.getRawHeaders(b'accept-encoding') is not None: if re.search('gzip', request.headers.getRawHeaders(b'accept-encoding')[0]): self.obj.client_supports_gzip = True # 2: Content delivery stage # we need to verify if the requested resource is local (/antanistaticmap/*) or remote # because some checks must be done only for remote requests; # in fact local content is always served (css, js, and png in fact are used in errors) if resource_is_local: # the requested resource is local, we deliver it directly try: if staticpath == "dev/null": content = "A" * random.randint(20, 1024) self.setHeader(b'content-type', 'text/plain') defer.returnValue(self.contentFinish(content)) elif staticpath == "stats/yesterday": self.setHeader(b'content-type', 'application/json') content = yield rpc("get_yesterday_stats") defer.returnValue(self.contentFinish(content)) elif staticpath == "notification": ################################################################# # Here we need to parse POST data in x-www-form-urlencoded format ################################################################# content_receiver = BodyReceiver(defer.Deferred()) self.bodyProducer.startProducing(content_receiver) yield self.bodyProducer.finished content = ''.join(content_receiver._data) args = {} ctype = self.requestHeaders.getRawHeaders(b'content-type') if ctype is not None: ctype = ctype[0] if self.method == b"POST" and ctype: key, pdict = parse_header(ctype) if key == b'application/x-www-form-urlencoded': args.update(parse_qs(content, 1)) ################################################################# if 'by' in args and 'url' in args and 'comment' in args: tmp = [] tmp.append("From: Tor2web Node %s.%s <%s>\n" % (config.nodename, config.basehost, config.smtpmail)) tmp.append("To: %s\n" % config.smtpmailto_notifications) tmp.append("Subject: Tor2web Node (IPv4 %s, IPv6 %s): notification for %s\n" % (config.listen_ipv4, config.listen_ipv6, args['url'][0])) tmp.append("Content-Type: text/plain; charset=ISO-8859-1\n") tmp.append("Content-Transfer-Encoding: 8bit\n\n") tmp.append("BY: %s\n" % (args['by'][0])) tmp.append("URL: %s\n" % (args['url'][0])) tmp.append("COMMENT: %s\n" % (args['comment'][0])) message = StringIO(''.join(tmp)) try: sendmail(config.smtpuser, config.smtppass, config.smtpmail, config.smtpmailto_notifications, message, config.smtpdomain, config.smtpport) except Exception: pass self.setHeader(b'content-type', 'text/plain') defer.returnValue(self.contentFinish('')) else: if type(antanistaticmap[staticpath]) == str: filename, ext = os.path.splitext(staticpath) self.setHeader(b'content-type', mimetypes.types_map[ext]) content = antanistaticmap[staticpath] defer.returnValue(self.contentFinish(content)) elif type(antanistaticmap[staticpath]) == PageTemplate: defer.returnValue(flattenString(self, antanistaticmap[staticpath]).addCallback(self.contentFinish)) except Exception: pass self.sendError(404) defer.returnValue(NOT_DONE_YET) else: self.obj.uri = request.uri if not request.host: self.sendError(406, 'error_invalid_hostname.tpl') defer.returnValue(NOT_DONE_YET) if config.mode == "TRANSLATION": self.obj.onion = config.onion else: self.obj.onion = request.host.split(".")[0] + ".onion" rpc_log("detected <onion_url>.tor2web Hostname: %s" % self.obj.onion) if not verify_onion(self.obj.onion): self.sendError(406, 'error_invalid_hostname.tpl') defer.returnValue(NOT_DONE_YET) if config.mode == "ACCESSLIST": if not hashlib.md5(self.obj.onion) in access_list: self.sendError(403, 'error_hs_completely_blocked.tpl') defer.returnValue(NOT_DONE_YET) elif config.mode == "BLACKLIST": if hashlib.md5(self.obj.onion).hexdigest() in access_list: self.sendError(403, 'error_hs_completely_blocked.tpl') defer.returnValue(NOT_DONE_YET) if hashlib.md5(self.obj.onion + self.obj.uri).hexdigest() in access_list: self.sendError(403, 'error_hs_specific_page_blocked.tpl') defer.returnValue(NOT_DONE_YET) # we need to verify if the user is using tor; # on this condition it's better to redirect on the .onion if self.getClientIP() in tor_exits_list: self.redirect("http://" + self.obj.onion + request.uri) try: self.finish() except Exception: pass defer.returnValue(None) # Avoid image hotlinking if request.uri.lower().endswith(('gif','jpg','png')): if request.headers.getRawHeaders(b'referer') is not None and \ not config.basehost in request.headers.getRawHeaders(b'referer')[0].lower(): self.sendError(403) defer.returnValue(NOT_DONE_YET) # the requested resource is remote, we act as proxy self.process_request(request) parsed = urlparse(self.obj.address) self.var['address'] = self.obj.address self.var['onion'] = self.obj.onion.replace(".onion", "") self.var['path'] = parsed[2] if parsed[3] is not None and parsed[3] != '': self.var['path'] += '?' + parsed[3] agent = Agent(reactor, sockhost=config.sockshost, sockport=config.socksport, pool=self.pool) if config.dummyproxy is None: proxy_url = 's' + self.obj.address else: proxy_url = config.dummyproxy + parsed[2] + '?' + parsed[3] self.proxy_d = agent.request(self.method, proxy_url, self.obj.headers, bodyProducer=producer) self.proxy_d.addCallback(self.cbResponse) self.proxy_d.addErrback(self.handleError) defer.returnValue(NOT_DONE_YET)
def applyCloak(self, ip): if isIPv6Address(ip): return self.applyCloak6(ip) else: return self.applyCloak4(ip)