def __init__(self, host, port, bindAddress, connector, reactor=None):
# BaseClient.__init__ is invoked later
self.connector = connector
self.addr = (host, port)
whenDone = self.resolveAddress
err = None
skt = None
if abstract.isIPAddress(host):
self._requiresResolution = False
elif abstract.isIPv6Address(host):
self._requiresResolution = False
self.addr = _resolveIPv6(host, port)
self.addressFamily = socket.AF_INET6
self._addressType = address.IPv6Address
else:
self._requiresResolution = True
try:
skt = self.createInternetSocket()
except socket.error as se:
err = error.ConnectBindError(se.args[0], se.args[1])
whenDone = None
if whenDone and bindAddress is not None:
try:
if abstract.isIPv6Address(bindAddress[0]):
bindinfo = _resolveIPv6(*bindAddress)
else:
bindinfo = bindAddress
skt.bind(bindinfo)
except socket.error as se:
err = error.ConnectBindError(se.args[0], se.args[1])
whenDone = None
self._finishInit(whenDone, skt, err, reactor)
def write(self, datagram, addr=None):
"""
Write a datagram.
@param addr: should be a tuple (ip, port), can be None in connected
mode.
"""
if self._connectedAddr:
assert addr in (None, self._connectedAddr)
try:
return self.socket.send(datagram)
except OSError as se:
no = se.args[0]
if no == errno.WSAEINTR:
return self.write(datagram)
elif no == errno.WSAEMSGSIZE:
raise error.MessageLengthError("message too long")
elif no in (
errno.WSAECONNREFUSED,
errno.WSAECONNRESET,
ERROR_CONNECTION_REFUSED,
ERROR_PORT_UNREACHABLE,
):
self.protocol.connectionRefused()
else:
raise
else:
assert addr != None
if (not isIPAddress(addr[0]) and not isIPv6Address(addr[0])
and addr[0] != "<broadcast>"):
raise error.InvalidAddressError(
addr[0],
"write() only accepts IP addresses, not hostnames")
if isIPAddress(addr[0]) and self.addressFamily == socket.AF_INET6:
raise error.InvalidAddressError(
addr[0], "IPv6 port write() called with IPv4 address")
if isIPv6Address(addr[0]) and self.addressFamily == socket.AF_INET:
raise error.InvalidAddressError(
addr[0], "IPv4 port write() called with IPv6 address")
try:
return self.socket.sendto(datagram, addr)
except OSError as se:
no = se.args[0]
if no == errno.WSAEINTR:
return self.write(datagram, addr)
elif no == errno.WSAEMSGSIZE:
raise error.MessageLengthError("message too long")
elif no in (
errno.WSAECONNREFUSED,
errno.WSAECONNRESET,
ERROR_CONNECTION_REFUSED,
ERROR_PORT_UNREACHABLE,
):
# in non-connected UDP ECONNREFUSED is platform dependent,
# I think and the info is not necessarily useful.
# Nevertheless maybe we should call connectionRefused? XXX
return
else:
raise
def test_invalidWithScopeID(self):
"""
An otherwise invalid IPv6 address literal is still invalid with a
trailing scope identifier.
"""
self.assertFalse(isIPv6Address("%eth0"))
self.assertFalse(isIPv6Address(":%eth0"))
self.assertFalse(isIPv6Address("hello%eth0"))
def test_invalidWithScopeID(self):
"""
An otherwise invalid IPv6 address literal is still invalid with a
trailing scope identifier.
"""
self.assertFalse(isIPv6Address("%eth0"))
self.assertFalse(isIPv6Address(":%eth0"))
self.assertFalse(isIPv6Address("hello%eth0"))
def test_scopeID(self):
"""
An otherwise valid IPv6 address literal may also include a C{"%"}
followed by an arbitrary scope identifier.
"""
self.assertTrue(isIPv6Address("fe80::1%eth0"))
self.assertTrue(isIPv6Address("fe80::2%1"))
self.assertTrue(isIPv6Address("fe80::3%en2"))
def test_scopeID(self):
"""
An otherwise valid IPv6 address literal may also include a C{"%"}
followed by an arbitrary scope identifier.
"""
self.assertTrue(isIPv6Address("fe80::1%eth0"))
self.assertTrue(isIPv6Address("fe80::2%1"))
self.assertTrue(isIPv6Address("fe80::3%en2"))
def write(self, datagram, addr=None):
"""
Write a datagram.
@type datagram: L{bytes}
@param datagram: The datagram to be sent.
@type addr: L{tuple} containing L{str} as first element and L{int} as
second element, or L{None}
@param addr: A tuple of (I{stringified IPv4 or IPv6 address},
I{integer port number}); can be L{None} in connected mode.
"""
if self._connectedAddr:
assert addr in (None, self._connectedAddr)
try:
return self.socket.send(datagram)
except socket.error as se:
no = se.args[0]
if no == EINTR:
return self.write(datagram)
elif no == EMSGSIZE:
raise error.MessageLengthError("message too long")
elif no == ECONNREFUSED:
self.protocol.connectionRefused()
else:
raise
else:
assert addr != None
if (not abstract.isIPAddress(addr[0])
and not abstract.isIPv6Address(addr[0])
and addr[0] != "<broadcast>"):
raise error.InvalidAddressError(
addr[0],
"write() only accepts IP addresses, not hostnames")
if ((abstract.isIPAddress(addr[0]) or addr[0] == "<broadcast>")
and self.addressFamily == socket.AF_INET6):
raise error.InvalidAddressError(
addr[0],
"IPv6 port write() called with IPv4 or broadcast address")
if (abstract.isIPv6Address(addr[0])
and self.addressFamily == socket.AF_INET):
raise error.InvalidAddressError(
addr[0], "IPv4 port write() called with IPv6 address")
try:
return self.socket.sendto(datagram, addr)
except socket.error as se:
no = se.args[0]
if no == EINTR:
return self.write(datagram, addr)
elif no == EMSGSIZE:
raise error.MessageLengthError("message too long")
elif no == ECONNREFUSED:
# in non-connected UDP ECONNREFUSED is platform dependent, I
# think and the info is not necessarily useful. Nevertheless
# maybe we should call connectionRefused? XXX
return
else:
raise
def write(self, datagram, addr=None):
"""
Write a datagram.
@type datagram: C{str}
@param datagram: The datagram to be sent.
@type addr: C{tuple} containing C{str} as first element and C{int} as
second element, or L{None}
@param addr: A tuple of (I{stringified IPv4 or IPv6 address},
I{integer port number}); can be L{None} in connected mode.
"""
if self._connectedAddr:
assert addr in (None, self._connectedAddr)
try:
return self.socket.send(datagram)
except socket.error as se:
no = se.args[0]
if no == EINTR:
return self.write(datagram)
elif no == EMSGSIZE:
raise error.MessageLengthError("message too long")
elif no == ECONNREFUSED:
self.protocol.connectionRefused()
else:
raise
else:
assert addr != None
if (not abstract.isIPAddress(addr[0])
and not abstract.isIPv6Address(addr[0])
and addr[0] != "<broadcast>"):
raise error.InvalidAddressError(
addr[0],
"write() only accepts IP addresses, not hostnames")
if ((abstract.isIPAddress(addr[0]) or addr[0] == "<broadcast>")
and self.addressFamily == socket.AF_INET6):
raise error.InvalidAddressError(
addr[0],
"IPv6 port write() called with IPv4 or broadcast address")
if (abstract.isIPv6Address(addr[0])
and self.addressFamily == socket.AF_INET):
raise error.InvalidAddressError(
addr[0], "IPv4 port write() called with IPv6 address")
try:
return self.socket.sendto(datagram, addr)
except socket.error as se:
no = se.args[0]
if no == EINTR:
return self.write(datagram, addr)
elif no == EMSGSIZE:
raise error.MessageLengthError("message too long")
elif no == ECONNREFUSED:
# in non-connected UDP ECONNREFUSED is platform dependent, I
# think and the info is not necessarily useful. Nevertheless
# maybe we should call connectionRefused? XXX
return
else:
raise
def test_unicodeAndBytes(self):
"""
L{isIPv6Address} evaluates ASCII-encoded bytes as well as text.
"""
self.assertTrue(isIPv6Address(b"fe80::2%1"))
self.assertTrue(isIPv6Address(u"fe80::2%1"))
self.assertFalse(isIPv6Address(u"\u4321"))
self.assertFalse(isIPv6Address(u"hello%eth0"))
self.assertFalse(isIPv6Address(b"hello%eth0"))
def write(self, datagram, addr=None):
"""
Write a datagram.
@param addr: should be a tuple (ip, port), can be None in connected
mode.
"""
if self._connectedAddr:
assert addr in (None, self._connectedAddr)
try:
return self.socket.send(datagram)
except socket.error as se:
no = se.args[0]
if no == errno.WSAEINTR:
return self.write(datagram)
elif no == errno.WSAEMSGSIZE:
raise error.MessageLengthError("message too long")
elif no in (errno.WSAECONNREFUSED, errno.WSAECONNRESET,
ERROR_CONNECTION_REFUSED, ERROR_PORT_UNREACHABLE):
self.protocol.connectionRefused()
else:
raise
else:
assert addr != None
if (not isIPAddress(addr[0]) and not isIPv6Address(addr[0])
and addr[0] != "<broadcast>"):
raise error.InvalidAddressError(
addr[0],
"write() only accepts IP addresses, not hostnames")
if isIPAddress(addr[0]) and self.addressFamily == socket.AF_INET6:
raise error.InvalidAddressError(
addr[0], "IPv6 port write() called with IPv4 address")
if isIPv6Address(addr[0]) and self.addressFamily == socket.AF_INET:
raise error.InvalidAddressError(
addr[0], "IPv4 port write() called with IPv6 address")
try:
return self.socket.sendto(datagram, addr)
except socket.error as se:
no = se.args[0]
if no == errno.WSAEINTR:
return self.write(datagram, addr)
elif no == errno.WSAEMSGSIZE:
raise error.MessageLengthError("message too long")
elif no in (errno.WSAECONNREFUSED, errno.WSAECONNRESET,
ERROR_CONNECTION_REFUSED, ERROR_PORT_UNREACHABLE):
# in non-connected UDP ECONNREFUSED is platform dependent,
# I think and the info is not necessarily useful.
# Nevertheless maybe we should call connectionRefused? XXX
return
else:
raise
def preprocess(self, request):
request.headers = request.getAllHeaders()
request.hostname = request.getRequestHostname().split(':')[0]
request.port = request.getHost().port
if (request.hostname == 'localhost' or
isIPAddress(request.hostname) or
isIPv6Address(request.hostname)):
request.tid = 1
else:
request.tid = State.tenant_hostname_id_map.get(request.hostname, 1)
request.client_ip = request.headers.get('gl-forwarded-for')
request.client_proto = 'https'
if request.client_ip is None:
request.client_ip = request.getClientIP()
request.client_proto = 'http'
request.client_using_tor = request.client_ip in State.tor_exit_set or \
request.port == 8083
if 'x-tor2web' in request.headers:
request.client_using_tor = False
request.language = unicode(self.detect_language(request))
if 'multilang' in request.args:
request.language = None
def is_valid_matrix_server_name(string: str) -> bool:
"""Validate that the given string is a valid Matrix server name.
A string is a valid Matrix server name if it is one of the following, plus
an optional port:
a. IPv4 address
b. IPv6 literal (`[IPV6_ADDRESS]`)
c. A valid hostname
:param string: The string to validate
:return: Whether the input is a valid Matrix server name
"""
try:
host, port = parse_server_name(string)
except ValueError:
return False
valid_ipv4_addr = isIPAddress(host)
valid_ipv6_literal = (host[0] == "[" and host[-1] == "]"
and isIPv6Address(host[1:-1]))
return valid_ipv4_addr or valid_ipv6_literal or is_valid_hostname(host)
def resolveAddress(self):
if abstract.isIPAddress(self.addr[0]) or abstract.isIPv6Address(
self.addr[0]):
self._setRealAddress(self.addr[0])
else:
d = self.reactor.resolve(self.addr[0])
d.addCallbacks(self._setRealAddress, self.failIfNotConnected)
def noticed(self, user, channel, message):
# We only care about notices from the server, not from users.
# Users have a hostmask as "user", servers do not.
if '!' in user:
return
if self.factory.connregex is None:
return
match = self.factory.connregex.search(message)
if match is None:
return
d = match.groupdict()
nick = d['nick']
user = d['user']
ip = d['ip']
host = d.get("host", None)
if not isIPAddress(d['ip']) and not isIPv6Address(d['ip']):
return
hostmask = f'{nick}!{user}@{host or ip}'
scansets = set()
for mask, pattern, sets in self.factory.masks:
if pattern.match(hostmask) is not None:
scansets.update(sets)
if ip in self.immune_cache:
log.msg(f'Immunity given to {hostmask} (for IP {ip})')
result = None
elif ip in self.ip_cache:
result = self.ip_cache.get(ip)
log.msg(f'Cache hit for {hostmask}: {result}')
else:
log.msg(f'Scanning {hostmask} on scanners {" ".join(scansets)}')
result = yield self.factory.scanner.scan(ip, scansets)
self.ip_cache.set(ip, result, self.factory.cache_time)
if result is not None:
scanset, result = result
formats = {
'NICK': nick,
'USER': user,
'IP': ip,
'MASK': hostmask,
'DESC': result,
'CHAN': self.factory.channel
}
formats['UREAS'] = scanset.user_reason.format(**formats)
formats['OREAS'] = scanset.oper_reason.format(**formats)
for action in self.factory.actions:
self.sendLine(action.format(**formats))
log.msg('KILL {MASK} for {OREAS}'.format(**formats))
else:
log.msg(f'GOOD {hostmask}')
def apply(self, actionType, user, param, settingUser, uid, adding, *params, **kw):
if adding:
userHost = user.realHost
if isIPv6Address(userHost):
user.changeHost("cloak", self.applyIPv6Cloak(userHost))
elif isIPAddress(userHost):
user.changeHost("cloak", self.applyIPv4Cloak(userHost))
else:
if "." in userHost:
user.changeHost("cloak", self.applyHostCloak(userHost, user.ip))
else:
if isIPv6Address(user.ip):
return self.applyIPv6Cloak(user.ip)
else:
return self.applyIPv4Cloak(user.ip)
else:
user.resetHost("cloak")
class Server(object):
"""
Represents a server.
"""
def __init__(self, id=None, uri=None, sharedSecret=None, thisServer=False):
self.id = id
self.uri = uri
self.thisServer = thisServer
self.ips = set()
self.allowed_from_ips = set()
self.shared_secret = sharedSecret
self.v5 = False # Needs old style urn:uuid cu-address
def details(self):
if not hasattr(self, "ssl"):
self._parseDetails()
return (self.ssl, self.host, self.port, self.path,)
def check(self, ignoreIPLookupFailures=False):
# Check whether this matches the current server
parsed_uri = urlparse.urlparse(self.uri)
if parsed_uri.hostname == config.ServerHostName:
if parsed_uri.scheme == "http":
if config.HTTPPort:
self.thisServer = parsed_uri.port in (config.HTTPPort,) + tuple(config.BindHTTPPorts)
elif parsed_uri.scheme == "https":
if config.SSLPort:
self.thisServer = parsed_uri.port in (config.SSLPort,) + tuple(config.BindSSLPorts)
# Need to cache IP addresses
try:
ips = getIPsFromHost(parsed_uri.hostname)
except socket.gaierror, e:
msg = "Unable to lookup ip-addr for server '{}': {}".format(parsed_uri.hostname, str(e))
log.error(msg)
if ignoreIPLookupFailures:
ips = ()
else:
raise ValueError(msg)
self.ips = set(ips)
actual_ips = set()
for item in self.allowed_from_ips:
if not isIPAddress(item) and not isIPv6Address(item):
try:
ips = getIPsFromHost(item)
except socket.gaierror, e:
msg = "Unable to lookup ip-addr for allowed-from '{}': {}".format(item, str(e))
log.error(msg)
if not ignoreIPLookupFailures:
raise ValueError(msg)
else:
actual_ips.update(ips)
else:
actual_ips.add(item)
def __init__(self, skt, addr, factory, reactor, interface=''):
self.skt = skt
self.addr = addr
self.factory = factory
self.reactor = reactor
self._realPortNumber = skt.getsockname()[1]
self._addressType = address.IPv4Address
if abstract.isIPv6Address(interface):
self._addressType = address.IPv6Address
def __init__(self, port, factory, backlog=50, interface="", reactor=None):
self.port = port
self.factory = factory
self.backlog = backlog
self.interface = interface
self.reactor = reactor
if isIPv6Address(interface):
self.addressFamily = socket.AF_INET6
self._addressType = address.IPv6Address
def __init__(self, port, factory, backlog=50, interface='', reactor=None):
self.port = port
self.factory = factory
self.backlog = backlog
self.interface = interface
self.reactor = reactor
if isIPv6Address(interface):
self.addressFamily = socket.AF_INET6
self._addressType = address.IPv6Address
def _setAddressFamily(self):
"""
Resolve address family for the socket.
"""
if abstract.isIPv6Address(self.interface):
self.addressFamily = socket.AF_INET6
elif abstract.isIPAddress(self.interface):
self.addressFamily = socket.AF_INET
elif self.interface:
raise error.InvalidAddressError(self.interface, "not an IPv4 or IPv6 address.")
def apply(self, actionType, user, param, settingUser, uid, adding, *params,
**kw):
if adding:
userHost = user.realHost
if isIPv6Address(userHost):
user.changeHost("cloak", self.applyIPv6Cloak(userHost))
elif isIPAddress(userHost):
user.changeHost("cloak", self.applyIPv4Cloak(userHost))
else:
if "." in userHost:
user.changeHost("cloak",
self.applyHostCloak(userHost, user.ip))
else:
if isIPv6Address(user.ip):
return self.applyIPv6Cloak(user.ip)
else:
return self.applyIPv4Cloak(user.ip)
else:
user.resetHost("cloak")
def connect(self, host, port):
"""
'Connect' to remote server.
"""
if self._connectedAddr:
raise RuntimeError("already connected, reconnecting is not currently supported")
if not abstract.isIPAddress(host) and not abstract.isIPv6Address(host):
raise error.InvalidAddressError(host, "not an IPv4 or IPv6 address.")
self._connectedAddr = (host, port)
self.socket.connect((host, port))
def _aaaaRecords(self, name):
"""
Return a tuple of L{dns.RRHeader} instances for all of the IPv6
addresses in the hosts file.
"""
return tuple(
dns.RRHeader(name, dns.AAAA, dns.IN, self.ttl,
dns.Record_AAAA(addr, self.ttl))
for addr in searchFileForAll(FilePath(self.file), name)
if isIPv6Address(addr))
def connect(self, host, port):
"""
'Connect' to remote server.
"""
if self._connectedAddr:
raise RuntimeError("already connected, reconnecting is not currently supported")
if not abstract.isIPAddress(host) and not abstract.isIPv6Address(host):
raise error.InvalidAddressError(
host, 'not an IPv4 or IPv6 address.')
self._connectedAddr = (host, port)
self.socket.connect((host, port))
def _setAddressFamily(self):
"""
Resolve address family for the socket.
"""
if abstract.isIPv6Address(self.interface):
self.addressFamily = socket.AF_INET6
elif abstract.isIPAddress(self.interface):
self.addressFamily = socket.AF_INET
elif self.interface:
raise error.InvalidAddressError(self.interface,
'not an IPv4 or IPv6 address.')
def open_socket_listen(ip, port):
if isIPv6Address(ip):
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
else:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.setblocking(False)
s.bind((ip, port))
s.listen(1024)
return s
def listenTCP(self, port, factory, backlog=50, interface=''):
"""
Fake L{IReactorTCP.listenTCP}, that logs the call and
returns an L{IListeningPort}.
"""
self.tcpServers.append((port, factory, backlog, interface))
if isIPv6Address(interface):
address = IPv6Address('TCP', interface, port)
else:
address = IPv4Address('TCP', '0.0.0.0', port)
return _FakePort(address)
def __init__(self, host, port, factory, timeout, bindAddress, reactor=None):
if isinstance(port, _portNameType):
try:
port = socket.getservbyname(port, 'tcp')
except socket.error as e:
raise error.ServiceNameUnknownError(string="%s (%r)" % (e, port))
self.host, self.port = host, port
if abstract.isIPv6Address(host):
self._addressType = address.IPv6Address
self.bindAddress = bindAddress
base.BaseConnector.__init__(self, factory, timeout, reactor)
def __init__(self, port, factory, backlog=50, interface='', reactor=None):
"""Initialize with a numeric port to listen on.
"""
base.BasePort.__init__(self, reactor=reactor)
self.port = port
self.factory = factory
self.backlog = backlog
if abstract.isIPv6Address(interface):
self.addressFamily = socket.AF_INET6
self._addressType = address.IPv6Address
self.interface = interface
def __init__(self, hostname, ctx):
self._ctx = ctx
if isIPAddress(hostname) or isIPv6Address(hostname):
self._hostnameBytes = hostname.encode('ascii')
self._sendSNI = False
else:
self._hostnameBytes = _idnaBytes(hostname)
self._sendSNI = True
ctx.set_info_callback(_tolerateErrors(self._identityVerifyingInfoCallback))
def __init__(self, hostname: bytes, verify_certs):
self._verify_certs = verify_certs
_decoded = hostname.decode("ascii")
if isIPAddress(_decoded) or isIPv6Address(_decoded):
self._is_ip_address = True
else:
self._is_ip_address = False
self._hostnameBytes = hostname
self._hostnameASCII = self._hostnameBytes.decode("ascii")
def listenTCP(self, port, factory, backlog=50, interface=''):
"""
Fake L{reactor.listenTCP}, that logs the call and returns an
L{IListeningPort}.
"""
self.tcpServers.append((port, factory, backlog, interface))
if isIPv6Address(interface):
address = IPv6Address('TCP', interface, port)
else:
address = IPv4Address('TCP', '0.0.0.0', port)
return _FakePort(address)
def __init__(self, hostname, ctx):
self._ctx = ctx
if isIPAddress(hostname) or isIPv6Address(hostname):
self._hostnameBytes = hostname.encode('ascii')
self._sendSNI = False
else:
self._hostnameBytes = _idnaBytes(hostname)
self._sendSNI = True
ctx.set_info_callback(_tolerateErrors(self._identityVerifyingInfoCallback))
def connectTCP(self, host, port, factory, timeout=30, bindAddress=None):
"""
Fake L{reactor.connectTCP}, that logs the call and returns an
L{IConnector}.
"""
self.tcpClients.append((host, port, factory, timeout, bindAddress))
if isIPv6Address(host):
conn = _FakeConnector(IPv6Address('TCP', host, port))
else:
conn = _FakeConnector(IPv4Address('TCP', host, port))
factory.startedConnecting(conn)
return conn
def preprocess(self, request):
request.headers = request.getAllHeaders()
# Twisted annoyingly different between Py2/Py3
# which requires us to handle this specially in each
# case.
if sys.version[0] == '2':
request.hostname = request.getRequestHostname().decode('utf-8')
else:
request.hostname = request.getRequestHostname()
request.port = request.getHost().port
if (request.hostname == b'localhost' or isIPAddress(request.hostname)
or isIPv6Address(request.hostname)):
request.tid = 1
else:
request.tid = State.tenant_hostname_id_map.get(
request.hostname, None)
if request.tid == 1:
match = re.match(b'^/t/([0-9]+)(/.*)', request.path)
else:
match = re.match(b'^/t/(1)(/.*)', request.path)
if match is not None:
groups = match.groups()
tid = int(groups[0])
if tid in State.tenant_cache:
request.tid, request.path = tid, groups[1]
request.client_ip = request.getClientIP()
request.client_proto = b'https' if request.port in [443, 8443
] else b'http'
request.client_using_tor = request.client_ip in State.tor_exit_set or \
request.port == 8083
if isinstance(request.client_ip, binary_type):
request.client_ip = request.client_ip.decode('utf-8')
if 'x-tor2web' in request.headers:
request.client_using_tor = False
request.client_ua = request.headers.get(b'user-agent', u'')
request.client_mobile = re.match(b'Mobi|Android', request.client_ua,
re.IGNORECASE) is not None
request.language = text_type(self.detect_language(request))
if b'multilang' in request.args:
request.language = None
def connectTCP(self, host, port, factory, timeout=30, bindAddress=None):
"""
Fake L{reactor.connectTCP}, that logs the call and returns an
L{IConnector}.
"""
self.tcpClients.append((host, port, factory, timeout, bindAddress))
if isIPv6Address(host):
conn = _FakeConnector(IPv6Address('TCP', host, port))
else:
conn = _FakeConnector(IPv4Address('TCP', host, port))
factory.startedConnecting(conn)
return conn
def connect(self, protocolFactory):
"""
Implement L{IStreamClientEndpoint.connect} to connect via TCP,
once the hostname resolution is done.
"""
if isIPv6Address(self._host):
d = self._resolvedHostConnect(self._host, protocolFactory)
else:
d = self._nameResolution(self._host)
d.addCallback(lambda result: result[0][self._GAI_ADDRESS][self._GAI_ADDRESS_HOST])
d.addCallback(self._resolvedHostConnect, protocolFactory)
return d
def _setRealAddress(self, addr):
"""
Set the real IP address for this client.
Once the IP address is set, the socket is created using the correct
address family.
"""
if abstract.isIPv6Address(addr):
self.addressFamily = socket.AF_INET6
self._addressType = address.IPv6Address
self.realAddress = (addr, self.addr[1])
# create the socket and wait finish init after that
self.initConnection()
def applyHostCloak(self, host, ip):
# Find the last segments of the hostname.
index = len(host[::-1].split(".", 3)[-1])
# Cloak the first part of the host and leave the last segments alone.
hostmask = "{}-{}{}".format(self.ircd.config.get("cloaking_prefix", "txircd"), sha256(self.ircd.config.get("cloaking_salt", "") + host[:index]).hexdigest()[:8], host[index:])
# This is very rare since we only leave up to 3 segments uncloaked, but make sure the end result isn't too long.
if len(hostmask) > self.ircd.config.get("hostname_length", 64):
if isIPv6Address(ip):
return self.applyIPv6Cloak(ip)
else:
return self.applyIPv4Cloak(ip)
else:
return hostmask
def _setRealAddress(self, addr):
"""
Set the real IP address for this client.
Once the IP address is set, the socket is created using the correct
address family.
"""
if abstract.isIPv6Address(addr):
self.addressFamily = socket.AF_INET6
self._addressType = address.IPv6Address
self.realAddress = (addr, self.addr[1])
# create the socket and wait finish init after that
self.initConnection()
def connect(self, protocolFactory):
"""
Implement L{IStreamClientEndpoint.connect} to connect via TCP,
once the hostname resolution is done.
"""
if isIPv6Address(self._host):
d = self._resolvedHostConnect(self._host, protocolFactory)
else:
d = self._nameResolution(self._host)
d.addCallback(lambda result: result[0][self._GAI_ADDRESS]
[self._GAI_ADDRESS_HOST])
d.addCallback(self._resolvedHostConnect, protocolFactory)
return d
def connect(self, host, port):
"""
'Connect' to remote server.
"""
if self._connectedAddr:
raise RuntimeError(
"already connected, reconnecting is not currently supported "
"(talk to itamar if you want this)")
if not isIPAddress(host) and not isIPv6Address(host):
raise error.InvalidAddressError(host,
"not an IPv4 or IPv6 address.")
self._connectedAddr = (host, port)
self.socket.connect((host, port))
def connect(self, host, port):
"""
'Connect' to remote server.
"""
if self._connectedAddr:
raise RuntimeError(
"already connected, reconnecting is not currently supported "
"(talk to itamar if you want this)")
if not isIPAddress(host) and not isIPv6Address(host):
raise error.InvalidAddressError(
host, 'not an IPv4 or IPv6 address.')
self._connectedAddr = (host, port)
self.socket.connect((host, port))
def __init__(self, hostname):
if isIPAddress(hostname) or isIPv6Address(hostname):
self._hostnameBytes = hostname.encode("ascii")
self._is_ip_address = True
else:
# twisted's ClientTLSOptions falls back to the stdlib impl here if
# idna is not installed, but points out that lacks support for
# IDNA2008 (http://bugs.python.org/issue17305).
#
# We can rely on having idna.
self._hostnameBytes = idna.encode(hostname)
self._is_ip_address = False
self._hostnameASCII = self._hostnameBytes.decode("ascii")
def preprocess(self, request):
request.cors = False
request.headers = request.getAllHeaders()
request.hostname = request.getRequestHostname()
request.port = request.getHost().port
if (not State.tenant_cache[1].wizard_done
or request.hostname == b'localhost'
or isIPAddress(request.hostname)
or isIPv6Address(request.hostname)):
request.tid = 1
else:
request.tid = State.tenant_hostname_id_map.get(
request.hostname, None)
if request.tid == 1:
match = re.match(b'^/t/([0-9]+)(/.*)', request.path)
else:
match = re.match(b'^/t/(1)(/.*)', request.path)
if match is not None:
groups = match.groups()
tid = int(groups[0])
if tid in State.tenant_cache:
request.tid, request.path = tid, groups[1]
request.client_ip = request.getClientIP()
if isinstance(request.client_ip, bytes):
request.client_ip = request.client_ip.decode()
# Handle IPv4 mapping on IPv6
if request.client_ip.startswith('::ffff:'):
request.client_ip = request.client_ip[7:]
request.client_using_tor = request.client_ip in State.tor_exit_set or \
request.port == 8083
if 'x-tor2web' in request.headers:
request.client_using_tor = False
request.client_ua = request.headers.get(b'user-agent', b'')
request.client_mobile = re.match(b'Mobi|Android', request.client_ua,
re.IGNORECASE) is not None
request.language = self.detect_language(request)
if b'multilang' in request.args:
request.language = None
def connectionMade(self):
twunnel.logger.log(3, "trace: HTTPSTunnelOutputProtocol.connectionMade")
request = "CONNECT "
if isIPv6Address(self.factory.address) == True:
request = request + "[" + str(self.factory.address) + "]:" + str(self.factory.port)
else:
request = request + str(self.factory.address) + ":" + str(self.factory.port)
request = request + " HTTP/1.1\r\n"
if self.factory.configuration["PROXY_SERVER"]["ACCOUNT"]["NAME"] != "":
request = request + "Proxy-Authorization: Basic " + base64.standard_b64encode(self.factory.configuration["PROXY_SERVER"]["ACCOUNT"]["NAME"] + ":" + self.factory.configuration["PROXY_SERVER"]["ACCOUNT"]["PASSWORD"]) + "\r\n"
request = request + "\r\n"
self.transport.write(request)
def preprocess(self, request):
request.headers = request.getAllHeaders()
# Twisted annoyingly different between Py2/Py3
# which requires us to handle this specially in each
# case.
if sys.version[0] == '2':
request.hostname = request.getRequestHostname().decode('utf-8')
else:
request.hostname = request.getRequestHostname()
request.hostname = request.hostname.split(b':')[0]
request.port = request.getHost().port
if (request.hostname == b'localhost' or
isIPAddress(request.hostname) or
isIPv6Address(request.hostname)):
request.tid = 1
else:
request.tid = State.tenant_hostname_id_map.get(request.hostname, 1)
request.client_ip = request.headers.get(b'gl-forwarded-for')
request.client_proto = b'https'
if request.client_ip is None:
request.client_ip = request.getClientIP()
request.client_proto = b'http'
request.client_using_tor = request.client_ip in State.tor_exit_set or \
request.port == 8083
if isinstance(request.client_ip, binary_type):
request.client_ip = request.client_ip.decode('utf-8')
if 'x-tor2web' in request.headers:
request.client_using_tor = False
request.client_ua = request.headers.get(b'user-agent', u'')
request.language = text_type(self.detect_language(request))
if b'multilang' in request.args:
request.language = None
def processDataState2(self):
twunnel.logger.log(3, "trace: SOCKS5TunnelOutputProtocol.processDataState2")
addressType = 0x03
if isIPAddress(self.factory.address) == True:
addressType = 0x01
else:
if isIPv6Address(self.factory.address) == True:
addressType = 0x04
request = struct.pack("!BBB", 0x05, 0x01, 0x00)
if addressType == 0x01:
address = self.factory.address
address = socket.inet_pton(socket.AF_INET, address)
address, = struct.unpack("!I", address)
request = request + struct.pack("!BI", 0x01, address)
else:
if addressType == 0x03:
address = self.factory.address
addressLength = len(address)
request = request + struct.pack("!BB%ds" % addressLength, 0x03, addressLength, address)
else:
if addressType == 0x04:
address = self.factory.address
address = socket.inet_pton(socket.AF_INET6, address)
address1, address2, address3, address4 = struct.unpack("!IIII", address)
request = request + struct.pack("!BIIII", 0x04, address1, address2, address3, address4)
port = self.factory.port
request = request + struct.pack("!H", port)
self.transport.write(request)
self.dataState = 3
return True
def _query(self, *args):
"""
Get a new L{DNSDatagramProtocol} instance from L{_connectedProtocol},
issue a query to it using C{*args}, and arrange for it to be
disconnected from its transport after the query completes.
@param *args: Positional arguments to be passed to
L{DNSDatagramProtocol.query}.
@return: A L{Deferred} which will be called back with the result of the
query.
"""
if isIPv6Address(args[0][0]):
protocol = self._connectedProtocol(interface='::')
else:
protocol = self._connectedProtocol()
d = protocol.query(*args)
def cbQueried(result):
protocol.transport.stopListening()
return result
d.addBoth(cbQueried)
return d
def processDataState0(self):
logger.debug("SOCKS5TunnelOutputProtocol.processDataState0")
if len(self.data) < 2:
return
if ord(self.data[0]) != 0x05:
self.transport.loseConnection()
return
addressType = 0x03
if isIPAddress(self.factory.address) == True:
addressType = 0x01
else:
if isIPv6Address(self.factory.address) == True:
addressType = 0x04
request = struct.pack("!BBB", 0x05, 0x01, 0x00)
if addressType == 0x01:
address = struct.unpack("!I", socket.inet_aton(self.factory.address))[0]
request = request + struct.pack("!BI", 0x01, address)
else:
if addressType == 0x03:
address = str(self.factory.address)
addressLength = len(address)
request = request + struct.pack("!BB%ds" % addressLength, 0x03, addressLength, address)
else:
self.transport.loseConnection()
return
request = request + struct.pack("!H", self.factory.port)
self.transport.write(request)
self.data = ""
self.dataState = 1
try:
return self.socket.send(datagram)
except socket.error, se:
no = se.args[0]
if no == errno.WSAEINTR:
return self.write(datagram)
elif no == errno.WSAEMSGSIZE:
raise error.MessageLengthError("message too long")
elif no in (errno.WSAECONNREFUSED, errno.WSAECONNRESET,
ERROR_CONNECTION_REFUSED, ERROR_PORT_UNREACHABLE):
self.protocol.connectionRefused()
else:
raise
else:
assert addr != None
if (not isIPAddress(addr[0]) and not isIPv6Address(addr[0])
and addr[0] != "<broadcast>"):
raise error.InvalidAddressError(
addr[0],
"write() only accepts IP addresses, not hostnames")
if isIPAddress(addr[0]) and self.addressFamily == socket.AF_INET6:
raise error.InvalidAddressError(
addr[0], "IPv6 port write() called with IPv4 address")
if isIPv6Address(addr[0]) and self.addressFamily == socket.AF_INET:
raise error.InvalidAddressError(
addr[0], "IPv4 port write() called with IPv6 address")
try:
return self.socket.sendto(datagram, addr)
except socket.error, se:
no = se.args[0]
if no == errno.WSAEINTR:
def test_colon(self):
"""
A single C{":"} is not an IPv6 address literal.
"""
self.assertFalse(isIPv6Address(":"))
def test_loopback(self):
"""
C{"::1"} is the IPv6 loopback address literal.
"""
self.assertTrue(isIPv6Address("::1"))
def verify_resource_is_local(host, uri, path):
return isIPAddress(host) or isIPv6Address(host) or uri.startswith(path)
def test_empty(self):
"""
The empty string is not an IPv6 address literal.
"""
self.assertFalse(isIPv6Address(""))
def process(self):
request = Storage()
request.headers = self.requestHeaders
request.host = self.getRequestHostname()
request.uri = self.uri
content_length = self.getHeader(b'content-length')
transfer_encoding = self.getHeader(b'transfer-encoding')
staticpath = request.uri
staticpath = re.sub('/$', '/index.html', staticpath)
staticpath = re.sub('^(/antanistaticmap/)?', '', staticpath)
staticpath = re.sub('^/', '', staticpath)
resource_is_local = (config.mode != "TRANSLATION" and
(request.host == config.basehost or
request.host == 'www.' + config.basehost)) or \
isIPAddress(request.host) or \
isIPv6Address(request.host) or \
(config.overriderobotstxt and request.uri == '/robots.txt') or \
request.uri.startswith('/antanistaticmap/')
if content_length is not None:
self.bodyProducer.length = int(content_length)
producer = self.bodyProducer
request.headers.removeHeader(b'content-length')
elif transfer_encoding is not None:
producer = self.bodyProducer
request.headers.removeHeader(b'transfer-encoding')
else:
producer = None
if config.mirror is not None:
if config.basehost in config.mirror:
config.mirror.remove(config.basehost)
if len(config.mirror) > 1:
self.var['mirror'] = choice(config.mirror)
elif len(config.mirror) == 1:
self.var['mirror'] = config.mirror[0]
# we serve contents only over https
if not self.isSecure() and (config.transport != 'HTTP'):
self.redirect("https://" + request.host + request.uri)
self.finish()
defer.returnValue(None)
# 0: Request admission control stage
# we try to deny some ua/crawlers regardless the request is (valid or not) / (local or not)
# we deny EVERY request to known user agents reconized with pattern matching
if config.blockcrawl and request.headers.getRawHeaders(b'user-agent') is not None:
for ua in blocked_ua_list:
if re.match(ua, request.headers.getRawHeaders(b'user-agent')[0].lower()):
self.sendError(403, "error_blocked_ua.tpl")
defer.returnValue(NOT_DONE_YET)
# 1: Client capability assessment stage
if request.headers.getRawHeaders(b'accept-encoding') is not None:
if re.search('gzip', request.headers.getRawHeaders(b'accept-encoding')[0]):
self.obj.client_supports_gzip = True
# 2: Content delivery stage
# we need to verify if the requested resource is local (/antanistaticmap/*) or remote
# because some checks must be done only for remote requests;
# in fact local content is always served (css, js, and png in fact are used in errors)
if resource_is_local:
# the requested resource is local, we deliver it directly
try:
if staticpath == "dev/null":
content = "A" * random.randint(20, 1024)
self.setHeader(b'content-type', 'text/plain')
defer.returnValue(self.contentFinish(content))
elif staticpath == "stats/yesterday":
self.setHeader(b'content-type', 'application/json')
content = yield rpc("get_yesterday_stats")
defer.returnValue(self.contentFinish(content))
elif staticpath == "notification":
#################################################################
# Here we need to parse POST data in x-www-form-urlencoded format
#################################################################
content_receiver = BodyReceiver(defer.Deferred())
self.bodyProducer.startProducing(content_receiver)
yield self.bodyProducer.finished
content = ''.join(content_receiver._data)
args = {}
ctype = self.requestHeaders.getRawHeaders(b'content-type')
if ctype is not None:
ctype = ctype[0]
if self.method == b"POST" and ctype:
key, pdict = parse_header(ctype)
if key == b'application/x-www-form-urlencoded':
args.update(parse_qs(content, 1))
#################################################################
if 'by' in args and 'url' in args and 'comment' in args:
tmp = []
tmp.append("From: Tor2web Node %s.%s <%s>\n" % (config.nodename, config.basehost, config.smtpmail))
tmp.append("To: %s\n" % config.smtpmailto_notifications)
tmp.append("Subject: Tor2web Node (IPv4 %s, IPv6 %s): notification for %s\n" % (config.listen_ipv4, config.listen_ipv6, args['url'][0]))
tmp.append("Content-Type: text/plain; charset=ISO-8859-1\n")
tmp.append("Content-Transfer-Encoding: 8bit\n\n")
tmp.append("BY: %s\n" % (args['by'][0]))
tmp.append("URL: %s\n" % (args['url'][0]))
tmp.append("COMMENT: %s\n" % (args['comment'][0]))
message = StringIO(''.join(tmp))
try:
sendmail(config.smtpuser,
config.smtppass,
config.smtpmail,
config.smtpmailto_notifications,
message,
config.smtpdomain,
config.smtpport)
except Exception:
pass
self.setHeader(b'content-type', 'text/plain')
defer.returnValue(self.contentFinish(''))
else:
if type(antanistaticmap[staticpath]) == str:
filename, ext = os.path.splitext(staticpath)
self.setHeader(b'content-type', mimetypes.types_map[ext])
content = antanistaticmap[staticpath]
defer.returnValue(self.contentFinish(content))
elif type(antanistaticmap[staticpath]) == PageTemplate:
defer.returnValue(flattenString(self, antanistaticmap[staticpath]).addCallback(self.contentFinish))
except Exception:
pass
self.sendError(404)
defer.returnValue(NOT_DONE_YET)
else:
self.obj.uri = request.uri
if not request.host:
self.sendError(406, 'error_invalid_hostname.tpl')
defer.returnValue(NOT_DONE_YET)
if config.mode == "TRANSLATION":
self.obj.onion = config.onion
else:
self.obj.onion = request.host.split(".")[0] + ".onion"
rpc_log("detected <onion_url>.tor2web Hostname: %s" % self.obj.onion)
if not verify_onion(self.obj.onion):
self.sendError(406, 'error_invalid_hostname.tpl')
defer.returnValue(NOT_DONE_YET)
if config.mode == "ACCESSLIST":
if not hashlib.md5(self.obj.onion) in access_list:
self.sendError(403, 'error_hs_completely_blocked.tpl')
defer.returnValue(NOT_DONE_YET)
elif config.mode == "BLACKLIST":
if hashlib.md5(self.obj.onion).hexdigest() in access_list:
self.sendError(403, 'error_hs_completely_blocked.tpl')
defer.returnValue(NOT_DONE_YET)
if hashlib.md5(self.obj.onion + self.obj.uri).hexdigest() in access_list:
self.sendError(403, 'error_hs_specific_page_blocked.tpl')
defer.returnValue(NOT_DONE_YET)
# we need to verify if the user is using tor;
# on this condition it's better to redirect on the .onion
if self.getClientIP() in tor_exits_list:
self.redirect("http://" + self.obj.onion + request.uri)
try:
self.finish()
except Exception:
pass
defer.returnValue(None)
# Avoid image hotlinking
if request.uri.lower().endswith(('gif','jpg','png')):
if request.headers.getRawHeaders(b'referer') is not None and \
not config.basehost in request.headers.getRawHeaders(b'referer')[0].lower():
self.sendError(403)
defer.returnValue(NOT_DONE_YET)
# the requested resource is remote, we act as proxy
self.process_request(request)
parsed = urlparse(self.obj.address)
self.var['address'] = self.obj.address
self.var['onion'] = self.obj.onion.replace(".onion", "")
self.var['path'] = parsed[2]
if parsed[3] is not None and parsed[3] != '':
self.var['path'] += '?' + parsed[3]
agent = Agent(reactor, sockhost=config.sockshost, sockport=config.socksport, pool=self.pool)
if config.dummyproxy is None:
proxy_url = 's' + self.obj.address
else:
proxy_url = config.dummyproxy + parsed[2] + '?' + parsed[3]
self.proxy_d = agent.request(self.method,
proxy_url,
self.obj.headers, bodyProducer=producer)
self.proxy_d.addCallback(self.cbResponse)
self.proxy_d.addErrback(self.handleError)
defer.returnValue(NOT_DONE_YET)
def applyCloak(self, ip):
if isIPv6Address(ip):
return self.applyCloak6(ip)
else:
return self.applyCloak4(ip)
