def test_nonASCII(self):
"""
All IP addresses must be encodable as ASCII; non-ASCII should result in
a L{False} result.
"""
self.assertFalse(isIPAddress(b'\xff.notascii'))
self.assertFalse(isIPAddress(u'\u4321.notascii'))
def test_decimalDotted(self):
"""
L{isIPAddress} should return C{True} for any decimal dotted
representation of an IPv4 address.
"""
self.assertTrue(isIPAddress('0.1.2.3'))
self.assertTrue(isIPAddress('252.253.254.255'))
def test_invalidLetters(self):
"""
L{isIPAddress} should return C{False} for any non-decimal dotted
representation including letters.
"""
self.assertFalse(isIPAddress('a.2.3.4'))
self.assertFalse(isIPAddress('1.b.3.4'))
def test_invalidNegative(self):
"""
L{isIPAddress} should return C{False} for negative decimal values.
"""
self.assertFalse(isIPAddress('-1'))
self.assertFalse(isIPAddress('1.-2'))
self.assertFalse(isIPAddress('1.2.-3'))
self.assertFalse(isIPAddress('1.2.-3.4'))
def test_unicodeAndBytes(self):
"""
L{isIPAddress} evaluates ASCII-encoded bytes as well as text.
"""
self.assertFalse(isIPAddress(b'256.0.0.0'))
self.assertFalse(isIPAddress(u'256.0.0.0'))
self.assertTrue(isIPAddress(b'252.253.254.255'))
self.assertTrue(isIPAddress(u'252.253.254.255'))
def write(self, datagram, addr=None):
"""
Write a datagram.
@type datagram: L{bytes}
@param datagram: The datagram to be sent.
@type addr: L{tuple} containing L{str} as first element and L{int} as
second element, or L{None}
@param addr: A tuple of (I{stringified IPv4 or IPv6 address},
I{integer port number}); can be L{None} in connected mode.
"""
if self._connectedAddr:
assert addr in (None, self._connectedAddr)
try:
return self.socket.send(datagram)
except socket.error as se:
no = se.args[0]
if no == EINTR:
return self.write(datagram)
elif no == EMSGSIZE:
raise error.MessageLengthError("message too long")
elif no == ECONNREFUSED:
self.protocol.connectionRefused()
else:
raise
else:
assert addr != None
if (not abstract.isIPAddress(addr[0])
and not abstract.isIPv6Address(addr[0])
and addr[0] != "<broadcast>"):
raise error.InvalidAddressError(
addr[0],
"write() only accepts IP addresses, not hostnames")
if ((abstract.isIPAddress(addr[0]) or addr[0] == "<broadcast>")
and self.addressFamily == socket.AF_INET6):
raise error.InvalidAddressError(
addr[0],
"IPv6 port write() called with IPv4 or broadcast address")
if (abstract.isIPv6Address(addr[0])
and self.addressFamily == socket.AF_INET):
raise error.InvalidAddressError(
addr[0], "IPv4 port write() called with IPv6 address")
try:
return self.socket.sendto(datagram, addr)
except socket.error as se:
no = se.args[0]
if no == EINTR:
return self.write(datagram, addr)
elif no == EMSGSIZE:
raise error.MessageLengthError("message too long")
elif no == ECONNREFUSED:
# in non-connected UDP ECONNREFUSED is platform dependent, I
# think and the info is not necessarily useful. Nevertheless
# maybe we should call connectionRefused? XXX
return
else:
raise
def test_shortDecimalDotted(self):
"""
L{isIPAddress} should return C{False} for a dotted decimal
representation with fewer or more than four octets.
"""
self.assertFalse(isIPAddress('0'))
self.assertFalse(isIPAddress('0.1'))
self.assertFalse(isIPAddress('0.1.2'))
self.assertFalse(isIPAddress('0.1.2.3.4'))
def test_invalidPunctuation(self):
"""
L{isIPAddress} should return C{False} for a string containing
strange punctuation.
"""
self.assertFalse(isIPAddress(','))
self.assertFalse(isIPAddress('1,2'))
self.assertFalse(isIPAddress('1,2,3'))
self.assertFalse(isIPAddress('1.,.3,4'))
def test_invalidPositive(self):
"""
L{isIPAddress} should return C{False} for a string containing
positive decimal values greater than 255.
"""
self.assertFalse(isIPAddress('256.0.0.0'))
self.assertFalse(isIPAddress('0.256.0.0'))
self.assertFalse(isIPAddress('0.0.256.0'))
self.assertFalse(isIPAddress('0.0.0.256'))
self.assertFalse(isIPAddress('256.256.256.256'))
def write(self, datagram, addr=None):
"""
Write a datagram.
@param addr: should be a tuple (ip, port), can be None in connected
mode.
"""
if self._connectedAddr:
assert addr in (None, self._connectedAddr)
try:
return self.socket.send(datagram)
except socket.error as se:
no = se.args[0]
if no == errno.WSAEINTR:
return self.write(datagram)
elif no == errno.WSAEMSGSIZE:
raise error.MessageLengthError("message too long")
elif no in (errno.WSAECONNREFUSED, errno.WSAECONNRESET,
ERROR_CONNECTION_REFUSED, ERROR_PORT_UNREACHABLE):
self.protocol.connectionRefused()
else:
raise
else:
assert addr != None
if (not isIPAddress(addr[0]) and not isIPv6Address(addr[0])
and addr[0] != "<broadcast>"):
raise error.InvalidAddressError(
addr[0],
"write() only accepts IP addresses, not hostnames")
if isIPAddress(addr[0]) and self.addressFamily == socket.AF_INET6:
raise error.InvalidAddressError(
addr[0], "IPv6 port write() called with IPv4 address")
if isIPv6Address(addr[0]) and self.addressFamily == socket.AF_INET:
raise error.InvalidAddressError(
addr[0], "IPv4 port write() called with IPv6 address")
try:
return self.socket.sendto(datagram, addr)
except socket.error as se:
no = se.args[0]
if no == errno.WSAEINTR:
return self.write(datagram, addr)
elif no == errno.WSAEMSGSIZE:
raise error.MessageLengthError("message too long")
elif no in (errno.WSAECONNREFUSED, errno.WSAECONNRESET,
ERROR_CONNECTION_REFUSED, ERROR_PORT_UNREACHABLE):
# in non-connected UDP ECONNREFUSED is platform dependent,
# I think and the info is not necessarily useful.
# Nevertheless maybe we should call connectionRefused? XXX
return
else:
raise
def will_exit_to(self, host, port, protocol="TCP"):
"""Figure out whether this relay will allow any exit traffic to host:port.
If host is None, return True if there is any possible host that exit traffic
to that port would be allowed (and vice versa). Behavior when host AND port
are None is undefined (raises exception)"""
if not self.desc:
return False
#protocol is allowed to be DHT or TCP
if protocol == "DHT":
return self.desc.allowsDHT
if protocol != "TCP":
raise ValueError("Bad value for protocol: %s" % (protocol))
#for the vacuous case that we dont actually care
if not host and not port:
return True
#make sure that this host is an int:
intHost = None
if host and type(host) != types.IntType and isIPAddress(str(host)):
intHost = struct.unpack(">I", socket.inet_aton(host))[0]
port = int(port)
#if we're filtering based on both host and port:
if intHost and port:
#check the descriptor's exit policy:
return self.desc.will_exit_to(intHost, port)
#if we're just filtering based on host:
elif intHost:
return self.will_exit_to_host(intHost)
#if we're just filtering based on port:
elif port:
return self.will_exit_to_port(port)
#should be unreachable...
return False
def __init__(self, host, port, bindAddress, connector, reactor=None):
# BaseClient.__init__ is invoked later
self.connector = connector
self.addr = (host, port)
whenDone = self.resolveAddress
err = None
skt = None
if abstract.isIPAddress(host):
self._requiresResolution = False
elif abstract.isIPv6Address(host):
self._requiresResolution = False
self.addr = _resolveIPv6(host, port)
self.addressFamily = socket.AF_INET6
self._addressType = address.IPv6Address
else:
self._requiresResolution = True
try:
skt = self.createInternetSocket()
except socket.error as se:
err = error.ConnectBindError(se.args[0], se.args[1])
whenDone = None
if whenDone and bindAddress is not None:
try:
if abstract.isIPv6Address(bindAddress[0]):
bindinfo = _resolveIPv6(*bindAddress)
else:
bindinfo = bindAddress
skt.bind(bindinfo)
except socket.error as se:
err = error.ConnectBindError(se.args[0], se.args[1])
whenDone = None
self._finishInit(whenDone, skt, err, reactor)
def render_GET(self, request):
"""Renders the response to peers"""
args = request.args
peer_id = args.get('peer_id', [None])[0]
if peer_id is None:
return self._encode_failure('Peer ID not given')
if len(peer_id) != 20:
return self._encode_failure('Invalid peer id')
# TODO: eliminate ip option
ip_address = args.get('ip', [request.getClientIP()])[0]
if not isIPAddress(ip_address):
return self._encode_failure('Invalid IP Address')
port = args.get('port', [None])[0]
if port is None:
return self._encode_failure('Port number not given')
try:
port = int(port)
except ValueError:
return self._encode_failure('Invalid port number')
self._peer_manager.update_peer(peer_id, ip_address, port)
return self._encode_peers()
def get_value(self):
val = self.entry.get_text()
if val == "":
return val
if not isIPAddress(val):
raise Exception("Must be a valid IPv4 address.")
return val
def _send_remote_peer_request(self, infohash, callback):
#make sure we have a circuit to send it out on:
if self.circ and self.circ.is_done():
self.circ = None
if not self.circ:
self.circ = self.app.find_or_build_best_circuit(force=True, protocol="DHT")
if self.circ == None:
log_msg("Could not build circuit for DHT remote peer request", 0, "dht")
return
#generate the message: (version, infohash, peerList)
msg = ""
#header:
msg += Basic.write_byte(Node.VERSION)
#infohash:
msg += infohash
#peers:
for host, port in self.knownNodes:
#is this an IP address?
if isIPAddress(host):
msg += Basic.write_byte(0)
msg += struct.pack("!4sH", socket.inet_aton(host), port)
#otherwise, it's a URL that has to be resolved remotely
else:
msg += Basic.write_byte(1)
msg += Basic.write_lenstr(host)
msg += Basic.write_short(port)
self.circ.send_dht_request(msg, self.make_callback_wrapper(callback))
def resolveAddress(self):
host, port = self.addr
if isIPAddress(host):
return self.addr
else:
from twisted.internet import reactor
return reactor.resolve(host).addCallback(self._filterRealAddress)
def __init__(self, host, port, connect_callback=None,
disconnect_callback=None):
"""Avoid using this initializer directly; Instead, use the create()
static method, otherwise the messages won't be really delivered.
If you still need to use this directly and want to resolve the host
yourself, remember to call host_resolved() as soon as it's resolved.
@param host: The StatsD server host.
@param port: The StatsD server port.
@param connect_callback: The callback to invoke on connection.
@param disconnect_callback: The callback to invoke on disconnection.
"""
from twisted.internet import reactor
self.reactor = reactor
self.host = host
self.port = port
self.connect_callback = connect_callback
self.disconnect_callback = disconnect_callback
self.data_queue = DataQueue()
self.transport = None
self.transport_gateway = None
if abstract.isIPAddress(host):
self.host_resolved(host)
def unpickle(obj, fileName):
"""main unpickle function"""
config = configobj.ConfigObj(fileName, encoding="utf-8")
# generate the config spec:
for name in config.keys():
# dont load settings that no longer exist
if not obj.ranges.has_key(name):
continue
range = obj.ranges[name]
defaultVal = obj.defaults[name]
loadedVal = config[name]
try:
if type(range) == types.TupleType and type(range[0]) != types.StringType:
if type(range[0]) == types.FloatType:
loadedVal = float(loadedVal)
assert loadedVal <= range[1], "%s not in range %s" % (loadedVal, range)
assert loadedVal >= range[0], "%s not in range %s" % (loadedVal, range)
elif type(range[0]) == types.IntType:
loadedVal = int(loadedVal)
assert loadedVal <= range[1], "%s not in range %s" % (loadedVal, range)
assert loadedVal >= range[0], "%s not in range %s" % (loadedVal, range)
else:
raise Exception("Weird type for range: %s" % (range))
elif type(range) == types.TupleType and type(range[0]) == types.StringType:
assert loadedVal in range, "%s not in range %s" % (loadedVal, range)
elif type(defaultVal) == types.BooleanType:
if loadedVal.lower() in ("1", "true", "t", "y", "yes", "on"):
loadedVal = True
else:
loadedVal = False
# default to text entry for special types that need validation
elif type(range) == types.StringType:
if range == "folder":
assert os.path.isdir(loadedVal), "%s is not a valid directory" % (loadedVal)
elif range == "ip":
if loadedVal:
assert isIPAddress(loadedVal), "%s is not a valid ip address" % (loadedVal)
elif range in ("GB", "KBps"):
loadedVal = int(loadedVal)
elif range == "password":
pass
elif range == "scheduler":
pass
elif range == "anonymity level":
loadedVal = int(loadedVal)
assert loadedVal in (1, 2, 3), "anonymity value not in range (1,3)"
else:
regex = re.compile(range)
assert regex.match(loadedVal), "settings string (%s) does not match expression (%s)" % (
loadedVal,
range,
)
else:
raise Exception("Unknown range/value combination (%s, %s)" % (range, defaultVal))
except Exception, e:
log_msg("Bad option value (%s) for option %s" % (e, name), 0)
GUIController.get().show_msgbox("Bad option value (%s) for option %s" % (e, name))
setattr(obj, name, loadedVal)
def connect(self, host, port):
"""'Connect' to remote server."""
if self._connectedAddr:
raise RuntimeError, "already connected, reconnecting is not currently supported (talk to itamar if you want this)"
if not abstract.isIPAddress(host):
raise ValueError, "please pass only IP addresses, not domain names"
self._connectedAddr = (host, port)
self.socket.connect((host, port))
def handle_listening_connect(self, host, port):
self.state = "connecting"
if isIPAddress(host):
return defer.maybeDeferred(self._connectDone, host, port)
else:
d = self.reactor.resolve(host)
d.addCallback(self._connectDone, port)
return d
def resolve(self, name, timeout = (1, 3, 11, 45)):
"""Return a Deferred that will resolve a hostname.
"""
if not name:
return defer.succeed('0.0.0.0')
if abstract.isIPAddress(name):
return defer.succeed(name)
return self.resolver.getHostByName(name, timeout)
def resolve(self, name, timeout = (1, 3, 11, 45)):
"""Return a Deferred that will resolve a hostname.
"""
if not name:
# XXX - This is *less than* '::', and will screw up IPv6 servers
return defer.succeed('0.0.0.0')
if abstract.isIPAddress(name):
return defer.succeed(name)
return self.resolver.getHostByName(name, timeout)
def handle_disconnected_startListening(self):
self._bindSocket()
host, port = self.bindAddress
if isIPAddress(host):
return defer.maybeDeferred(self._connectSocket, host)
else:
d = self.reactor.resolve(host)
d.addCallback(self._connectSocket)
return d
class Server(object):
"""
Represents a server.
"""
def __init__(self, id=None, uri=None, sharedSecret=None, thisServer=False):
self.id = id
self.uri = uri
self.thisServer = thisServer
self.ips = set()
self.allowed_from_ips = set()
self.shared_secret = sharedSecret
self.v5 = False # Needs old style urn:uuid cu-address
def details(self):
if not hasattr(self, "ssl"):
self._parseDetails()
return (self.ssl, self.host, self.port, self.path,)
def check(self, ignoreIPLookupFailures=False):
# Check whether this matches the current server
parsed_uri = urlparse.urlparse(self.uri)
if parsed_uri.hostname == config.ServerHostName:
if parsed_uri.scheme == "http":
if config.HTTPPort:
self.thisServer = parsed_uri.port in (config.HTTPPort,) + tuple(config.BindHTTPPorts)
elif parsed_uri.scheme == "https":
if config.SSLPort:
self.thisServer = parsed_uri.port in (config.SSLPort,) + tuple(config.BindSSLPorts)
# Need to cache IP addresses
try:
ips = getIPsFromHost(parsed_uri.hostname)
except socket.gaierror, e:
msg = "Unable to lookup ip-addr for server '{}': {}".format(parsed_uri.hostname, str(e))
log.error(msg)
if ignoreIPLookupFailures:
ips = ()
else:
raise ValueError(msg)
self.ips = set(ips)
actual_ips = set()
for item in self.allowed_from_ips:
if not isIPAddress(item) and not isIPv6Address(item):
try:
ips = getIPsFromHost(item)
except socket.gaierror, e:
msg = "Unable to lookup ip-addr for allowed-from '{}': {}".format(item, str(e))
log.error(msg)
if not ignoreIPLookupFailures:
raise ValueError(msg)
else:
actual_ips.update(ips)
else:
actual_ips.add(item)
def noticed(self, user, channel, message):
# We only care about notices from the server, not from users.
# Users have a hostmask as "user", servers do not.
if '!' in user:
return
if self.factory.connregex is None:
return
match = self.factory.connregex.search(message)
if match is None:
return
d = match.groupdict()
nick = d['nick']
user = d['user']
ip = d['ip']
host = d.get("host", None)
if not isIPAddress(d['ip']) and not isIPv6Address(d['ip']):
return
hostmask = f'{nick}!{user}@{host or ip}'
scansets = set()
for mask, pattern, sets in self.factory.masks:
if pattern.match(hostmask) is not None:
scansets.update(sets)
if ip in self.ip_cache:
result = self.ip_cache.get(ip)
log.msg(f'Cache hit for {hostmask}: {result}')
else:
log.msg(f'Scanning {hostmask} on scanners {" ".join(scansets)}')
result = yield self.factory.scanner.scan(ip, scansets)
self.ip_cache.set(ip, result)
if result is not None:
scanset, result = result
formats = {
'NICK': nick,
'USER': user,
'IP': ip,
'MASK': hostmask,
'DESC': result,
'CHAN': self.factory.channel
}
formats['UREAS'] = scanset.user_reason.format(**formats)
formats['OREAS'] = scanset.oper_reason.format(**formats)
for action in self.factory.actions:
self.sendLine(action.format(**formats))
log.msg('KILL {MASK} for {OREAS}'.format(**formats))
else:
log.msg(f'GOOD {hostmask}')
def connect(self, host, port):
"""
'Connect' to remote server.
"""
if self._connectedAddr:
raise RuntimeError, "already connected, reconnecting is not currently supported (talk to itamar if you want this)"
if not abstract.isIPAddress(host):
raise ValueError, "please pass only IP addresses, not domain names"
self._connectedAddr = (host, port)
self.socket.connect((host, port))
def connect(self, host, port):
"""
'Connect' to remote server.
"""
if self._connectedAddr:
raise RuntimeError("already connected, reconnecting is not currently supported")
if not abstract.isIPAddress(host) and not abstract.isIPv6Address(host):
raise error.InvalidAddressError(host, "not an IPv4 or IPv6 address.")
self._connectedAddr = (host, port)
self.socket.connect((host, port))
def _setAddressFamily(self):
"""
Resolve address family for the socket.
"""
if abstract.isIPv6Address(self.interface):
self.addressFamily = socket.AF_INET6
elif abstract.isIPAddress(self.interface):
self.addressFamily = socket.AF_INET
elif self.interface:
raise error.InvalidAddressError(self.interface, "not an IPv4 or IPv6 address.")
def _aRecords(self, name):
"""
Return a tuple of L{dns.RRHeader} instances for all of the IPv4
addresses in the hosts file.
"""
return tuple(
dns.RRHeader(name, dns.A, dns.IN, self.ttl, dns.Record_A(addr, self.ttl))
for addr in searchFileForAll(FilePath(self.file), name)
if isIPAddress(addr)
)
def __init__(self, hostname, ctx):
self._ctx = ctx
if isIPAddress(hostname) or isIPv6Address(hostname):
self._hostnameBytes = hostname.encode('ascii')
self._sendSNI = False
else:
self._hostnameBytes = _idnaBytes(hostname)
self._sendSNI = True
ctx.set_info_callback(_tolerateErrors(self._identityVerifyingInfoCallback))
def __init__(self, hostname, ctx):
self._ctx = ctx
if isIPAddress(hostname) or isIPv6Address(hostname):
self._hostnameBytes = hostname.encode('ascii')
self._sendSNI = False
else:
self._hostnameBytes = _idnaBytes(hostname)
self._sendSNI = True
ctx.set_info_callback(_tolerateErrors(self._identityVerifyingInfoCallback))
def _aaaaRecords(self, name):
"""
Return a tuple of L{dns.RRHeader} instances for all of the IPv6
addresses in the hosts file.
"""
return tuple([
dns.RRHeader(name, dns.AAAA, dns.IN, self.ttl,
dns.Record_AAAA(addr, self.ttl))
for addr
in searchFileForAll(FilePath(self.file), name)
if not isIPAddress(addr)])
def resolve(
self, name: str,
timeout: Sequence[int] = (1, 3, 11, 45)) -> Deferred[str]:
"""
Return a Deferred that will resolve a hostname."""
if not name:
# XXX - This is *less than* '::', and will screw up IPv6 servers
return defer.succeed("0.0.0.0")
if abstract.isIPAddress(name):
return defer.succeed(name)
return self.resolver.getHostByName(name, timeout)
def is_local_ip(addr):
"""@param addr: check if this IP address is from the local network or machine
@returns: True if this is a local address, False otherwise"""
if not isIPAddress(addr):
raise Exception("That's not even an IP address! %s" % (addr))
v = [int(x) for x in addr.split(".")]
if v[0] == 10 or v[0] == 127 or v[:2] in ([192, 168], [169, 254]):
return True
if v[0] == 172 and v[1] >= 16 and v[1] <= 31:
return True
return False
def _setAddressFamily(self):
"""
Resolve address family for the socket.
"""
if abstract.isIPv6Address(self.interface):
self.addressFamily = socket.AF_INET6
elif abstract.isIPAddress(self.interface):
self.addressFamily = socket.AF_INET
elif self.interface:
raise error.InvalidAddressError(self.interface,
'not an IPv4 or IPv6 address.')
def is_local_ip(addr):
"""@param addr: check if this IP address is from the local network or machine
@returns: True if this is a local address, False otherwise"""
if not isIPAddress(addr):
raise Exception("That's not even an IP address! %s" % (addr))
v = [int(x) for x in addr.split(".")]
if v[0] == 10 or v[0] == 127 or v[:2] in ([192, 168], [169, 254]):
return True
if v[0] == 172 and v[1] >= 16 and v[1] <= 31:
return True
return False
def connect(self, host, port):
"""
'Connect' to remote server.
"""
if self._connectedAddr:
raise RuntimeError("already connected, reconnecting is not currently supported")
if not abstract.isIPAddress(host) and not abstract.isIPv6Address(host):
raise error.InvalidAddressError(
host, 'not an IPv4 or IPv6 address.')
self._connectedAddr = (host, port)
self.socket.connect((host, port))
def __init__(self, hostname: bytes, verify_certs):
self._verify_certs = verify_certs
_decoded = hostname.decode("ascii")
if isIPAddress(_decoded) or isIPv6Address(_decoded):
self._is_ip_address = True
else:
self._is_ip_address = False
self._hostnameBytes = hostname
self._hostnameASCII = self._hostnameBytes.decode("ascii")
def preprocess(self, request):
request.headers = request.getAllHeaders()
# Twisted annoyingly different between Py2/Py3
# which requires us to handle this specially in each
# case.
if sys.version[0] == '2':
request.hostname = request.getRequestHostname().decode('utf-8')
else:
request.hostname = request.getRequestHostname()
request.port = request.getHost().port
if (request.hostname == b'localhost' or isIPAddress(request.hostname)
or isIPv6Address(request.hostname)):
request.tid = 1
else:
request.tid = State.tenant_hostname_id_map.get(
request.hostname, None)
if request.tid == 1:
match = re.match(b'^/t/([0-9]+)(/.*)', request.path)
else:
match = re.match(b'^/t/(1)(/.*)', request.path)
if match is not None:
groups = match.groups()
tid = int(groups[0])
if tid in State.tenant_cache:
request.tid, request.path = tid, groups[1]
request.client_ip = request.getClientIP()
request.client_proto = b'https' if request.port in [443, 8443
] else b'http'
request.client_using_tor = request.client_ip in State.tor_exit_set or \
request.port == 8083
if isinstance(request.client_ip, binary_type):
request.client_ip = request.client_ip.decode('utf-8')
if 'x-tor2web' in request.headers:
request.client_using_tor = False
request.client_ua = request.headers.get(b'user-agent', u'')
request.client_mobile = re.match(b'Mobi|Android', request.client_ua,
re.IGNORECASE) is not None
request.language = text_type(self.detect_language(request))
if b'multilang' in request.args:
request.language = None
def connect(self, host, port):
"""
'Connect' to remote server.
"""
if self._connectedAddr:
raise RuntimeError(
"already connected, reconnecting is not currently supported "
"(talk to itamar if you want this)")
if not isIPAddress(host) and not isIPv6Address(host):
raise error.InvalidAddressError(
host, 'not an IPv4 or IPv6 address.')
self._connectedAddr = (host, port)
self.socket.connect((host, port))
def connect(self, host, port):
"""
'Connect' to remote server.
"""
if self._connectedAddr:
raise RuntimeError(
"already connected, reconnecting is not currently supported "
"(talk to itamar if you want this)")
if not isIPAddress(host) and not isIPv6Address(host):
raise error.InvalidAddressError(host,
"not an IPv4 or IPv6 address.")
self._connectedAddr = (host, port)
self.socket.connect((host, port))
def __init__(self, hostname):
if isIPAddress(hostname) or isIPv6Address(hostname):
self._hostnameBytes = hostname.encode("ascii")
self._is_ip_address = True
else:
# twisted's ClientTLSOptions falls back to the stdlib impl here if
# idna is not installed, but points out that lacks support for
# IDNA2008 (http://bugs.python.org/issue17305).
#
# We can rely on having idna.
self._hostnameBytes = idna.encode(hostname)
self._is_ip_address = False
self._hostnameASCII = self._hostnameBytes.decode("ascii")
def server_matches_acl_event(server_name, acl_event):
"""Check if the given server is allowed by the ACL event
Args:
server_name (str): name of server, without any port part
acl_event (EventBase): m.room.server_acl event
Returns:
bool: True if this server is allowed by the ACLs
"""
logger.debug("Checking %s against acl %s", server_name, acl_event.content)
# first of all, check if literal IPs are blocked, and if so, whether the
# server name is a literal IP
allow_ip_literals = acl_event.content.get("allow_ip_literals", True)
if not isinstance(allow_ip_literals, bool):
logger.warn("Ignorning non-bool allow_ip_literals flag")
allow_ip_literals = True
if not allow_ip_literals:
# check for ipv6 literals. These start with '['.
if server_name[0] == '[':
return False
# check for ipv4 literals. We can just lift the routine from twisted.
if isIPAddress(server_name):
return False
# next, check the deny list
deny = acl_event.content.get("deny", [])
if not isinstance(deny, (list, tuple)):
logger.warn("Ignorning non-list deny ACL %s", deny)
deny = []
for e in deny:
if _acl_entry_matches(server_name, e):
# logger.info("%s matched deny rule %s", server_name, e)
return False
# then the allow list.
allow = acl_event.content.get("allow", [])
if not isinstance(allow, (list, tuple)):
logger.warn("Ignorning non-list allow ACL %s", allow)
allow = []
for e in allow:
if _acl_entry_matches(server_name, e):
# logger.info("%s matched allow rule %s", server_name, e)
return True
# everything else should be rejected.
# logger.info("%s fell through", server_name)
return False
def server_matches_acl_event(server_name: str, acl_event: EventBase) -> bool:
"""Check if the given server is allowed by the ACL event
Args:
server_name: name of server, without any port part
acl_event: m.room.server_acl event
Returns:
True if this server is allowed by the ACLs
"""
logger.debug("Checking %s against acl %s", server_name, acl_event.content)
# first of all, check if literal IPs are blocked, and if so, whether the
# server name is a literal IP
allow_ip_literals = acl_event.content.get("allow_ip_literals", True)
if not isinstance(allow_ip_literals, bool):
logger.warning("Ignorning non-bool allow_ip_literals flag")
allow_ip_literals = True
if not allow_ip_literals:
# check for ipv6 literals. These start with '['.
if server_name[0] == "[":
return False
# check for ipv4 literals. We can just lift the routine from twisted.
if isIPAddress(server_name):
return False
# next, check the deny list
deny = acl_event.content.get("deny", [])
if not isinstance(deny, (list, tuple)):
logger.warning("Ignorning non-list deny ACL %s", deny)
deny = []
for e in deny:
if _acl_entry_matches(server_name, e):
# logger.info("%s matched deny rule %s", server_name, e)
return False
# then the allow list.
allow = acl_event.content.get("allow", [])
if not isinstance(allow, (list, tuple)):
logger.warning("Ignorning non-list allow ACL %s", allow)
allow = []
for e in allow:
if _acl_entry_matches(server_name, e):
# logger.info("%s matched allow rule %s", server_name, e)
return True
# everything else should be rejected.
# logger.info("%s fell through", server_name)
return False
def _get_IP_addresses(host_address):
"""
Resolves an an address/URL to a list of IPv4 addresses
"""
if isIPAddress(host_address):
returnValue([host_address])
else:
answers, auth, addit = yield DNSclient.lookupAddress(host_address)
if answers:
IP_addresses = []
for answer in answers:
IP_addresses.append(answer.payload.dottedQuad())
returnValue(IP_addresses)
else:
returnValue(None)
def _resolve_then_connect(self, c):
if abstract.isIPAddress(c.host):
self._connect(c)
return c
df = self.reactor.resolve(c.host)
if debug: print 'resolving', c.host
def set_host(ip):
if debug: print 'resolved', c.host, ip
c.host = ip
self._connect(c)
def error(f):
# too lazy to figure out how to fail properly, so just connect
self._connect(c)
df.addCallbacks(set_host, error)
return c
def preprocess(self, request):
request.cors = False
request.headers = request.getAllHeaders()
request.hostname = request.getRequestHostname()
request.port = request.getHost().port
if (not State.tenant_cache[1].wizard_done
or request.hostname == b'localhost'
or isIPAddress(request.hostname)
or isIPv6Address(request.hostname)):
request.tid = 1
else:
request.tid = State.tenant_hostname_id_map.get(
request.hostname, None)
if request.tid == 1:
match = re.match(b'^/t/([0-9]+)(/.*)', request.path)
else:
match = re.match(b'^/t/(1)(/.*)', request.path)
if match is not None:
groups = match.groups()
tid = int(groups[0])
if tid in State.tenant_cache:
request.tid, request.path = tid, groups[1]
request.client_ip = request.getClientIP()
if isinstance(request.client_ip, bytes):
request.client_ip = request.client_ip.decode()
# Handle IPv4 mapping on IPv6
if request.client_ip.startswith('::ffff:'):
request.client_ip = request.client_ip[7:]
request.client_using_tor = request.client_ip in State.tor_exit_set or \
request.port == 8083
if 'x-tor2web' in request.headers:
request.client_using_tor = False
request.client_ua = request.headers.get(b'user-agent', b'')
request.client_mobile = re.match(b'Mobi|Android', request.client_ua,
re.IGNORECASE) is not None
request.language = self.detect_language(request)
if b'multilang' in request.args:
request.language = None
def _get_IP_addresses(hostname):
"""
Resolves an an address/URL to a list of IPv4 addresses
"""
if isIPAddress(hostname):
returnValue([hostname])
else:
try:
answers, auth, addit = yield DNSclient.lookupAddress(hostname)
except Exception as exc: # Too many different DNS failures to catch...
log.error('DNS Resolution failure: %r for name: %r', exc, hostname)
returnValue([])
returnValue(
[answer.payload.dottedQuad()
for answer in answers if answer.type == dns.A])
def _collect_hosts(hosts):
"""
Resolve hostnames into (IPv4, port) tuples.
:param hosts:
A list or comma-separated string of hostnames which may also include
port numbers. All of the following are valid::
b'host'
u'host'
b'host:1234'
u'host:1234,host:2345'
b'host:1234 , host:2345 '
[u'host1', b'host2']
[b'host:1234', b'host:2345']
Hostnames must be ASCII (IDN is not supported). The default Kafka port
of 9092 is implied when no port is given.
:returns:
A list of unique (IPv4, port) tuples. For example::
[('127.0.0.1', 9092), ('127.0.0.2', 9092)]
If DNS resolution fails, an empty list is returned.
:rtype: :class:`list` of (:class:`str`, :class:`int`) instances
"""
if isinstance(hosts, bytes):
hosts = hosts.split(b',')
elif isinstance(hosts, _unicode):
hosts = hosts.split(u',')
result = set()
for host_port in hosts:
# FIXME This won't handle IPv6 addresses
res = nativeString(host_port).split(':')
host = res[0].strip()
port = int(res[1].strip()) if len(res) > 1 else DefaultKafkaPort
if isIPAddress(host):
ip_addresses = [host]
else:
ip_addresses = yield _get_IP_addresses(host)
result.update((address, port) for address in ip_addresses)
returnValue(list(result))
def processDataState2(self):
twunnel.logger.log(
3, "trace: SOCKS5TunnelOutputProtocol.processDataState2")
addressType = 0x03
if isIPAddress(self.factory.address) == True:
addressType = 0x01
else:
if isIPv6Address(self.factory.address) == True:
addressType = 0x04
request = struct.pack("!BBB", 0x05, 0x01, 0x00)
if addressType == 0x01:
address = self.factory.address
address = socket.inet_pton(socket.AF_INET, address)
address, = struct.unpack("!I", address)
request = request + struct.pack("!BI", 0x01, address)
else:
if addressType == 0x03:
address = self.factory.address
addressLength = len(address)
request = request + struct.pack("!BB%ds" % addressLength, 0x03,
addressLength, address)
else:
if addressType == 0x04:
address = self.factory.address
address = socket.inet_pton(socket.AF_INET6, address)
address1, address2, address3, address4 = struct.unpack(
"!IIII", address)
request = request + struct.pack(
"!BIIII", 0x04, address1, address2, address3, address4)
port = self.factory.port
request = request + struct.pack("!H", port)
self.transport.write(request)
self.dataState = 3
return True
def endpoint_from_hint_obj(hint, tor, reactor):
if tor:
if isinstance(hint, (DirectTCPV1Hint, TorTCPV1Hint)):
# this Tor object will throw ValueError for non-public IPv4
# addresses and any IPv6 address
try:
return tor.stream_via(hint.hostname, hint.port)
except ValueError:
return None
return None
if isinstance(hint, DirectTCPV1Hint):
# avoid DNS lookup unless necessary
if isIPAddress(hint.hostname):
return TCP4ClientEndpoint(reactor, hint.hostname, hint.port)
if isIPv6Address(hint.hostname):
return TCP6ClientEndpoint(reactor, hint.hostname, hint.port)
return HostnameEndpoint(reactor, hint.hostname, hint.port)
return None
def or_conn_status_event(self, event):
"""Called when an OR connection's status changes if listening to
ORCONNSTATUS events.
@param event: the event structure from the Tor controller
@type event: ORConnEvent"""
self.log_event(event, "OR_CONN")
#is this an IP address or a relay?
vals = event.endpoint.split(":")
if len(vals) == 2 and isIPAddress(vals[0]):
#TODO: handle these events, maybe look up the relay by IP/Port?
pass
#better be a hexId:
else:
hexId = TorUtils.get_hex_id(event.endpoint)
#do we know about that router?
r = self.torApp.get_relay(hexId)
if r:
r.on_or_event(event)
def preprocess(self, request):
request.to_be_anonymized = True
request.headers = request.getAllHeaders()
# Twisted annoyingly different between Py2/Py3
# which requires us to handle this specially in each
# case.
if sys.version[0] == '2':
request.hostname = request.getRequestHostname().decode('utf-8')
else:
request.hostname = request.getRequestHostname()
request.hostname = request.hostname.split(b':')[0]
request.port = request.getHost().port
if (request.hostname == b'localhost' or
isIPAddress(request.hostname) or
isIPv6Address(request.hostname)):
request.tid = 1
else:
request.tid = State.tenant_hostname_id_map.get(request.hostname, 1)
request.client_ip = request.headers.get(b'gl-forwarded-for')
request.client_proto = b'https'
if request.client_ip is None:
request.client_ip = request.getClientIP()
request.client_proto = b'http'
request.client_using_tor = request.client_ip in State.tor_exit_set or \
request.port == 8083
if isinstance(request.client_ip, binary_type):
request.client_ip = request.client_ip.decode('utf-8')
if 'x-tor2web' in request.headers:
request.client_using_tor = False
request.client_ua = request.headers.get(b'user-agent', u'')
request.language = text_type(self.detect_language(request))
if b'multilang' in request.args:
request.language = None
def preprocess(self, request):
request.headers = request.getAllHeaders()
request.hostname = request.getRequestHostname()
request.port = request.getHost().port
if (request.hostname == b'localhost' or isIPAddress(request.hostname)
or isIPv6Address(request.hostname)):
request.tid = 1
else:
request.tid = State.tenant_hostname_id_map.get(
request.hostname, None)
if request.tid == 1:
match = re.match(b'^/t/([0-9]+)(/.*)', request.path)
else:
match = re.match(b'^/t/(1)(/.*)', request.path)
if match is not None:
groups = match.groups()
tid = int(groups[0])
if tid in State.tenant_cache:
request.tid, request.path = tid, groups[1]
request.client_ip = request.getClientIP()
request.client_proto = b'https' if request.port in [443, 8443
] else b'http'
request.client_using_tor = request.client_ip in State.tor_exit_set or \
request.port == 8083
if isinstance(request.client_ip, bytes):
request.client_ip = request.client_ip.decode('utf-8')
if 'x-tor2web' in request.headers:
request.client_using_tor = False
request.client_ua = request.headers.get(b'user-agent', b'')
request.client_mobile = re.match(b'Mobi|Android', request.client_ua,
re.IGNORECASE) is not None
request.language = str(self.detect_language(request))
if b'multilang' in request.args:
request.language = None
def do_udp_check(host, port, send, expect, timeout=None):
"""Generic connection check function."""
if not isIPAddress(host):
try:
ip = yield reactor.resolve(host, timeout=(1, timeout))
except DNSLookupError:
raise ValueError("dns resolution failed")
else:
ip = host
deferred = Deferred()
protocol = UDPCheckProtocol(ip, port, send, expect, deferred, timeout)
reactor.listenUDP(0, protocol)
try:
yield deferred
except TimeoutError:
if ip == host:
raise ValueError("timed out")
else:
raise ValueError("timed out waiting for {}".format(ip))
def apply(self, actionType, user, param, settingUser, uid, adding, *params,
**kw):
if adding:
userHost = user.realHost
if isIPv6Address(userHost):
user.changeHost("cloak", self.applyIPv6Cloak(userHost))
elif isIPAddress(userHost):
user.changeHost("cloak", self.applyIPv4Cloak(userHost))
else:
if "." in userHost:
user.changeHost("cloak",
self.applyHostCloak(userHost, user.ip))
else:
if isIPv6Address(user.ip):
return self.applyIPv6Cloak(user.ip)
else:
return self.applyIPv4Cloak(user.ip)
else:
user.resetHost("cloak")
def join(self, deferred=None):
"""See L{apt_p2p.interfaces.IDHT}.
@param deferred: the deferred to callback when the join is complete
(optional, defaults to creating a new deferred and returning it)
"""
# Check for multiple simultaneous joins
if self.joining:
if deferred:
deferred.errback(DHTError("a join is already in progress"))
return
else:
raise DHTError, "a join is already in progress"
if deferred:
self.joining = deferred
else:
self.joining = defer.Deferred()
if self.config is None:
self.joining.errback(DHTError("configuration not loaded"))
return self.joining
# Create the new khashmir instance
if not self.khashmir:
self.khashmir = Khashmir(self.config, self.cache_dir)
self.outstandingJoins = 0
for node in self.bootstrap:
host, port = node.rsplit(':', 1)
port = int(port)
self.outstandingJoins += 1
# Translate host names into IP addresses
if isIPAddress(host):
self._join_gotIP(host, port)
else:
reactor.resolve(host).addCallbacks(self._join_gotIP,
self._join_resolveFailed,
callbackArgs=(port, ),
errbackArgs=(host, port))
return self.joining
def connectionMade(self):
twunnel.logger.log(3,
"trace: SOCKS4TunnelOutputProtocol.connectionMade")
addressType = 0x03
if isIPAddress(self.factory.address) == True:
addressType = 0x01
request = struct.pack("!BB", 0x04, 0x01)
port = self.factory.port
request = request + struct.pack("!H", port)
address = 0
if addressType == 0x01:
address = self.factory.address
address = socket.inet_pton(socket.AF_INET, address)
address, = struct.unpack("!I", address)
else:
if addressType == 0x03:
address = 1
request = request + struct.pack("!I", address)
name = self.factory.configuration["PROXY_SERVER"]["ACCOUNT"]["NAME"]
name = name + "\x00"
nameLength = len(name)
request = request + struct.pack("!%ds" % nameLength, name)
if addressType == 0x03:
address = self.factory.address
address = address + "\x00"
addressLength = len(address)
request = request + struct.pack("!%ds" % addressLength, address)
self.transport.write(request)
self.dataState = 0
def do_tcp_check(host, port, tls=False, tls_verify=True, timeout=None):
"""Generic connection check function."""
if not isIPAddress(host):
try:
ip = yield reactor.resolve(host, timeout=(1, timeout))
except DNSLookupError:
raise ValueError("dns resolution failed")
else:
ip = host
creator = ClientCreator(reactor, TCPCheckProtocol)
try:
if tls:
context = VerifyingContextFactory(tls_verify, CA_CERTS)
yield creator.connectSSL(ip, port, context, timeout=timeout)
else:
yield creator.connectTCP(ip, port, timeout=timeout)
except TimeoutError:
if ip == host:
raise ValueError("timed out")
else:
raise ValueError("timed out connecting to {}".format(ip))
def processDataState0(self):
logger.debug("SOCKS5TunnelOutputProtocol.processDataState0")
if len(self.data) < 2:
return
if ord(self.data[0]) != 0x05:
self.transport.loseConnection()
return
addressType = 0x03
if isIPAddress(self.factory.address) == True:
addressType = 0x01
else:
if isIPv6Address(self.factory.address) == True:
addressType = 0x04
request = struct.pack("!BBB", 0x05, 0x01, 0x00)
if addressType == 0x01:
address = struct.unpack("!I",
socket.inet_aton(self.factory.address))[0]
request = request + struct.pack("!BI", 0x01, address)
else:
if addressType == 0x03:
address = str(self.factory.address)
addressLength = len(address)
request = request + struct.pack("!BB%ds" % addressLength, 0x03,
addressLength, address)
else:
self.transport.loseConnection()
return
request = request + struct.pack("!H", self.factory.port)
self.transport.write(request)
self.data = ""
self.dataState = 1