def _parse_lsof_line(line):
line_comp = line.split()
if not line:
return None, None, None, None # blank line
elif len(line_comp) != 10:
raise ValueError('lines are expected to have ten fields: %s' % line)
elif line_comp[9] != '(ESTABLISHED)':
return None, None, None, None # connection isn't established
elif not line_comp[1].isdigit():
raise ValueError('expected the pid (which is the second value) to be an integer: %s' % line)
pid = int(line_comp[1])
cmd = line_comp[0]
port_map = line_comp[8]
if '->' not in port_map:
raise ValueError("'%s' is expected to be a '->' separated mapping" % port_map)
local, remote = port_map.split('->', 1)
if ':' not in local or ':' not in remote:
raise ValueError("'%s' is expected to be 'address:port' entries" % port_map)
local_port = local.split(':', 1)[1]
remote_port = remote.split(':', 1)[1]
if not connection.is_valid_port(local_port):
raise ValueError("'%s' isn't a valid port" % local_port)
elif not connection.is_valid_port(remote_port):
raise ValueError("'%s' isn't a valid port" % remote_port)
return int(local_port), int(remote_port), pid, cmd
def _parse_lsof_line(line):
line_comp = line.split()
if not line:
return None, None, None, None # blank line
elif len(line_comp) != 10:
raise ValueError('lines are expected to have ten fields: %s' % line)
elif line_comp[9] != '(ESTABLISHED)':
return None, None, None, None # connection isn't established
elif not line_comp[1].isdigit():
raise ValueError('expected the pid (which is the second value) to be an integer: %s' % line)
pid = int(line_comp[1])
cmd = line_comp[0]
port_map = line_comp[8]
if '->' not in port_map:
raise ValueError("'%s' is expected to be a '->' separated mapping" % port_map)
local, remote = port_map.split('->', 1)
if ':' not in local or ':' not in remote:
raise ValueError("'%s' is expected to be 'address:port' entries" % port_map)
local_port = local.split(':', 1)[1]
remote_port = remote.split(':', 1)[1]
if not connection.is_valid_port(local_port):
raise ValueError("'%s' isn't a valid port" % local_port)
elif not connection.is_valid_port(remote_port):
raise ValueError("'%s' isn't a valid port" % remote_port)
return int(local_port), int(remote_port), pid, cmd
def __init__(self, address: str, or_port: Union[int, str], dir_port: Union[int, str], fingerprint: str, nickname: str, orport_v6: Tuple[str, int]) -> None:
identifier = '%s (%s)' % (fingerprint, nickname) if nickname else fingerprint
if not connection.is_valid_ipv4_address(address):
raise ValueError('%s has an invalid IPv4 address: %s' % (identifier, address))
elif not connection.is_valid_port(or_port):
raise ValueError('%s has an invalid ORPort: %s' % (identifier, or_port))
elif not connection.is_valid_port(dir_port):
raise ValueError('%s has an invalid DirPort: %s' % (identifier, dir_port))
elif not tor_tools.is_valid_fingerprint(fingerprint):
raise ValueError('%s has an invalid fingerprint: %s' % (identifier, fingerprint))
elif nickname and not tor_tools.is_valid_nickname(nickname):
raise ValueError('%s has an invalid nickname: %s' % (fingerprint, nickname))
if orport_v6:
if not isinstance(orport_v6, tuple) or len(orport_v6) != 2:
raise ValueError('%s orport_v6 should be a two value tuple: %s' % (identifier, str(orport_v6)))
elif not connection.is_valid_ipv6_address(orport_v6[0]):
raise ValueError('%s has an invalid IPv6 address: %s' % (identifier, orport_v6[0]))
elif not connection.is_valid_port(orport_v6[1]):
raise ValueError('%s has an invalid IPv6 port: %s' % (identifier, orport_v6[1]))
self.address = address
self.or_port = int(or_port)
self.dir_port = int(dir_port)
self.fingerprint = fingerprint
self.nickname = nickname
self.orport_v6 = (orport_v6[0], int(orport_v6[1])) if orport_v6 else None
def from_remote(timeout = 60):
"""
Reads and parses tor's latest fallback directories `from
gitweb.torproject.org
<https://gitweb.torproject.org/tor.git/plain/src/or/fallback_dirs.inc>`_.
Note that while convenient, this reliance on GitWeb means you should alway
call with a fallback, such as...
::
try:
fallback_directories = stem.descriptor.remote.from_remote()
except IOError:
fallback_directories = stem.descriptor.remote.from_cache()
:param int timeout: seconds to wait before timing out the request
:returns: **dict** of **str** fingerprints to their
:class:`~stem.descriptor.remote.FallbackDirectory`
:raises: **IOError** if unable to retrieve the fallback directories
"""
try:
fallback_dir_page = str_tools._to_unicode(urllib.urlopen(GITWEB_FALLBACK_DIR_URL, timeout = timeout).read())
except:
exc = sys.exc_info()[1]
raise IOError("Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_DIR_URL, exc))
# Example of an entry...
#
# "5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33"
# " weight=43680",
results = {}
for line in fallback_dir_page.splitlines():
if line.startswith('"'):
addr_line_match = re.match('"([\d\.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*', line)
if addr_line_match:
address, dir_port, or_port, fingerprint = addr_line_match.groups()
if not connection.is_valid_ipv4_address(address):
raise IOError('%s has an invalid address: %s' % (fingerprint, address))
elif not connection.is_valid_port(or_port):
raise IOError('%s has an invalid or_port: %s' % (fingerprint, or_port))
elif not connection.is_valid_port(dir_port):
raise IOError('%s has an invalid dir_port: %s' % (fingerprint, dir_port))
elif not tor_tools.is_valid_fingerprint(fingerprint):
raise IOError('%s has an invalid fingerprint: %s' % (fingerprint, fingerprint))
results[fingerprint] = FallbackDirectory(
address = address,
or_port = int(or_port),
dir_port = int(dir_port),
fingerprint = fingerprint,
)
return results
def from_cache():
"""
Provides fallback directory information cached with Stem. Unlike
:func:`~stem.descriptor.remote.FallbackDirectory.from_remote` this doesn't
have any system requirements, and is faster too. Only drawback is that
these fallback directories are only as up to date as the Stem release we're
using.
:returns: **dict** of **str** fingerprints to their
:class:`~stem.descriptor.remote.FallbackDirectory`
"""
conf = stem.util.conf.Config()
conf.load(CACHE_PATH)
results = {}
for fingerprint in set([key.split('.')[0] for key in conf.keys()]):
if fingerprint in ('tor_commit', 'stem_commit'):
continue
attr = {}
for attr_name in ('address', 'or_port', 'dir_port', 'orport6_address', 'orport6_port'):
key = '%s.%s' % (fingerprint, attr_name)
attr[attr_name] = conf.get(key)
if not attr[attr_name] and not attr_name.startswith('orport6_'):
raise IOError("'%s' is missing from %s" % (key, CACHE_PATH))
if not connection.is_valid_ipv4_address(attr['address']):
raise IOError("'%s.address' was an invalid IPv4 address (%s)" % (fingerprint, attr['address']))
elif not connection.is_valid_port(attr['or_port']):
raise IOError("'%s.or_port' was an invalid port (%s)" % (fingerprint, attr['or_port']))
elif not connection.is_valid_port(attr['dir_port']):
raise IOError("'%s.dir_port' was an invalid port (%s)" % (fingerprint, attr['dir_port']))
elif attr['orport6_address'] and not connection.is_valid_ipv6_address(attr['orport6_address']):
raise IOError("'%s.orport6_address' was an invalid IPv6 address (%s)" % (fingerprint, attr['orport6_address']))
elif attr['orport6_port'] and not connection.is_valid_port(attr['orport6_port']):
raise IOError("'%s.orport6_port' was an invalid port (%s)" % (fingerprint, attr['orport6_port']))
if attr['orport6_address'] and attr['orport6_port']:
orport_v6 = (attr['orport6_address'], int(attr['orport6_port']))
else:
orport_v6 = None
results[fingerprint] = FallbackDirectory(
address = attr['address'],
or_port = int(attr['or_port']),
dir_port = int(attr['dir_port']),
fingerprint = fingerprint,
orport_v6 = orport_v6,
)
return results
def from_cache():
"""
Provides fallback directory information cached with Stem. Unlike
:func:`~stem.descriptor.remote.FallbackDirectory.from_remote` this doesn't
have any system requirements, and is faster too. Only drawback is that
these fallback directories are only as up to date as the Stem release we're
using.
:returns: **dict** of **str** fingerprints to their
:class:`~stem.descriptor.remote.FallbackDirectory`
"""
conf = stem.util.conf.Config()
conf.load(CACHE_PATH)
results = {}
for fingerprint in set([key.split('.')[0] for key in conf.keys()]):
if fingerprint in ('tor_commit', 'stem_commit'):
continue
attr = {}
for attr_name in ('address', 'or_port', 'dir_port', 'orport6_address', 'orport6_port'):
key = '%s.%s' % (fingerprint, attr_name)
attr[attr_name] = conf.get(key)
if not attr[attr_name] and not attr_name.startswith('orport6_'):
raise IOError("'%s' is missing from %s" % (key, CACHE_PATH))
if not connection.is_valid_ipv4_address(attr['address']):
raise IOError("'%s.address' was an invalid IPv4 address (%s)" % (fingerprint, attr['address']))
elif not connection.is_valid_port(attr['or_port']):
raise IOError("'%s.or_port' was an invalid port (%s)" % (fingerprint, attr['or_port']))
elif not connection.is_valid_port(attr['dir_port']):
raise IOError("'%s.dir_port' was an invalid port (%s)" % (fingerprint, attr['dir_port']))
elif attr['orport6_address'] and not connection.is_valid_ipv6_address(attr['orport6_address']):
raise IOError("'%s.orport6_address' was an invalid IPv6 address (%s)" % (fingerprint, attr['orport6_address']))
elif attr['orport6_port'] and not connection.is_valid_port(attr['orport6_port']):
raise IOError("'%s.orport6_port' was an invalid port (%s)" % (fingerprint, attr['orport6_port']))
if attr['orport6_address'] and attr['orport6_port']:
orport_v6 = (attr['orport6_address'], int(attr['orport6_port']))
else:
orport_v6 = None
results[fingerprint] = FallbackDirectory(
address = attr['address'],
or_port = int(attr['or_port']),
dir_port = int(attr['dir_port']),
fingerprint = fingerprint,
orport_v6 = orport_v6,
)
return results
def _parse(self):
self.endpoint_fingerprint = None
self.endpoint_nickname = None
self.endpoint_address = None
self.endpoint_port = None
try:
self.endpoint_fingerprint, self.endpoint_nickname = \
stem.control._parse_circ_entry(self.endpoint)
except stem.ProtocolError:
if not ':' in self.endpoint:
raise stem.ProtocolError("ORCONN endpoint is neither a relay nor 'address:port': %s" % self)
address, port = self.endpoint.split(':', 1)
if not connection.is_valid_port(port):
raise stem.ProtocolError("ORCONN's endpoint location's port is invalid: %s" % self)
self.endpoint_address = address
self.endpoint_port = int(port)
if self.circ_count is not None:
if not self.circ_count.isdigit():
raise stem.ProtocolError("ORCONN event got a non-numeric circuit count (%s): %s" % (self.circ_count, self))
self.circ_count = int(self.circ_count)
self._log_if_unrecognized('status', stem.ORStatus)
self._log_if_unrecognized('reason', stem.ORClosureReason)
def _parse(self):
self.endpoint_fingerprint = None
self.endpoint_nickname = None
self.endpoint_address = None
self.endpoint_port = None
try:
self.endpoint_fingerprint, self.endpoint_nickname = \
stem.control._parse_circ_entry(self.endpoint)
except stem.ProtocolError:
if not ':' in self.endpoint:
raise stem.ProtocolError(
"ORCONN endpoint is neither a relay nor 'address:port': %s"
% self)
address, port = self.endpoint.split(':', 1)
if not connection.is_valid_port(port):
raise stem.ProtocolError(
"ORCONN's endpoint location's port is invalid: %s" % self)
self.endpoint_address = address
self.endpoint_port = int(port)
if self.circ_count is not None:
if not self.circ_count.isdigit():
raise stem.ProtocolError(
"ORCONN event got a non-numeric circuit count (%s): %s" %
(self.circ_count, self))
self.circ_count = int(self.circ_count)
self._log_if_unrecognized('status', stem.ORStatus)
self._log_if_unrecognized('reason', stem.ORClosureReason)
def __init__(self, control_port, password):
assert is_valid_port(control_port), "Invalid port: %s" % control_port
address = '127.0.0.1'
cp = ControlPort(address, control_port)
super().__init__(cp)
self.authenticate(password=password)
def _parse(self):
if self.target is None:
raise stem.ProtocolError("STREAM event didn't have a target: %s" %
self)
else:
if not ':' in self.target:
raise stem.ProtocolError(
"Target location must be of the form 'address:port': %s" %
self)
address, port = self.target.split(':', 1)
if not connection.is_valid_port(port, allow_zero=True):
raise stem.ProtocolError(
"Target location's port is invalid: %s" % self)
self.target_address = address
self.target_port = int(port)
if self.source_addr is None:
self.source_address = None
self.source_port = None
else:
if not ':' in self.source_addr:
raise stem.ProtocolError(
"Source location must be of the form 'address:port': %s" %
self)
address, port = self.source_addr.split(':', 1)
if not connection.is_valid_port(port, allow_zero=True):
raise stem.ProtocolError(
"Source location's port is invalid: %s" % self)
self.source_address = address
self.source_port = int(port)
# spec specifies a circ_id of zero if the stream is unattached
if self.circ_id == "0":
self.circ_id = None
self._log_if_unrecognized('reason', stem.StreamClosureReason)
self._log_if_unrecognized('remote_reason', stem.StreamClosureReason)
self._log_if_unrecognized('purpose', stem.StreamPurpose)
def _parse(self):
if self.type not in ('server', 'client'):
raise stem.ProtocolError("Transport type should either be 'server' or 'client': %s" % self)
if not connection.is_valid_ipv4_address(self.address) and \
not connection.is_valid_ipv6_address(self.address):
raise stem.ProtocolError("Transport address isn't a valid IPv4 or IPv6 address: %s" % self)
if not connection.is_valid_port(self.port):
raise stem.ProtocolError('Transport port is invalid: %s' % self)
self.port = int(self.port)
def _parse(self):
if self.target is None:
raise stem.ProtocolError("STREAM event didn't have a target: %s" % self)
else:
if not ':' in self.target:
raise stem.ProtocolError("Target location must be of the form 'address:port': %s" % self)
address, port = self.target.split(':', 1)
if not connection.is_valid_port(port, allow_zero = True):
raise stem.ProtocolError("Target location's port is invalid: %s" % self)
self.target_address = address
self.target_port = int(port)
if self.source_addr is None:
self.source_address = None
self.source_port = None
else:
if not ':' in self.source_addr:
raise stem.ProtocolError("Source location must be of the form 'address:port': %s" % self)
address, port = self.source_addr.split(':', 1)
if not connection.is_valid_port(port, allow_zero = True):
raise stem.ProtocolError("Source location's port is invalid: %s" % self)
self.source_address = address
self.source_port = int(port)
# spec specifies a circ_id of zero if the stream is unattached
if self.circ_id == "0":
self.circ_id = None
self._log_if_unrecognized('reason', stem.StreamClosureReason)
self._log_if_unrecognized('remote_reason', stem.StreamClosureReason)
self._log_if_unrecognized('purpose', stem.StreamPurpose)
def _parse(self):
if self.type not in ('server', 'client'):
raise stem.ProtocolError(
"Transport type should either be 'server' or 'client': %s" %
self)
if not connection.is_valid_ipv4_address(self.address) and \
not connection.is_valid_ipv6_address(self.address):
raise stem.ProtocolError(
"Transport address isn't a valid IPv4 or IPv6 address: %s" %
self)
if not connection.is_valid_port(self.port):
raise stem.ProtocolError('Transport port is invalid: %s' % self)
self.port = int(self.port)
def from_remote(timeout=60):
"""
Reads and parses tor's latest fallback directories `from
gitweb.torproject.org
<https://gitweb.torproject.org/tor.git/plain/src/or/fallback_dirs.inc>`_.
Note that while convenient, this reliance on GitWeb means you should alway
call with a fallback, such as...
::
try:
fallback_directories = stem.descriptor.remote.from_remote()
except IOError:
fallback_directories = stem.descriptor.remote.from_cache()
:param int timeout: seconds to wait before timing out the request
:returns: **dict** of **str** fingerprints to their
:class:`~stem.descriptor.remote.FallbackDirectory`
:raises: **IOError** if unable to retrieve the fallback directories
"""
try:
fallback_dir_page = str_tools._to_unicode(
urllib.urlopen(GITWEB_FALLBACK_DIR_URL,
timeout=timeout).read())
except:
exc = sys.exc_info()[1]
raise IOError(
"Unable to download tor's fallback directories from %s: %s" %
(GITWEB_FALLBACK_DIR_URL, exc))
# Example of an entry...
#
# "5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33"
# " ipv6=[2a03:b0c0:0:1010::a4:b001]:9001"
# " weight=43680",
results, attr = {}, {}
for line in fallback_dir_page.splitlines():
if line.startswith('"'):
addr_line_match = re.match(
'"([\d\.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*', line)
ipv6_line_match = re.match('" ipv6=\[([\da-f:]+)\]:(\d+)"',
line)
if addr_line_match:
address, dir_port, or_port, fingerprint = addr_line_match.groups(
)
if not connection.is_valid_ipv4_address(address):
raise IOError('%s has an invalid IPv4 address: %s' %
(fingerprint, address))
elif not connection.is_valid_port(or_port):
raise IOError('%s has an invalid or_port: %s' %
(fingerprint, or_port))
elif not connection.is_valid_port(dir_port):
raise IOError('%s has an invalid dir_port: %s' %
(fingerprint, dir_port))
elif not tor_tools.is_valid_fingerprint(fingerprint):
raise IOError('%s has an invalid fingerprint: %s' %
(fingerprint, fingerprint))
attr = {
'address': address,
'or_port': int(or_port),
'dir_port': int(dir_port),
'fingerprint': fingerprint,
}
elif ipv6_line_match:
address, port = ipv6_line_match.groups()
if not connection.is_valid_ipv6_address(address):
raise IOError('%s has an invalid IPv6 address: %s' %
(fingerprint, address))
elif not connection.is_valid_port(port):
raise IOError(
'%s has an invalid ORPort for its IPv6 endpoint: %s'
% (fingerprint, port))
attr['orport_v6'] = (address, int(port))
elif line.startswith('" weight=') and 'fingerprint' in attr:
results[attr.get('fingerprint')] = FallbackDirectory(
address=attr.get('address'),
or_port=attr.get('or_port'),
dir_port=attr.get('dir_port'),
fingerprint=attr.get('fingerprint'),
orport_v6=attr.get('orport_v6'),
)
attr = {}
return results
def _parse_v2(fallback_dir_page):
# Example of an entry...
#
# "5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB"
# " ipv6=[2a01:4f8:162:51e2::2]:9001"
# /* nickname=rueckgrat */
# /* extrainfo=1 */
results, attr = {}, {}
for line in fallback_dir_page.splitlines():
addr_line_match = re.match(
'"([\d\.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*', line)
nickname_match = re.match('/\* nickname=(\S+) \*/', line)
has_extrainfo_match = re.match('/\* extrainfo=([0-1]) \*/', line)
ipv6_line_match = re.match('" ipv6=\[([\da-f:]+)\]:(\d+)"', line)
if addr_line_match:
address, dir_port, or_port, fingerprint = addr_line_match.groups(
)
if not connection.is_valid_ipv4_address(address):
raise IOError('%s has an invalid IPv4 address: %s' %
(fingerprint, address))
elif not connection.is_valid_port(or_port):
raise IOError('%s has an invalid or_port: %s' %
(fingerprint, or_port))
elif not connection.is_valid_port(dir_port):
raise IOError('%s has an invalid dir_port: %s' %
(fingerprint, dir_port))
elif not tor_tools.is_valid_fingerprint(fingerprint):
raise IOError('%s has an invalid fingerprint: %s' %
(fingerprint, fingerprint))
attr = {
'address': address,
'or_port': int(or_port),
'dir_port': int(dir_port),
'fingerprint': fingerprint,
}
elif ipv6_line_match:
address, port = ipv6_line_match.groups()
if not connection.is_valid_ipv6_address(address):
raise IOError('%s has an invalid IPv6 address: %s' %
(fingerprint, address))
elif not connection.is_valid_port(port):
raise IOError(
'%s has an invalid ORPort for its IPv6 endpoint: %s' %
(fingerprint, port))
attr['orport_v6'] = (address, int(port))
elif nickname_match:
nickname = nickname_match.group(1)
if not tor_tools.is_valid_nickname(nickname):
raise IOError('%s has an invalid nickname: %s' %
(fingerprint, nickname))
attr['nickname'] = nickname
elif has_extrainfo_match:
attr['has_extrainfo'] = has_extrainfo_match.group(1) == '1'
results[attr.get('fingerprint')] = FallbackDirectory(
address=attr.get('address'),
or_port=attr.get('or_port'),
dir_port=attr.get('dir_port'),
fingerprint=attr.get('fingerprint'),
nickname=attr.get('nickname'),
has_extrainfo=attr.get('has_extrainfo', False),
orport_v6=attr.get('orport_v6'),
)
attr = {}
return results
def _parse_v1(fallback_dir_page):
# Example of an entry...
#
# "5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33"
# " ipv6=[2a03:b0c0:0:1010::a4:b001]:9001"
# " weight=43680",
# TODO: this method can be removed once gitweb provides a v2 formatted document
results, attr = {}, {}
for line in fallback_dir_page.splitlines():
if line.startswith('"'):
addr_line_match = re.match(
'"([\d\.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*', line)
ipv6_line_match = re.match('" ipv6=\[([\da-f:]+)\]:(\d+)"',
line)
if addr_line_match:
address, dir_port, or_port, fingerprint = addr_line_match.groups(
)
if not connection.is_valid_ipv4_address(address):
raise IOError('%s has an invalid IPv4 address: %s' %
(fingerprint, address))
elif not connection.is_valid_port(or_port):
raise IOError('%s has an invalid or_port: %s' %
(fingerprint, or_port))
elif not connection.is_valid_port(dir_port):
raise IOError('%s has an invalid dir_port: %s' %
(fingerprint, dir_port))
elif not tor_tools.is_valid_fingerprint(fingerprint):
raise IOError('%s has an invalid fingerprint: %s' %
(fingerprint, fingerprint))
attr = {
'address': address,
'or_port': int(or_port),
'dir_port': int(dir_port),
'fingerprint': fingerprint,
}
elif ipv6_line_match:
address, port = ipv6_line_match.groups()
if not connection.is_valid_ipv6_address(address):
raise IOError('%s has an invalid IPv6 address: %s' %
(fingerprint, address))
elif not connection.is_valid_port(port):
raise IOError(
'%s has an invalid ORPort for its IPv6 endpoint: %s'
% (fingerprint, port))
attr['orport_v6'] = (address, int(port))
elif line.startswith('" weight=') and 'fingerprint' in attr:
results[attr.get('fingerprint')] = FallbackDirectory(
address=attr.get('address'),
or_port=attr.get('or_port'),
dir_port=attr.get('dir_port'),
fingerprint=attr.get('fingerprint'),
orport_v6=attr.get('orport_v6'),
)
attr = {}
return results