def test_parse_file(self): """ Try parsing a document via the _parse_file() function. """ # parse file and assert values descriptors = list(_parse_file(io.BytesIO(TEST_DESC))) self.assertEqual(3, len(descriptors)) self.assertTrue(isinstance(descriptors[0], TorDNSEL)) desc = descriptors[1] self.assertTrue(is_valid_fingerprint(desc.fingerprint)) self.assertEqual('00FF300624FECA7F40515C8D854EE925332580D6', desc.fingerprint) self.assertEqual(datetime.datetime(2013, 8, 18, 7, 2, 14), desc.published) self.assertEqual(datetime.datetime(2013, 8, 18, 9, 2, 58), desc.last_status) self.assertEqual(3, len(desc.exit_addresses)) exit = desc.exit_addresses[0] self.assertEqual('82.252.181.153', exit[0]) self.assertEqual(datetime.datetime(2013, 8, 18, 8, 3, 1), exit[1]) # block content raises value error extra = b'ExtraContent goes here\n' descriptors = _parse_file(io.BytesIO(TEST_DESC + extra), validate=True) self.assertRaises(ValueError, list, descriptors) # malformed fingerprint raises value errors extra = b'ExitNode 030B22437D99B2DB2908B747B6' self.assertRaises( ValueError, list, _parse_file(io.BytesIO(TEST_DESC + extra), validate=True)) # malformed date raises value errors self.assertRaises( ValueError, list, _parse_file(io.BytesIO(TEST_DESC + MALFORMED_ENTRY_1), validate=True)) # skip exit address if malformed date and validate is False desc = next(_parse_file(io.BytesIO(MALFORMED_ENTRY_2), validate=False)) self.assertTrue(is_valid_fingerprint(desc.fingerprint)) self.assertEqual('030B22437D99B2DB2908B747B6962EAD13AB4038', desc.fingerprint) self.assertEqual(0, len(desc.exit_addresses)) self.assertEqual('@type tordnsel 1.0', str(desc.type_annotation()))
def __init__(self, address: str, or_port: Union[int, str], dir_port: Union[int, str], fingerprint: str, nickname: str, orport_v6: Tuple[str, int]) -> None: identifier = '%s (%s)' % (fingerprint, nickname) if nickname else fingerprint if not connection.is_valid_ipv4_address(address): raise ValueError('%s has an invalid IPv4 address: %s' % (identifier, address)) elif not connection.is_valid_port(or_port): raise ValueError('%s has an invalid ORPort: %s' % (identifier, or_port)) elif not connection.is_valid_port(dir_port): raise ValueError('%s has an invalid DirPort: %s' % (identifier, dir_port)) elif not tor_tools.is_valid_fingerprint(fingerprint): raise ValueError('%s has an invalid fingerprint: %s' % (identifier, fingerprint)) elif nickname and not tor_tools.is_valid_nickname(nickname): raise ValueError('%s has an invalid nickname: %s' % (fingerprint, nickname)) if orport_v6: if not isinstance(orport_v6, tuple) or len(orport_v6) != 2: raise ValueError('%s orport_v6 should be a two value tuple: %s' % (identifier, str(orport_v6))) elif not connection.is_valid_ipv6_address(orport_v6[0]): raise ValueError('%s has an invalid IPv6 address: %s' % (identifier, orport_v6[0])) elif not connection.is_valid_port(orport_v6[1]): raise ValueError('%s has an invalid IPv6 port: %s' % (identifier, orport_v6[1])) self.address = address self.or_port = int(or_port) self.dir_port = int(dir_port) self.fingerprint = fingerprint self.nickname = nickname self.orport_v6 = (orport_v6[0], int(orport_v6[1])) if orport_v6 else None
def from_remote(timeout = 60): """ Reads and parses tor's latest fallback directories `from gitweb.torproject.org <https://gitweb.torproject.org/tor.git/plain/src/or/fallback_dirs.inc>`_. Note that while convenient, this reliance on GitWeb means you should alway call with a fallback, such as... :: try: fallback_directories = stem.descriptor.remote.from_remote() except IOError: fallback_directories = stem.descriptor.remote.from_cache() :param int timeout: seconds to wait before timing out the request :returns: **dict** of **str** fingerprints to their :class:`~stem.descriptor.remote.FallbackDirectory` :raises: **IOError** if unable to retrieve the fallback directories """ try: fallback_dir_page = str_tools._to_unicode(urllib.urlopen(GITWEB_FALLBACK_DIR_URL, timeout = timeout).read()) except: exc = sys.exc_info()[1] raise IOError("Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_DIR_URL, exc)) # Example of an entry... # # "5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33" # " weight=43680", results = {} for line in fallback_dir_page.splitlines(): if line.startswith('"'): addr_line_match = re.match('"([\d\.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*', line) if addr_line_match: address, dir_port, or_port, fingerprint = addr_line_match.groups() if not connection.is_valid_ipv4_address(address): raise IOError('%s has an invalid address: %s' % (fingerprint, address)) elif not connection.is_valid_port(or_port): raise IOError('%s has an invalid or_port: %s' % (fingerprint, or_port)) elif not connection.is_valid_port(dir_port): raise IOError('%s has an invalid dir_port: %s' % (fingerprint, dir_port)) elif not tor_tools.is_valid_fingerprint(fingerprint): raise IOError('%s has an invalid fingerprint: %s' % (fingerprint, fingerprint)) results[fingerprint] = FallbackDirectory( address = address, or_port = int(or_port), dir_port = int(dir_port), fingerprint = fingerprint, ) return results
def __init__(self, address: Optional[str] = None, or_port: Optional[Union[int, str]] = None, dir_port: Optional[Union[int, str]] = None, fingerprint: Optional[str] = None, nickname: Optional[str] = None, orport_v6: Optional[Tuple[str, int]] = None, v3ident: Optional[str] = None) -> None: super(Authority, self).__init__(address, or_port, dir_port, fingerprint, nickname, orport_v6) if v3ident and not tor_tools.is_valid_fingerprint(v3ident): identifier = '%s (%s)' % (fingerprint, nickname) if nickname else fingerprint raise ValueError('%s has an invalid v3ident: %s' % (identifier, v3ident)) self.v3ident = v3ident
def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, orport_v6 = None, v3ident = None, is_bandwidth_authority = False): super(Authority, self).__init__(address, or_port, dir_port, fingerprint, nickname, orport_v6) if v3ident and not tor_tools.is_valid_fingerprint(v3ident): identifier = '%s (%s)' % (fingerprint, nickname) if nickname else fingerprint raise ValueError('%s has an invalid v3ident: %s' % (identifier, v3ident)) self.v3ident = v3ident self.is_bandwidth_authority = is_bandwidth_authority
def test_parse_file(self): """ Try parsing a document via the _parse_file() function. """ # parse file and assert values descriptors = list(_parse_file(io.BytesIO(TEST_DESC))) self.assertEqual(3, len(descriptors)) self.assertTrue(isinstance(descriptors[0], TorDNSEL)) desc = descriptors[1] self.assertTrue(is_valid_fingerprint(desc.fingerprint)) self.assertEqual("00FF300624FECA7F40515C8D854EE925332580D6", desc.fingerprint) self.assertEqual(datetime.datetime(2013, 8, 18, 7, 2, 14), desc.published) self.assertEqual(datetime.datetime(2013, 8, 18, 9, 2, 58), desc.last_status) self.assertEqual(3, len(desc.exit_addresses)) exit = desc.exit_addresses[0] self.assertEqual("82.252.181.153", exit[0]) self.assertEqual(datetime.datetime(2013, 8, 18, 8, 3, 1), exit[1]) # block content raises value error extra = b"ExtraContent goes here\n" descriptors = _parse_file(io.BytesIO(TEST_DESC + extra)) self.assertRaises(ValueError, list, descriptors) # malformed fingerprint raises value errors extra = b"ExitNode 030B22437D99B2DB2908B747B6" self.assertRaises(ValueError, list, _parse_file(io.BytesIO(TEST_DESC + extra))) # malformed date raises value errors self.assertRaises(ValueError, list, _parse_file(io.BytesIO(TEST_DESC + MALFORMED_ENTRY_1))) # skip exit address if malformed date and validate is False desc = _parse_file(io.BytesIO(MALFORMED_ENTRY_2), validate=False).next() self.assertTrue(is_valid_fingerprint(desc.fingerprint)) self.assertEqual("030B22437D99B2DB2908B747B6962EAD13AB4038", desc.fingerprint) self.assertEqual(0, len(desc.exit_addresses))
def from_remote(timeout=60): """ Reads and parses tor's latest fallback directories `from gitweb.torproject.org <https://gitweb.torproject.org/tor.git/plain/src/or/fallback_dirs.inc>`_. Note that while convenient, this reliance on GitWeb means you should alway call with a fallback, such as... :: try: fallback_directories = stem.descriptor.remote.from_remote() except IOError: fallback_directories = stem.descriptor.remote.from_cache() :param int timeout: seconds to wait before timing out the request :returns: **dict** of **str** fingerprints to their :class:`~stem.descriptor.remote.FallbackDirectory` :raises: **IOError** if unable to retrieve the fallback directories """ try: fallback_dir_page = str_tools._to_unicode( urllib.urlopen(GITWEB_FALLBACK_DIR_URL, timeout=timeout).read()) except: exc = sys.exc_info()[1] raise IOError( "Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_DIR_URL, exc)) # Example of an entry... # # "5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33" # " ipv6=[2a03:b0c0:0:1010::a4:b001]:9001" # " weight=43680", results, attr = {}, {} for line in fallback_dir_page.splitlines(): if line.startswith('"'): addr_line_match = re.match( '"([\d\.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*', line) ipv6_line_match = re.match('" ipv6=\[([\da-f:]+)\]:(\d+)"', line) if addr_line_match: address, dir_port, or_port, fingerprint = addr_line_match.groups( ) if not connection.is_valid_ipv4_address(address): raise IOError('%s has an invalid IPv4 address: %s' % (fingerprint, address)) elif not connection.is_valid_port(or_port): raise IOError('%s has an invalid or_port: %s' % (fingerprint, or_port)) elif not connection.is_valid_port(dir_port): raise IOError('%s has an invalid dir_port: %s' % (fingerprint, dir_port)) elif not tor_tools.is_valid_fingerprint(fingerprint): raise IOError('%s has an invalid fingerprint: %s' % (fingerprint, fingerprint)) attr = { 'address': address, 'or_port': int(or_port), 'dir_port': int(dir_port), 'fingerprint': fingerprint, } elif ipv6_line_match: address, port = ipv6_line_match.groups() if not connection.is_valid_ipv6_address(address): raise IOError('%s has an invalid IPv6 address: %s' % (fingerprint, address)) elif not connection.is_valid_port(port): raise IOError( '%s has an invalid ORPort for its IPv6 endpoint: %s' % (fingerprint, port)) attr['orport_v6'] = (address, int(port)) elif line.startswith('" weight=') and 'fingerprint' in attr: results[attr.get('fingerprint')] = FallbackDirectory( address=attr.get('address'), or_port=attr.get('or_port'), dir_port=attr.get('dir_port'), fingerprint=attr.get('fingerprint'), orport_v6=attr.get('orport_v6'), ) attr = {} return results
from __future__ import print_function from __future__ import unicode_literals import sys import stem.descriptor.remote as remote import stem.util.tor_tools as tor_tools if len(sys.argv) <= 1: print('Usage: %s fingerprint ...' % sys.argv[0]) sys.exit(1) input_list = sys.argv[1:] for fingerprint in input_list: if not tor_tools.is_valid_fingerprint(fingerprint): print("'%s' isn't a valid relay fingerprint" % fingerprint) sys.exit(2) found_list = [] desc_query = remote.get_server_descriptors(input_list, retries=3, timeout=30) for desc in desc_query.run(): assert desc.fingerprint in input_list # Skip duplicates on retries if desc.fingerprint in found_list: continue found_list.append(desc.fingerprint) if not desc.dir_port: print("# %s needs a DirPort" % desc.fingerprint) else:
def _parse_v2(fallback_dir_page): # Example of an entry... # # "5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB" # " ipv6=[2a01:4f8:162:51e2::2]:9001" # /* nickname=rueckgrat */ # /* extrainfo=1 */ results, attr = {}, {} for line in fallback_dir_page.splitlines(): addr_line_match = re.match( '"([\d\.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*', line) nickname_match = re.match('/\* nickname=(\S+) \*/', line) has_extrainfo_match = re.match('/\* extrainfo=([0-1]) \*/', line) ipv6_line_match = re.match('" ipv6=\[([\da-f:]+)\]:(\d+)"', line) if addr_line_match: address, dir_port, or_port, fingerprint = addr_line_match.groups( ) if not connection.is_valid_ipv4_address(address): raise IOError('%s has an invalid IPv4 address: %s' % (fingerprint, address)) elif not connection.is_valid_port(or_port): raise IOError('%s has an invalid or_port: %s' % (fingerprint, or_port)) elif not connection.is_valid_port(dir_port): raise IOError('%s has an invalid dir_port: %s' % (fingerprint, dir_port)) elif not tor_tools.is_valid_fingerprint(fingerprint): raise IOError('%s has an invalid fingerprint: %s' % (fingerprint, fingerprint)) attr = { 'address': address, 'or_port': int(or_port), 'dir_port': int(dir_port), 'fingerprint': fingerprint, } elif ipv6_line_match: address, port = ipv6_line_match.groups() if not connection.is_valid_ipv6_address(address): raise IOError('%s has an invalid IPv6 address: %s' % (fingerprint, address)) elif not connection.is_valid_port(port): raise IOError( '%s has an invalid ORPort for its IPv6 endpoint: %s' % (fingerprint, port)) attr['orport_v6'] = (address, int(port)) elif nickname_match: nickname = nickname_match.group(1) if not tor_tools.is_valid_nickname(nickname): raise IOError('%s has an invalid nickname: %s' % (fingerprint, nickname)) attr['nickname'] = nickname elif has_extrainfo_match: attr['has_extrainfo'] = has_extrainfo_match.group(1) == '1' results[attr.get('fingerprint')] = FallbackDirectory( address=attr.get('address'), or_port=attr.get('or_port'), dir_port=attr.get('dir_port'), fingerprint=attr.get('fingerprint'), nickname=attr.get('nickname'), has_extrainfo=attr.get('has_extrainfo', False), orport_v6=attr.get('orport_v6'), ) attr = {} return results
def _parse_v1(fallback_dir_page): # Example of an entry... # # "5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33" # " ipv6=[2a03:b0c0:0:1010::a4:b001]:9001" # " weight=43680", # TODO: this method can be removed once gitweb provides a v2 formatted document results, attr = {}, {} for line in fallback_dir_page.splitlines(): if line.startswith('"'): addr_line_match = re.match( '"([\d\.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*', line) ipv6_line_match = re.match('" ipv6=\[([\da-f:]+)\]:(\d+)"', line) if addr_line_match: address, dir_port, or_port, fingerprint = addr_line_match.groups( ) if not connection.is_valid_ipv4_address(address): raise IOError('%s has an invalid IPv4 address: %s' % (fingerprint, address)) elif not connection.is_valid_port(or_port): raise IOError('%s has an invalid or_port: %s' % (fingerprint, or_port)) elif not connection.is_valid_port(dir_port): raise IOError('%s has an invalid dir_port: %s' % (fingerprint, dir_port)) elif not tor_tools.is_valid_fingerprint(fingerprint): raise IOError('%s has an invalid fingerprint: %s' % (fingerprint, fingerprint)) attr = { 'address': address, 'or_port': int(or_port), 'dir_port': int(dir_port), 'fingerprint': fingerprint, } elif ipv6_line_match: address, port = ipv6_line_match.groups() if not connection.is_valid_ipv6_address(address): raise IOError('%s has an invalid IPv6 address: %s' % (fingerprint, address)) elif not connection.is_valid_port(port): raise IOError( '%s has an invalid ORPort for its IPv6 endpoint: %s' % (fingerprint, port)) attr['orport_v6'] = (address, int(port)) elif line.startswith('" weight=') and 'fingerprint' in attr: results[attr.get('fingerprint')] = FallbackDirectory( address=attr.get('address'), or_port=attr.get('or_port'), dir_port=attr.get('dir_port'), fingerprint=attr.get('fingerprint'), orport_v6=attr.get('orport_v6'), ) attr = {} return results