def use_directory_mirrors(
self) -> stem.descriptor.networkstatus.NetworkStatusDocumentV3:
"""
Downloads the present consensus and configures ourselves to use directory
mirrors, in addition to authorities.
:returns: :class:`~stem.descriptor.networkstatus.NetworkStatusDocumentV3`
from which we got the directory mirrors
:raises: **Exception** if unable to determine the directory mirrors
"""
directories = [
auth for auth in stem.directory.Authority.from_cache().values()
if auth.nickname not in DIR_PORT_BLACKLIST
]
new_endpoints = set([
stem.DirPort(directory.address, directory.dir_port)
for directory in directories
])
consensus = list(
self.get_consensus(
document_handler=stem.descriptor.DocumentHandler.DOCUMENT).run(
))[0] # type: ignore
for desc in consensus.routers.values():
if stem.Flag.V2DIR in desc.flags and desc.dir_port:
new_endpoints.add(stem.DirPort(desc.address, desc.dir_port))
# we need our endpoints to be a list rather than set for random.choice()
self._endpoints = list(new_endpoints)
return consensus
def test_query_download(self):
"""
Check Query functionality when we successfully download a descriptor.
"""
query = stem.descriptor.remote.Query(
TEST_RESOURCE,
'server-descriptor 1.0',
endpoints=[stem.DirPort('128.31.0.39', 9131)],
compression=Compression.PLAINTEXT,
validate=True,
skip_crypto_validation=not test.require.CRYPTOGRAPHY_AVAILABLE,
)
self.assertEqual(stem.DirPort('128.31.0.39', 9131),
query._pick_endpoint())
descriptors = list(query)
self.assertEqual(1, len(descriptors))
desc = descriptors[0]
self.assertEqual('moria1', desc.nickname)
self.assertEqual('128.31.0.34', desc.address)
self.assertEqual('9695DFC35FFEB861329B9F1AB04C46397020CE31',
desc.fingerprint)
self.assertEqual(TEST_DESCRIPTOR, desc.get_bytes())
def test_equality(self):
self.assertTrue(
stem.ORPort('12.34.56.78', 80) == stem.ORPort('12.34.56.78', 80))
self.assertTrue(
stem.ORPort('12.34.56.78', 80, [1, 2, 3]) == stem.ORPort(
'12.34.56.78', 80, [1, 2, 3]))
self.assertFalse(
stem.ORPort('12.34.56.78', 80) == stem.ORPort('12.34.56.88', 80))
self.assertFalse(
stem.ORPort('12.34.56.78', 80) == stem.ORPort('12.34.56.78', 443))
self.assertFalse(
stem.ORPort('12.34.56.78', 80, [2, 3]) == stem.ORPort(
'12.34.56.78', 80, [1, 2, 3]))
self.assertTrue(
stem.DirPort('12.34.56.78', 80) == stem.DirPort('12.34.56.78', 80))
self.assertFalse(
stem.DirPort('12.34.56.78', 80) == stem.DirPort('12.34.56.88', 80))
self.assertFalse(
stem.DirPort('12.34.56.78', 80) == stem.DirPort(
'12.34.56.78', 443))
self.assertFalse(
stem.ORPort('12.34.56.78', 80) == stem.DirPort('12.34.56.78', 80))
self.assertFalse(
stem.DirPort('12.34.56.78', 80) == stem.ORPort('12.34.56.78', 80))
def test_malformed_content(self):
"""
Query with malformed descriptor content.
"""
query = stem.descriptor.remote.Query(
TEST_RESOURCE,
'server-descriptor 1.0',
endpoints = [stem.DirPort('128.31.0.39', 9131)],
compression = Compression.PLAINTEXT,
validate = True,
)
# checking via the iterator
descriptors = list(query)
self.assertEqual(0, len(descriptors))
self.assertEqual(ValueError, type(query.error))
self.assertEqual("Descriptor must have a 'router' entry", str(query.error))
# check via the run() method
self.assertRaises(ValueError, query.run)
query.stop()
def test_using_dirport(self):
"""
Download a descriptor through the DirPort.
"""
reply = stem.descriptor.remote.their_server_descriptor(
endpoints = [stem.DirPort('12.34.56.78', 1100)],
validate = True,
)
self.assertEqual(1, len(list(reply)))
self.assertEqual('moria1', list(reply)[0].nickname)
self.assertEqual(5, len(reply.reply_headers))
def test_downloading_via_dirport(self):
moria1 = stem.directory.Authority.from_cache()['moria1']
desc = list(
stem.descriptor.remote.their_server_descriptor(
endpoints=[stem.DirPort(moria1.address, moria1.dir_port)],
fall_back_to_authority=False,
).run())[0]
self.assertEqual('moria1', desc.nickname)
self.assertTrue(
isinstance(desc,
stem.descriptor.server_descriptor.ServerDescriptor))
def test_constructor(self):
endpoint = stem.ORPort('12.34.56.78', 80)
self.assertEqual('12.34.56.78', endpoint.address)
self.assertEqual(80, endpoint.port)
self.assertEqual(None, endpoint.link_protocols)
endpoint = stem.ORPort('12.34.56.78', 80, [3])
self.assertEqual('12.34.56.78', endpoint.address)
self.assertEqual(80, endpoint.port)
self.assertEqual([3], endpoint.link_protocols)
endpoint = stem.DirPort('12.34.56.78', 80)
self.assertEqual('12.34.56.78', endpoint.address)
self.assertEqual(80, endpoint.port)
def test_can_iterate_multiple_times(self):
query = stem.descriptor.remote.Query(
TEST_RESOURCE,
'server-descriptor 1.0',
endpoints=[stem.DirPort('128.31.0.39', 9131)],
compression=Compression.PLAINTEXT,
validate=True,
skip_crypto_validation=not test.require.CRYPTOGRAPHY_AVAILABLE,
)
# check that iterating over the query provides the descriptors each time
self.assertEqual(1, len(list(query)))
self.assertEqual(1, len(list(query)))
self.assertEqual(1, len(list(query)))
def _pick_endpoint(self, use_authority = False):
"""
Provides an endpoint to query. If we have multiple endpoints then one
is picked at random.
:param bool use_authority: ignores our endpoints and uses a directory
authority instead
:returns: **str** for the url being queried by this request
"""
if use_authority or not self.endpoints:
picked = random.choice([auth for auth in stem.directory.Authority.from_cache().values() if auth.nickname not in ('tor26', 'Bifroest')])
return stem.DirPort(picked.address, picked.dir_port)
else:
return random.choice(self.endpoints)
def _pick_endpoint(self, use_authority = False):
"""
Provides an endpoint to query. If we have multiple endpoints then one
is picked at random.
:param bool use_authority: ignores our endpoints and uses a directory
authority instead
:returns: :class:`stem.Endpoint` for the location to be downloaded
from by this request
"""
if use_authority or not self.endpoints:
picked = random.choice([auth for auth in stem.directory.Authority.from_cache().values() if auth.nickname not in DIR_PORT_BLACKLIST])
return stem.DirPort(picked.address, picked.dir_port)
else:
return random.choice(self.endpoints)
def test_using_authorities(self):
"""
Fetches a descriptor from each of the directory authorities. This is
intended to check that DIRECTORY_AUTHORITIES is still up to date (that
addresses and ports haven't changed).
This is hardcoded to fetch moria1's descriptor. If its fingerprint changes
then this test will need to be updated.
"""
queries = []
for nickname, authority in stem.directory.Authority.from_cache().items(
):
if nickname in stem.descriptor.remote.DIR_PORT_BLACKLIST:
continue
queries.append((stem.descriptor.remote.Query(
'/tor/server/fp/9695DFC35FFEB861329B9F1AB04C46397020CE31',
'server-descriptor 1.0',
endpoints=[
stem.DirPort(authority.address, authority.dir_port)
],
timeout=30,
validate=True,
), authority))
for query, authority in queries:
try:
descriptors = list(query.run())
except Exception as exc:
for query, _ in queries:
query.stop()
self.fail('Unable to use %s (%s:%s, %s): %s' %
(authority.nickname, authority.address,
authority.dir_port, type(exc), exc))
finally:
query.stop()
self.assertEqual(1, len(descriptors))
self.assertEqual('moria1', descriptors[0].nickname)
def get_vote(self, authority, **query_args):
"""
Provides the present vote for a given directory authority.
:param stem.directory.Authority authority: authority for which to retrieve a vote for
:param query_args: additional arguments for the
:class:`~stem.descriptor.remote.Query` constructor
:returns: :class:`~stem.descriptor.remote.Query` for the router status
entries
"""
resource = '/tor/status-vote/current/authority'
if 'endpoint' not in query_args:
query_args['endpoints'] = [
stem.DirPort(authority.address, authority.dir_port)
]
return self.query(resource, **query_args)
def test_query_with_timeout(self, dirport_mock):
def urlopen_call(*args, **kwargs):
time.sleep(0.06)
raise socket.timeout('connection timed out')
dirport_mock.side_effect = urlopen_call
query = stem.descriptor.remote.Query(
TEST_RESOURCE,
'server-descriptor 1.0',
endpoints=[stem.DirPort('128.31.0.39', 9131)],
fall_back_to_authority=False,
timeout=0.1,
validate=True,
)
# After two requests we'll have reached our total permissable timeout.
# It would be nice to check that we don't make a third, but this
# assertion has proved unreliable so only checking for the exception.
self.assertRaises(stem.DownloadTimeout, query.run)
def __init__(self, resource, descriptor_type = None, endpoints = None, compression = None, retries = 2, fall_back_to_authority = False, timeout = None, start = True, block = False, validate = False, document_handler = stem.descriptor.DocumentHandler.ENTRIES, **kwargs):
if not resource.startswith('/'):
raise ValueError("Resources should start with a '/': %s" % resource)
if resource.endswith('.z'):
compression = [Compression.GZIP]
resource = resource[:-2]
elif compression is None:
compression = [Compression.PLAINTEXT]
else:
if isinstance(compression, str):
compression = [compression] # caller provided only a single option
if Compression.ZSTD in compression and not stem.prereq.is_zstd_available():
compression.remove(Compression.ZSTD)
if Compression.LZMA in compression and not stem.prereq.is_lzma_available():
compression.remove(Compression.LZMA)
if not compression:
compression = [Compression.PLAINTEXT]
if descriptor_type:
self.descriptor_type = descriptor_type
else:
self.descriptor_type = _guess_descriptor_type(resource)
self.endpoints = []
if endpoints:
for endpoint in endpoints:
if isinstance(endpoint, tuple) and len(endpoint) == 2:
self.endpoints.append(stem.DirPort(endpoint[0], endpoint[1])) # TODO: remove this in stem 2.0
elif isinstance(endpoint, (stem.ORPort, stem.DirPort)):
self.endpoints.append(endpoint)
else:
raise ValueError("Endpoints must be an stem.ORPort, stem.DirPort, or two value tuple. '%s' is a %s." % (endpoint, type(endpoint).__name__))
self.resource = resource
self.compression = compression
self.retries = retries
self.fall_back_to_authority = fall_back_to_authority
self.content = None
self.error = None
self.is_done = False
self.download_url = None
self.start_time = None
self.timeout = timeout
self.runtime = None
self.validate = validate
self.document_handler = document_handler
self.reply_headers = None
self.kwargs = kwargs
self._downloader_thread = None
self._downloader_thread_lock = threading.RLock()
if start:
self.start()
if block:
self.run(True)
import collections
import getopt
import sys
import stem
import stem.descriptor.remote
import stem.util.connection
import stem.util.tor_tools
# By default downloading moria1's server descriptor from itself.
DEFAULT_ARGS = {
'descriptor_type': 'server',
'fingerprint': '9695DFC35FFEB861329B9F1AB04C46397020CE31',
'download_from': stem.DirPort('128.31.0.34', 9131),
'print_help': False,
}
VALID_TYPES = ('server', 'extrainfo', 'consensus')
HELP_TEXT = """\
Downloads a descriptor through Tor's ORPort or DirPort.
-t, --type TYPE descriptor type to download, options are:
%s
-f, --fingerprint FP relay to download the descriptor of
--orport ADDRESS:PORT ORPort to download from
--dirport ADDRESS:PORT DirPort to download from
-h, --help presents this help
""" % ', '.join(VALID_TYPES)