class TestDescriptorDownloader(unittest.TestCase):
def tearDown(self):
# prevent our mocks from impacting other tests
stem.descriptor.remote.SINGLETON_DOWNLOADER = None
@patch('stem.client.Relay.connect', _orport_mock(TEST_DESCRIPTOR))
def test_using_orport(self):
"""
Download a descriptor through the ORPort.
"""
reply = stem.descriptor.remote.their_server_descriptor(
endpoints = [stem.ORPort('12.34.56.78', 1100)],
validate = True,
)
self.assertEqual(1, len(list(reply)))
self.assertEqual('moria1', list(reply)[0].nickname)
self.assertEqual(5, len(reply.reply_headers))
def test_orport_response_code_headers(self):
"""
When successful Tor provides a '200 OK' status, but we should accept other 2xx
response codes, reason text, and recognize HTTP errors.
"""
response_code_headers = (
b'HTTP/1.0 200 OK\r\n',
b'HTTP/1.0 205 OK\r\n',
b'HTTP/1.0 200 This is also alright\r\n',
)
for header in response_code_headers:
with patch('stem.client.Relay.connect', _orport_mock(TEST_DESCRIPTOR, response_code_header = header)):
stem.descriptor.remote.their_server_descriptor(
endpoints = [stem.ORPort('12.34.56.78', 1100)],
validate = True,
).run()
with patch('stem.client.Relay.connect', _orport_mock(TEST_DESCRIPTOR, response_code_header = b'HTTP/1.0 500 Kaboom\r\n')):
request = stem.descriptor.remote.their_server_descriptor(
endpoints = [stem.ORPort('12.34.56.78', 1100)],
validate = True,
)
self.assertRaisesWith(stem.ProtocolError, "Response should begin with HTTP success, but was 'HTTP/1.0 500 Kaboom'", request.run)
@patch(URL_OPEN, _dirport_mock(TEST_DESCRIPTOR))
def test_using_dirport(self):
"""
Download a descriptor through the DirPort.
"""
reply = stem.descriptor.remote.their_server_descriptor(
endpoints = [stem.DirPort('12.34.56.78', 1100)],
validate = True,
)
self.assertEqual(1, len(list(reply)))
self.assertEqual('moria1', list(reply)[0].nickname)
self.assertEqual(5, len(reply.reply_headers))
def test_gzip_url_override(self):
query = stem.descriptor.remote.Query(TEST_RESOURCE, start = False)
self.assertEqual([Compression.PLAINTEXT], query.compression)
self.assertEqual(TEST_RESOURCE, query.resource)
query = stem.descriptor.remote.Query(TEST_RESOURCE + '.z', compression = Compression.PLAINTEXT, start = False)
self.assertEqual([Compression.GZIP], query.compression)
self.assertEqual(TEST_RESOURCE, query.resource)
def test_zstd_support_check(self):
with patch('stem.prereq.is_zstd_available', Mock(return_value = True)):
query = stem.descriptor.remote.Query(TEST_RESOURCE, compression = Compression.ZSTD, start = False)
self.assertEqual([Compression.ZSTD], query.compression)
with patch('stem.prereq.is_zstd_available', Mock(return_value = False)):
query = stem.descriptor.remote.Query(TEST_RESOURCE, compression = Compression.ZSTD, start = False)
self.assertEqual([Compression.PLAINTEXT], query.compression)
def test_lzma_support_check(self):
with patch('stem.prereq.is_lzma_available', Mock(return_value = True)):
query = stem.descriptor.remote.Query(TEST_RESOURCE, compression = Compression.LZMA, start = False)
self.assertEqual([Compression.LZMA], query.compression)
with patch('stem.prereq.is_lzma_available', Mock(return_value = False)):
query = stem.descriptor.remote.Query(TEST_RESOURCE, compression = Compression.LZMA, start = False)
self.assertEqual([Compression.PLAINTEXT], query.compression)
@patch(URL_OPEN, _dirport_mock(read_resource('compressed_identity'), encoding = 'identity'))
def test_compression_plaintext(self):
"""
Download a plaintext descriptor.
"""
descriptors = list(stem.descriptor.remote.get_server_descriptors(
'9695DFC35FFEB861329B9F1AB04C46397020CE31',
compression = Compression.PLAINTEXT,
validate = True,
))
self.assertEqual(1, len(descriptors))
self.assertEqual('moria1', descriptors[0].nickname)
@patch(URL_OPEN, _dirport_mock(read_resource('compressed_gzip'), encoding = 'gzip'))
def test_compression_gzip(self):
"""
Download a gip compressed descriptor.
"""
descriptors = list(stem.descriptor.remote.get_server_descriptors(
'9695DFC35FFEB861329B9F1AB04C46397020CE31',
compression = Compression.GZIP,
validate = True,
))
self.assertEqual(1, len(descriptors))
self.assertEqual('moria1', descriptors[0].nickname)
@patch(URL_OPEN, _dirport_mock(read_resource('compressed_zstd'), encoding = 'x-zstd'))
def test_compression_zstd(self):
"""
Download a zstd compressed descriptor.
"""
if not stem.prereq.is_zstd_available():
self.skipTest('(requires zstd module)')
return
descriptors = list(stem.descriptor.remote.get_server_descriptors(
'9695DFC35FFEB861329B9F1AB04C46397020CE31',
compression = Compression.ZSTD,
validate = True,
))
self.assertEqual(1, len(descriptors))
self.assertEqual('moria1', descriptors[0].nickname)
@patch(URL_OPEN, _dirport_mock(read_resource('compressed_lzma'), encoding = 'x-tor-lzma'))
def test_compression_lzma(self):
"""
Download a lzma compressed descriptor.
"""
if not stem.prereq.is_lzma_available():
self.skipTest('(requires lzma module)')
return
descriptors = list(stem.descriptor.remote.get_server_descriptors(
'9695DFC35FFEB861329B9F1AB04C46397020CE31',
compression = Compression.LZMA,
validate = True,
))
self.assertEqual(1, len(descriptors))
self.assertEqual('moria1', descriptors[0].nickname)
@patch(URL_OPEN, _dirport_mock(TEST_DESCRIPTOR))
def test_reply_headers(self):
query = stem.descriptor.remote.get_server_descriptors('9695DFC35FFEB861329B9F1AB04C46397020CE31', start = False)
self.assertEqual(None, query.reply_headers) # initially we don't have a reply
query.run()
self.assertEqual('Fri, 13 Apr 2018 16:35:50 GMT', query.reply_headers.get('date'))
self.assertEqual('application/octet-stream', query.reply_headers.get('content-type'))
self.assertEqual('97.103.17.56', query.reply_headers.get('x-your-address-is'))
self.assertEqual('no-cache', query.reply_headers.get('pragma'))
self.assertEqual('identity', query.reply_headers.get('content-encoding'))
# getting headers should be case insensitive
self.assertEqual('identity', query.reply_headers.get('CoNtEnT-ENCODING'))
# request a header that isn't present
self.assertEqual(None, query.reply_headers.get('no-such-header'))
self.assertEqual('default', query.reply_headers.get('no-such-header', 'default'))
descriptors = list(query)
self.assertEqual(1, len(descriptors))
self.assertEqual('moria1', descriptors[0].nickname)
@patch(URL_OPEN, _dirport_mock(TEST_DESCRIPTOR))
def test_query_download(self):
"""
Check Query functionality when we successfully download a descriptor.
"""
query = stem.descriptor.remote.Query(
TEST_RESOURCE,
'server-descriptor 1.0',
endpoints = [('128.31.0.39', 9131)],
compression = Compression.PLAINTEXT,
validate = True,
)
self.assertEqual(stem.DirPort('128.31.0.39', 9131), query._pick_endpoint())
descriptors = list(query)
self.assertEqual(1, len(descriptors))
desc = descriptors[0]
self.assertEqual('moria1', desc.nickname)
self.assertEqual('128.31.0.34', desc.address)
self.assertEqual('9695DFC35FFEB861329B9F1AB04C46397020CE31', desc.fingerprint)
self.assertEqual(TEST_DESCRIPTOR.strip(), desc.get_bytes())
@patch(URL_OPEN, _dirport_mock(b'some malformed stuff'))
def test_query_with_malformed_content(self):
"""
Query with malformed descriptor content.
"""
query = stem.descriptor.remote.Query(
TEST_RESOURCE,
'server-descriptor 1.0',
endpoints = [('128.31.0.39', 9131)],
compression = Compression.PLAINTEXT,
validate = True,
)
# checking via the iterator
expected_error_msg = 'Content conform to being a server descriptor:\nsome malformed stuff'
descriptors = list(query)
self.assertEqual(0, len(descriptors))
self.assertEqual(ValueError, type(query.error))
self.assertEqual(expected_error_msg, str(query.error))
# check via the run() method
self.assertRaises(ValueError, query.run)
@patch(URL_OPEN)
def test_query_with_timeout(self, dirport_mock):
def urlopen_call(*args, **kwargs):
time.sleep(0.06)
raise socket.timeout('connection timed out')
dirport_mock.side_effect = urlopen_call
query = stem.descriptor.remote.Query(
TEST_RESOURCE,
'server-descriptor 1.0',
endpoints = [('128.31.0.39', 9131)],
fall_back_to_authority = False,
timeout = 0.1,
validate = True,
)
# After two requests we'll have reached our total permissable timeout.
# Check that we don't make a third.
self.assertRaises(socket.timeout, query.run)
self.assertEqual(2, dirport_mock.call_count)
def test_query_with_invalid_endpoints(self):
invalid_endpoints = {
'hello': "'h' is a str.",
('hello',): "'hello' is a str.",
(15,): "'15' is a int.",
(('12.34.56.78', 15, 'third arg'),): "'('12.34.56.78', 15, 'third arg')' is a tuple.",
}
for endpoints, error_suffix in invalid_endpoints.items():
expected_error = 'Endpoints must be an stem.ORPort, stem.DirPort, or two value tuple. ' + error_suffix
self.assertRaisesWith(ValueError, expected_error, stem.descriptor.remote.Query, TEST_RESOURCE, 'server-descriptor 1.0', endpoints = endpoints)
@patch(URL_OPEN, _dirport_mock(TEST_DESCRIPTOR))
def test_can_iterate_multiple_times(self):
query = stem.descriptor.remote.Query(
TEST_RESOURCE,
'server-descriptor 1.0',
endpoints = [('128.31.0.39', 9131)],
compression = Compression.PLAINTEXT,
validate = True,
)
# check that iterating over the query provides the descriptors each time
self.assertEqual(1, len(list(query)))
self.assertEqual(1, len(list(query)))
self.assertEqual(1, len(list(query)))
class TestDescriptorDownloader(unittest.TestCase):
def tearDown(self):
# prevent our mocks from impacting other tests
stem.descriptor.remote.SINGLETON_DOWNLOADER = None
@mock_download(TEST_DESCRIPTOR)
def test_initial_startup(self):
"""
Check that the query can begin downloading in the background when first
constructed.
"""
query = stem.descriptor.remote.get_server_descriptors(
'9695DFC35FFEB861329B9F1AB04C46397020CE31', start=False)
self.assertTrue(query._downloader_task is None)
query = stem.descriptor.remote.get_server_descriptors(
'9695DFC35FFEB861329B9F1AB04C46397020CE31', start=True)
self.assertTrue(query._downloader_task is not None)
query.stop()
def test_stop(self):
"""
Stop a complete, in-process, and unstarted query.
"""
# stop a completed query
with mock_download(TEST_DESCRIPTOR):
query = stem.descriptor.remote.get_server_descriptors(
'9695DFC35FFEB861329B9F1AB04C46397020CE31')
self.assertTrue(query._loop_thread.is_alive())
query.run() # complete the query
self.assertFalse(query._loop_thread.is_alive())
self.assertFalse(query._downloader_task.cancelled())
query.stop() # nothing to do
self.assertFalse(query._loop_thread.is_alive())
self.assertFalse(query._downloader_task.cancelled())
# stop an in-process query
def pause(*args):
time.sleep(5)
with patch('stem.descriptor.remote.Query._download_from',
Mock(side_effect=pause)):
query = stem.descriptor.remote.get_server_descriptors(
'9695DFC35FFEB861329B9F1AB04C46397020CE31')
query.stop() # terminates in-process query
self.assertFalse(query._loop_thread.is_alive())
self.assertTrue(query._downloader_task.cancelled())
# stop an unstarted query
query = stem.descriptor.remote.get_server_descriptors(
'9695DFC35FFEB861329B9F1AB04C46397020CE31', start=False)
query.stop() # nothing to do
self.assertTrue(query._loop_thread is None)
self.assertTrue(query._downloader_task is None)
@mock_download(TEST_DESCRIPTOR)
def test_download(self):
"""
Simply download and parse a descriptor.
"""
reply = stem.descriptor.remote.their_server_descriptor(
endpoints=[stem.ORPort('12.34.56.78', 1100)],
validate=True,
skip_crypto_validation=not test.require.CRYPTOGRAPHY_AVAILABLE,
)
self.assertEqual(1, len(list(reply)))
self.assertEqual(5, len(reply.reply_headers))
desc = list(reply)[0]
self.assertEqual('moria1', desc.nickname)
self.assertEqual('128.31.0.34', desc.address)
self.assertEqual('9695DFC35FFEB861329B9F1AB04C46397020CE31',
desc.fingerprint)
self.assertEqual(TEST_DESCRIPTOR.rstrip(), desc.get_bytes())
def test_response_header_code(self):
"""
When successful Tor provides a '200 OK' status, but we should accept other 2xx
response codes, reason text, and recognize HTTP errors.
"""
response_code_headers = (
b'HTTP/1.0 200 OK\r\n',
b'HTTP/1.0 205 OK\r\n',
b'HTTP/1.0 200 This is also alright\r\n',
)
for header in response_code_headers:
with mock_download(TEST_DESCRIPTOR, response_code_header=header):
stem.descriptor.remote.their_server_descriptor(
endpoints=[stem.ORPort('12.34.56.78', 1100)],
validate=True,
skip_crypto_validation=not test.require.
CRYPTOGRAPHY_AVAILABLE,
).run()
with mock_download(TEST_DESCRIPTOR,
response_code_header=b'HTTP/1.0 500 Kaboom\r\n'):
request = stem.descriptor.remote.their_server_descriptor(
endpoints=[stem.ORPort('12.34.56.78', 1100)],
validate=True,
skip_crypto_validation=not test.require.CRYPTOGRAPHY_AVAILABLE,
)
self.assertRaisesRegexp(
stem.ProtocolError,
"^Response should begin with HTTP success, but was 'HTTP/1.0 500 Kaboom'",
request.run)
@mock_download(TEST_DESCRIPTOR)
def test_reply_header_data(self):
query = stem.descriptor.remote.get_server_descriptors(
'9695DFC35FFEB861329B9F1AB04C46397020CE31', start=False)
self.assertEqual(
None, query.reply_headers) # initially we don't have a reply
query.run()
self.assertEqual('Fri, 13 Apr 2018 16:35:50 GMT',
query.reply_headers.get('Date'))
self.assertEqual('application/octet-stream',
query.reply_headers.get('Content-Type'))
self.assertEqual('97.103.17.56',
query.reply_headers.get('X-Your-Address-Is'))
self.assertEqual('no-cache', query.reply_headers.get('Pragma'))
self.assertEqual('identity',
query.reply_headers.get('Content-Encoding'))
# request a header that isn't present
self.assertEqual(None, query.reply_headers.get('no-such-header'))
self.assertEqual('default',
query.reply_headers.get('no-such-header', 'default'))
descriptors = list(query)
self.assertEqual(1, len(descriptors))
self.assertEqual('moria1', descriptors[0].nickname)
def test_gzip_url_override(self):
query = stem.descriptor.remote.Query(TEST_RESOURCE + '.z',
compression=Compression.PLAINTEXT,
start=False)
self.assertEqual([stem.descriptor.Compression.GZIP], query.compression)
self.assertEqual(TEST_RESOURCE, query.resource)
@mock_download(read_resource('compressed_identity'), encoding='identity')
def test_compression_plaintext(self):
"""
Download a plaintext descriptor.
"""
query = stem.descriptor.remote.get_server_descriptors(
'9695DFC35FFEB861329B9F1AB04C46397020CE31',
compression=Compression.PLAINTEXT,
validate=True,
skip_crypto_validation=not test.require.CRYPTOGRAPHY_AVAILABLE,
)
descriptors = list(query)
self.assertEqual(1, len(descriptors))
self.assertEqual('moria1', descriptors[0].nickname)
@mock_download(read_resource('compressed_gzip'), encoding='gzip')
def test_compression_gzip(self):
"""
Download a gip compressed descriptor.
"""
query = stem.descriptor.remote.get_server_descriptors(
'9695DFC35FFEB861329B9F1AB04C46397020CE31',
compression=Compression.GZIP,
validate=True,
skip_crypto_validation=not test.require.CRYPTOGRAPHY_AVAILABLE,
)
descriptors = list(query)
self.assertEqual(1, len(descriptors))
self.assertEqual('moria1', descriptors[0].nickname)
@mock_download(read_resource('compressed_zstd'), encoding='x-zstd')
def test_compression_zstd(self):
"""
Download a zstd compressed descriptor.
"""
if not Compression.ZSTD.available:
self.skipTest('(requires zstd module)')
query = stem.descriptor.remote.get_server_descriptors(
'9695DFC35FFEB861329B9F1AB04C46397020CE31',
compression=Compression.ZSTD,
validate=True,
)
descriptors = list(query)
self.assertEqual(1, len(descriptors))
self.assertEqual('moria1', descriptors[0].nickname)
@mock_download(read_resource('compressed_lzma'), encoding='x-tor-lzma')
def test_compression_lzma(self):
"""
Download a lzma compressed descriptor.
"""
if not Compression.LZMA.available:
self.skipTest('(requires lzma module)')
query = stem.descriptor.remote.get_server_descriptors(
'9695DFC35FFEB861329B9F1AB04C46397020CE31',
compression=Compression.LZMA,
validate=True,
)
descriptors = list(query)
self.assertEqual(1, len(descriptors))
self.assertEqual('moria1', descriptors[0].nickname)
@mock_download(TEST_DESCRIPTOR)
def test_each_getter(self):
"""
Surface level exercising of each getter method for downloading descriptors.
"""
queries = []
downloader = stem.descriptor.remote.get_instance()
queries.append(downloader.get_server_descriptors())
queries.append(downloader.get_extrainfo_descriptors())
queries.append(downloader.get_microdescriptors('test-hash'))
queries.append(downloader.get_consensus())
queries.append(
downloader.get_vote(
stem.directory.Authority.from_cache()['moria1']))
queries.append(downloader.get_key_certificates())
queries.append(downloader.get_bandwidth_file())
queries.append(downloader.get_detached_signatures())
for query in queries:
query.stop()
@mock_download(b'some malformed stuff')
def test_malformed_content(self):
"""
Query with malformed descriptor content.
"""
query = stem.descriptor.remote.Query(
TEST_RESOURCE,
'server-descriptor 1.0',
endpoints=[stem.DirPort('128.31.0.39', 9131)],
compression=Compression.PLAINTEXT,
validate=True,
)
# checking via the iterator
descriptors = list(query)
self.assertEqual(0, len(descriptors))
self.assertEqual(ValueError, type(query.error))
self.assertEqual("Descriptor must have a 'router' entry",
str(query.error))
# check via the run() method
self.assertRaises(ValueError, query.run)
def test_query_with_invalid_endpoints(self):
invalid_endpoints = {
'hello': "'h' is a str.",
('hello', ): "'hello' is a str.",
(('hello', ), ): "'('hello',)' is a tuple.",
(15, ): "'15' is a int.",
}
for endpoints, error_suffix in invalid_endpoints.items():
expected_error = 'Endpoints must be an stem.ORPort or stem.DirPort. ' + error_suffix
self.assertRaisesWith(ValueError,
expected_error,
stem.descriptor.remote.Query,
TEST_RESOURCE,
'server-descriptor 1.0',
endpoints=endpoints)
@mock_download(TEST_DESCRIPTOR)
def test_can_iterate_multiple_times(self):
query = stem.descriptor.remote.Query(
TEST_RESOURCE,
'server-descriptor 1.0',
endpoints=[stem.DirPort('128.31.0.39', 9131)],
compression=Compression.PLAINTEXT,
validate=True,
skip_crypto_validation=not test.require.CRYPTOGRAPHY_AVAILABLE,
)
# check that iterating over the query provides the descriptors each time
self.assertEqual(1, len(list(query)))
self.assertEqual(1, len(list(query)))
self.assertEqual(1, len(list(query)))