async def _download_from(self, endpoint: stem.Endpoint) -> bytes:
http_request = '\r\n'.join((
'GET %s HTTP/1.0' % self.resource,
'Accept-Encoding: %s' %
', '.join(map(lambda c: c.encoding, self.compression)),
'User-Agent: %s' % stem.USER_AGENT,
)) + '\r\n\r\n'
if isinstance(endpoint, stem.ORPort):
link_protocols = endpoint.link_protocols if endpoint.link_protocols else [
3
]
async with await stem.client.Relay.connect(
endpoint.address, endpoint.port, link_protocols) as relay:
async with await relay.create_circuit() as circ:
return await circ.directory(http_request, stream_id=1)
elif isinstance(endpoint, stem.DirPort):
reader, writer = await asyncio.open_connection(
endpoint.address, endpoint.port)
writer.write(str_tools._to_bytes(http_request))
return await reader.read()
else:
raise ValueError(
"BUG: endpoints can only be ORPorts or DirPorts, '%s' was a %s"
% (endpoint, type(endpoint).__name__))
def __init__(self, circ_id, command, data, digest = 0, stream_id = 0, recognized = 0, unused = b''):
if 'hash' in str(type(digest)).lower():
# Unfortunately hashlib generates from a dynamic private class so
# isinstance() isn't such a great option. With python2/python3 the
# name is 'hashlib.HASH' whereas PyPy calls it just 'HASH' or 'Hash'.
digest_packed = digest.digest()[:RELAY_DIGEST_SIZE.size]
digest = RELAY_DIGEST_SIZE.unpack(digest_packed)
elif isinstance(digest, (bytes, str)):
digest_packed = digest[:RELAY_DIGEST_SIZE.size]
digest = RELAY_DIGEST_SIZE.unpack(digest_packed)
elif isinstance(digest, int):
pass
else:
raise ValueError('RELAY cell digest must be a hash, string, or int but was a %s' % type(digest).__name__)
super(RelayCell, self).__init__(circ_id, unused)
self.command, self.command_int = RelayCommand.get(command)
self.recognized = recognized
self.stream_id = stream_id
self.digest = digest
self.data = str_tools._to_bytes(data)
if digest == 0:
if not stream_id and self.command in STREAM_ID_REQUIRED:
raise ValueError('%s relay cells require a stream id' % self.command)
elif stream_id and self.command in STREAM_ID_DISALLOWED:
raise ValueError('%s relay cells concern the circuit itself and cannot have a stream id' % self.command)
def __start_tor(self, name, control_port, socks_port, hs_port_mapping=None):
logging.info("Starting Tor {0} process with ControlPort={1}, SocksPort={2}...".format(name, control_port, socks_port))
tor_datadir = "{0}/tor-{1}".format(self.datadir_path, name)
if not os.path.exists(tor_datadir): os.makedirs(tor_datadir)
tor_config_template = "ORPort 0\nDirPort 0\nControlPort {0}\nSocksPort {1}\nSocksListenAddress 127.0.0.1\nClientOnly 1\n\
WarnUnsafeSocks 0\nSafeLogging 0\nMaxCircuitDirtiness 60 seconds\nUseEntryGuards 0\nDataDirectory {2}\nLog INFO stdout\n"
tor_config = tor_config_template.format(control_port, socks_port, tor_datadir)
tor_logpath = "{0}/onionperf.tor.log".format(tor_datadir)
tor_writable = util.FileWritable(tor_logpath)
logging.info("Logging Tor {0} process output to {1}".format(name, tor_logpath))
# from stem.process import launch_tor_with_config
# tor_subp = launch_tor_with_config(tor_config, tor_cmd=self.tor_bin_path, completion_percent=100, init_msg_handler=None, timeout=None, take_ownership=False)
tor_cmd = "{0} -f -".format(self.tor_bin_path)
tor_stdin_bytes = str_tools._to_bytes(tor_config)
tor_ready_str = "Bootstrapped 100"
tor_ready_ev = threading.Event()
tor_args = (tor_cmd, tor_datadir, tor_writable, self.done_event, tor_stdin_bytes, tor_ready_str, tor_ready_ev)
tor_watchdog = threading.Thread(target=watchdog_thread_task, name="tor_{0}_watchdog".format(name), args=tor_args)
tor_watchdog.start()
self.threads.append(tor_watchdog)
# wait until Tor finishes bootstrapping
tor_ready_ev.wait()
torctl_logpath = "{0}/onionperf.torctl.log".format(tor_datadir)
torctl_writable = util.FileWritable(torctl_logpath)
logging.info("Logging Tor {0} control port monitor output to {1}".format(name, torctl_logpath))
# give a few seconds to make sure Tor had time to start listening on the control port
time.sleep(3)
torctl_events = [e for e in monitor.get_supported_torctl_events() if e not in ['DEBUG', 'INFO', 'NOTICE', 'WARN', 'ERR']]
newnym_interval_seconds = 300
torctl_args = (control_port, torctl_writable, torctl_events, newnym_interval_seconds, self.done_event)
torctl_helper = threading.Thread(target=monitor.tor_monitor_run, name="torctl_{0}_helper".format(name), args=torctl_args)
torctl_helper.start()
self.threads.append(torctl_helper)
if hs_port_mapping is not None:
logging.info("Creating ephemeral hidden service with v2 onions...")
with Controller.from_port(port=control_port) as torctl:
torctl.authenticate()
response = torctl.create_ephemeral_hidden_service(hs_port_mapping, detached=True, await_publication=True)
self.hs_service_id = response.service_id
self.hs_control_port = control_port
logging.info("Ephemeral hidden service is available at {0}.onion".format(response.service_id))
logging.info("Creating ephemeral hidden service with v3 onions...")
with Controller.from_port(port=control_port) as torctl:
torctl.authenticate()
response = torctl.create_ephemeral_hidden_service(hs_port_mapping, detached=True, await_publication=True, key_content='ED25519-V3')
self.hs_v3_service_id = response.service_id
self.hs_v3_control_port = control_port
logging.info("Ephemeral hidden service is available at {0}.onion".format(response.service_id))
return tor_writable, torctl_writable
def _parse(self):
content = str(self).lstrip("NEWCONSENSUS\n").rstrip("\nOK")
self.desc = list(stem.descriptor.router_status_entry._parse_file(
io.BytesIO(str_tools._to_bytes(content)),
True,
entry_class = stem.descriptor.router_status_entry.RouterStatusEntryV3,
))
def _parse(self):
content = str(self).lstrip("NEWCONSENSUS\n").rstrip("\nOK")
self.desc = list(stem.descriptor.router_status_entry._parse_file(
io.BytesIO(str_tools._to_bytes(content)),
True,
entry_class = stem.descriptor.router_status_entry.RouterStatusEntryV3,
))
def _parse(self):
content = str(self).lstrip('NEWCONSENSUS\n').rstrip('\nOK')
# TODO: For stem 2.0.0 consider changing 'desc' to 'descriptors' to match
# our other events.
self.desc = list(stem.descriptor.router_status_entry._parse_file(
io.BytesIO(str_tools._to_bytes(content)),
True,
entry_class = stem.descriptor.router_status_entry.RouterStatusEntryV3,
))
def __start_tor(self, name, control_port, socks_port, hs_port_mapping=None):
logging.info("Starting Tor {0} process with ControlPort={1}, SocksPort={2}...".format(name, control_port, socks_port))
tor_datadir = "{0}/tor-{1}".format(self.datadir_path, name)
if not os.path.exists(tor_datadir): os.makedirs(tor_datadir)
tor_config_template = "ORPort 0\nDirPort 0\nControlPort {0}\nSocksPort {1}\nSocksListenAddress 127.0.0.1\nClientOnly 1\n\
WarnUnsafeSocks 0\nSafeLogging 0\nMaxCircuitDirtiness 60 seconds\nUseEntryGuards 0\nDataDirectory {2}\nLog INFO stdout\n"
tor_config = tor_config_template.format(control_port, socks_port, tor_datadir)
tor_logpath = "{0}/onionperf.tor.log".format(tor_datadir)
tor_writable = util.FileWritable(tor_logpath)
logging.info("Logging Tor {0} process output to {1}".format(name, tor_logpath))
# from stem.process import launch_tor_with_config
# tor_subp = launch_tor_with_config(tor_config, tor_cmd=self.tor_bin_path, completion_percent=100, init_msg_handler=None, timeout=None, take_ownership=False)
tor_cmd = "{0} -f -".format(self.tor_bin_path)
tor_stdin_bytes = str_tools._to_bytes(tor_config)
tor_ready_str = "Bootstrapped 100"
tor_ready_ev = threading.Event()
tor_args = (tor_cmd, tor_datadir, tor_writable, self.done_event, tor_stdin_bytes, tor_ready_str, tor_ready_ev)
tor_watchdog = threading.Thread(target=watchdog_thread_task, name="tor_{0}_watchdog".format(name), args=tor_args)
tor_watchdog.start()
self.threads.append(tor_watchdog)
# wait until Tor finishes bootstrapping
tor_ready_ev.wait()
torctl_logpath = "{0}/onionperf.torctl.log".format(tor_datadir)
torctl_writable = util.FileWritable(torctl_logpath)
logging.info("Logging Tor {0} control port monitor output to {1}".format(name, torctl_logpath))
# give a few seconds to make sure Tor had time to start listening on the control port
time.sleep(3)
torctl_events = [e for e in monitor.get_supported_torctl_events() if e not in ['DEBUG', 'INFO', 'NOTICE', 'WARN', 'ERR']]
newnym_interval_seconds = 300
torctl_args = (control_port, torctl_writable, torctl_events, newnym_interval_seconds, self.done_event)
torctl_helper = threading.Thread(target=monitor.tor_monitor_run, name="torctl_{0}_helper".format(name), args=torctl_args)
torctl_helper.start()
self.threads.append(torctl_helper)
if hs_port_mapping is not None:
logging.info("Creating ephemeral hidden service...")
with Controller.from_port(port=control_port) as torctl:
torctl.authenticate()
response = torctl.create_ephemeral_hidden_service(hs_port_mapping, detached=True, await_publication=True)
self.hs_service_id = response.service_id
self.hs_control_port = control_port
logging.info("Ephemeral hidden service is available at {0}.onion".format(response.service_id))
return tor_writable, torctl_writable
def test_to_int(self):
"""
Checks the _to_int() function.
"""
test_inputs = {
'': 0,
'h': 104,
'hi': 26729,
'hello': 448378203247,
str_tools._to_bytes('hello'): 448378203247,
str_tools._to_unicode('hello'): 448378203247,
}
for arg, expected in test_inputs.items():
self.assertEqual(expected, str_tools._to_int(arg))
def test_to_int(self):
"""
Checks the _to_int() function.
"""
test_inputs = {
'': 0,
'h': 104,
'hi': 26729,
'hello': 448378203247,
str_tools._to_bytes('hello'): 448378203247,
str_tools._to_unicode('hello'): 448378203247,
}
for arg, expected in test_inputs.items():
self.assertEqual(expected, str_tools._to_int(arg))
def _parse(self):
if self.address == 'UNKNOWN':
self.address = None
self.directory_fingerprint = None
self.directory_nickname = None
try:
self.directory_fingerprint, self.directory_nickname = \
stem.control._parse_circ_entry(self.directory)
except stem.ProtocolError:
raise stem.ProtocolError("HS_DESC_CONTENT's directory doesn't match a ServerSpec: %s" % self)
self.descriptor = list(stem.descriptor.hidden_service_descriptor._parse_file(
io.BytesIO(str_tools._to_bytes('\n'.join(str(self).splitlines()[1:]))),
))[0]
def __start_tor(self, name, control_port, socks_port, hs_port_mapping=None):
logging.info("Starting Tor {0} process with ControlPort={1}, SocksPort={2}...".format(name, control_port, socks_port))
tor_datadir = "{0}/tor-{1}".format(self.datadir_path, name)
key_path_v3 = "{0}/os_key_v3".format(self.privatedir_path)
if not os.path.exists(tor_datadir): os.makedirs(tor_datadir)
tor_config = self.create_tor_config(control_port,socks_port,tor_datadir,name)
tor_confpath = "{0}/torrc".format(tor_datadir)
with open(tor_confpath, 'wt') as f:
f.write(tor_config)
tor_logpath = "{0}/onionperf.tor.log".format(tor_datadir)
tor_writable = util.FileWritable(tor_logpath)
logging.info("Logging Tor {0} process output to {1}".format(name, tor_logpath))
# from stem.process import launch_tor_with_config
# tor_subp = launch_tor_with_config(tor_config, tor_cmd=self.tor_bin_path, completion_percent=100, init_msg_handler=None, timeout=None, take_ownership=False)
tor_cmd = "{0} -f -".format(self.tor_bin_path)
tor_stdin_bytes = str_tools._to_bytes(tor_config)
tor_ready_str = "Bootstrapped 100"
tor_ready_ev = threading.Event()
tor_args = (tor_cmd, tor_datadir, tor_writable, self.done_event, tor_stdin_bytes, tor_ready_str, tor_ready_ev, False)
tor_watchdog = threading.Thread(target=watchdog_thread_task, name="tor_{0}_watchdog".format(name), args=tor_args)
tor_watchdog.start()
self.threads.append(tor_watchdog)
# wait until Tor finishes bootstrapping
tor_ready_ev.wait()
torctl_logpath = "{0}/onionperf.torctl.log".format(tor_datadir)
torctl_writable = util.FileWritable(torctl_logpath)
logging.info("Logging Tor {0} control port monitor output to {1}".format(name, torctl_logpath))
# give a few seconds to make sure Tor had time to start listening on the control port
time.sleep(3)
torctl_events = [e for e in monitor.get_supported_torctl_events() if e not in ['DEBUG', 'INFO', 'NOTICE', 'WARN', 'ERR']]
newnym_interval_seconds = 300
torctl_args = (control_port, torctl_writable, torctl_events, newnym_interval_seconds, self.drop_guards_interval_hours, self.done_event)
torctl_helper = threading.Thread(target=monitor.tor_monitor_run, name="torctl_{0}_helper".format(name), args=torctl_args)
torctl_helper.start()
self.threads.append(torctl_helper)
if hs_port_mapping is not None:
self.start_onion_service(control_port, hs_port_mapping, key_path_v3)
return tor_writable, torctl_writable
def _parse(self):
if self.address == 'UNKNOWN':
self.address = None
self.directory_fingerprint = None
self.directory_nickname = None
try:
self.directory_fingerprint, self.directory_nickname = \
stem.control._parse_circ_entry(self.directory)
except stem.ProtocolError:
raise stem.ProtocolError("HS_DESC_CONTENT's directory doesn't match a ServerSpec: %s" % self)
# skip the first line (our positional arguments) and last ('OK')
desc_content = str_tools._to_bytes('\n'.join(str(self).splitlines()[1:-1]))
self.descriptor = None
if desc_content:
self.descriptor = list(stem.descriptor.hidden_service_descriptor._parse_file(io.BytesIO(desc_content)))[0]