def test_functional_destinations(conf, cb, rl, persistent_launch_tor):
good_destination = Destination('https://127.0.0.1:28888', 1024, False)
bad_destination = Destination('https://example.example', 1024, False)
session = requests_utils.make_session(persistent_launch_tor, 10)
# Choose a relay that is not an exit
relay = [r for r in rl.relays if r.nickname == 'relay1mbyteMAB'][0]
# Choose an exit, for this test it does not matter the bandwidth
helper = rl.exits_not_bad_allowing_port(bad_destination.port)[0]
circuit_path = [relay.fingerprint, helper.fingerprint]
# Build a circuit.
circuit_id, _ = cb.build_circuit(circuit_path)
# fail three times in a row
is_usable, response = connect_to_destination_over_circuit(
bad_destination, circuit_id, session, persistent_launch_tor, 1024)
is_usable, response = connect_to_destination_over_circuit(
bad_destination, circuit_id, session, persistent_launch_tor, 1024)
is_usable, response = connect_to_destination_over_circuit(
bad_destination, circuit_id, session, persistent_launch_tor, 1024)
destination_list = DestinationList(conf,
[good_destination, bad_destination], cb,
rl, persistent_launch_tor)
functional_destinations = destination_list.functional_destinations
assert [good_destination] == functional_destinations
def test_connect_to_destination_over_circuit_fail(persistent_launch_tor, dests,
cb, rl):
bad_destination = Destination('https://example.example', 1024, False)
session = requests_utils.make_session(persistent_launch_tor, 10)
# Choose a relay that is not an exit
relay = [r for r in rl.relays if r.nickname == 'relay1mbyteMAB'][0]
# Choose an exit, for this test it does not matter the bandwidth
helper = rl.exits_not_bad_allowing_port(bad_destination.port)[0]
circuit_path = [relay.fingerprint, helper.fingerprint]
# Build a circuit.
circuit_id, _ = cb.build_circuit(circuit_path)
# Perform "usability test"
is_usable, response = connect_to_destination_over_circuit(
bad_destination, circuit_id, session, persistent_launch_tor, 1024)
assert is_usable is False
# because it is the first time it fails, failures aren't count
assert bad_destination.is_functional()
# fail three times in a row
is_usable, response = connect_to_destination_over_circuit(
bad_destination, circuit_id, session, persistent_launch_tor, 1024)
is_usable, response = connect_to_destination_over_circuit(
bad_destination, circuit_id, session, persistent_launch_tor, 1024)
assert not bad_destination.is_functional()
def test_make_session(conf, persistent_launch_tor, dests):
uuid_str = str(uuid.uuid4())
settings.init_http_headers(conf.get('scanner', 'nickname'), uuid_str,
str(persistent_launch_tor.get_version()))
session = requests_utils.make_session(
persistent_launch_tor, conf.getfloat('general', 'http_timeout'))
assert session._timeout == conf.getfloat('general', 'http_timeout')
# Because there is not an stream attached to a circuit, this will timeout.
response = None
try:
response = session.get(dests.next().url, verify=False)
except requests.exceptions.ConnectTimeout:
pass
assert response is None
def _perform_usability_test(self):
self._usability_lock.acquire()
log.debug('Perform usability tests')
cont = self._cont
timeout = self._usability_test_timeout
session = requests_utils.make_session(cont, timeout)
usable_dests = []
for dest in self._all_dests:
possible_exits = [
e for e in self._rl.exits
if e.can_exit_to(dest.hostname, dest.port)
]
# Keep the fastest 10% of exits, or 3, whichever is larger
num_keep = int(max(3, len(possible_exits) * 0.1))
possible_exits = sorted(possible_exits,
key=lambda e: e.bandwidth,
reverse=True)
exits = possible_exits[0:num_keep]
if len(exits) < 1:
log.warning("There are no exits to perform usability tests.")
continue
# Try three times to build a circuit to test this destination
circ_id = None
for _ in range(0, 3):
# Pick a random exit
exit = self._rng.choice(exits)
circ_id = self._cb.build_circuit([None, exit.fingerprint])
if circ_id:
break
if not circ_id:
log.warning(
'Unable to build a circuit to test the usability '
'of %s. Assuming it isn\'t usable.', dest.url)
continue
log.debug('Built circ %s %s to test usability of %s', circ_id,
stem_utils.circuit_str(cont, circ_id), dest.url)
is_usable, data = dest.is_usable(circ_id, session, cont)
if not is_usable:
log.warning(data)
self._cb.close_circuit(circ_id)
continue
assert is_usable
log.debug('%s seems usable so we will keep it', dest.url)
usable_dests.append(dest)
self._cb.close_circuit(circ_id)
self._usable_dests = usable_dests
self._last_usability_test = time.time()
self._usability_lock.release()
def test_connect_to_destination_over_circuit_success(persistent_launch_tor,
dests, cb, rl):
destination = dests.next()
session = requests_utils.make_session(persistent_launch_tor, 10)
# Choose a relay that is not an exit
relay = [r for r in rl.relays if r.nickname == 'relay1mbyteMAB'][0]
# Choose an exit, for this test it does not matter the bandwidth
helper = rl.exits_not_bad_allowing_port(destination.port)[0]
circuit_path = [relay.fingerprint, helper.fingerprint]
# build a circuit
circuit_id, _ = cb.build_circuit(circuit_path)
# Perform "usability test"
is_usable, response = connect_to_destination_over_circuit(
destination, circuit_id, session, persistent_launch_tor, 1024)
assert is_usable is True
assert 'content_length' in response
assert destination.is_functional()
def measure_relay(args, conf, destinations, cb, rl, relay):
"""
Select a Web server, a relay to build the circuit,
build the circuit and measure the bandwidth of the given relay.
:return Result: a measurement Result object
"""
log.debug('Measuring %s %s', relay.nickname, relay.fingerprint)
our_nick = conf['scanner']['nickname']
s = requests_utils.make_session(cb.controller,
conf.getfloat('general', 'http_timeout'))
# Probably because the scanner is stopping.
if s is None:
if settings.end_event.is_set():
return None
else:
# In future refactor this should be returned from the make_session
reason = "Unable to get proxies."
log.debug(reason + ' to measure %s %s', relay.nickname,
relay.fingerprint)
return [
ResultError(relay, [], '', our_nick, msg=reason),
]
# Pick a destionation
dest = destinations.next()
# When there're no any functional destinations.
if not dest:
# NOTE: When there're still functional destinations but only one of
# them fail, the error will be included in `ResultErrorStream`.
# Since this is being executed in a thread, the scanner can not
# be stop here, but the `end_event` signal can be set so that the
# main thread stop the scanner.
# It might be useful to store the fact that the destinations fail,
# so store here the error, and set the signal once the error is stored
# (in `resultump`).
log.critical("There are not any functional destinations.\n"
"It is recommended to set several destinations so that "
"the scanner can continue if one fails.")
reason = "No functional destinations"
# Resultdump will set end_event after storing the error
return [
ResultErrorDestination(relay, [], '', our_nick, msg=reason),
]
# Pick a relay to help us measure the given relay. If the given relay is an
# exit, then pick a non-exit. Otherwise pick an exit.
# Instead of ensuring that the relay can exit to all IPs, try first with
# the relay as an exit, if it can exit to some IPs.
if relay.is_exit_not_bad_allowing_port(dest.port):
r = create_path_relay(relay, dest, rl, cb, relay_as_entry=False)
else:
r = create_path_relay(relay, dest, rl, cb)
# When `error_no_helper` is triggered because a helper is not found, what
# can happen in test networks with very few relays, it returns a list with
# the error.
if len(r) == 1:
return r
circ_fps, nicknames, exit_policy = r
# Build the circuit
circ_id, reason = cb.build_circuit(circ_fps)
# If the circuit failed to get created, bad luck, it will be created again
# with other helper.
# Here we won't have the case that an exit tried to build the circuit as
# entry and failed (#40029), cause not checking that it can exit all IPs.
if not circ_id:
return error_no_circuit(circ_fps, nicknames, reason, relay, dest,
our_nick)
log.debug('Built circuit with path %s (%s) to measure %s (%s)', circ_fps,
nicknames, relay.fingerprint, relay.nickname)
# Make a connection to the destination
is_usable, usable_data = connect_to_destination_over_circuit(
dest, circ_id, s, cb.controller, dest._max_dl)
# In the case that the relay was used as an exit, but could not exit
# to the Web server, try again using it as entry, to avoid that it would
# always fail when there's only one Web server.
if not is_usable and \
relay.is_exit_not_bad_allowing_port(dest.port):
log.debug(
"Exit %s (%s) that can't exit all ips, with exit policy %s, failed"
" to connect to %s via circuit %s (%s). Reason: %s. Trying again "
"with it as entry.", relay.fingerprint, relay.nickname,
exit_policy, dest.url, circ_fps, nicknames, usable_data)
r = create_path_relay(relay, dest, rl, cb)
if len(r) == 1:
return r
circ_fps, nicknames, exit_policy = r
circ_id, reason = cb.build_circuit(circ_fps)
if not circ_id:
log.info(
"Exit %s (%s) that can't exit all ips, failed to create "
" circuit as entry: %s (%s).", relay.fingerprint,
relay.nickname, circ_fps, nicknames)
return error_no_circuit(circ_fps, nicknames, reason, relay, dest,
our_nick)
log.debug('Built circuit with path %s (%s) to measure %s (%s)',
circ_fps, nicknames, relay.fingerprint, relay.nickname)
is_usable, usable_data = connect_to_destination_over_circuit(
dest, circ_id, s, cb.controller, dest._max_dl)
if not is_usable:
log.debug(
'Failed to connect to %s to measure %s (%s) via circuit '
'%s (%s). Exit policy: %s. Reason: %s.', dest.url,
relay.fingerprint, relay.nickname, circ_fps, nicknames,
exit_policy, usable_data)
cb.close_circuit(circ_id)
return [
ResultErrorStream(relay,
circ_fps,
dest.url,
our_nick,
msg=usable_data),
]
assert is_usable
assert 'content_length' in usable_data
# FIRST: measure RTT
rtts, reason = measure_rtt_to_server(s, conf, dest,
usable_data['content_length'])
if rtts is None:
log.debug(
'Unable to measure RTT for %s (%s) to %s via circuit '
'%s (%s): %s', relay.fingerprint, relay.nickname, dest.url,
circ_fps, nicknames, reason)
cb.close_circuit(circ_id)
return [
ResultErrorStream(relay,
circ_fps,
dest.url,
our_nick,
msg=str(reason)),
]
# SECOND: measure bandwidth
bw_results, reason = measure_bandwidth_to_server(
s, conf, dest, usable_data['content_length'])
if bw_results is None:
log.debug(
'Failed to measure %s (%s) via circuit %s (%s) to %s. Exit'
' policy: %s. Reason: %s.', relay.fingerprint, relay.nickname,
circ_fps, nicknames, dest.url, exit_policy, reason)
cb.close_circuit(circ_id)
return [
ResultErrorStream(relay,
circ_fps,
dest.url,
our_nick,
msg=str(reason)),
]
cb.close_circuit(circ_id)
# Finally: store result
log.debug('Success measurement for %s (%s) via circuit %s (%s) to %s',
relay.fingerprint, relay.nickname, circ_fps, nicknames, dest.url)
return [
ResultSuccess(rtts, bw_results, relay, circ_fps, dest.url, our_nick),
]
def measure_relay(args, conf, destinations, cb, rl, relay):
s = requests_utils.make_session(cb.controller,
conf.getfloat('general', 'http_timeout'))
# Pick a destionation
dest = destinations.next()
if not dest:
log.warning('Unable to get destination to measure %s %s',
relay.nickname, relay.fingerprint[0:8])
return None
# Pick an exit
exits = rl.exits_can_exit_to(dest.hostname, dest.port)
exits = [e for e in exits if e.fingerprint != relay.fingerprint]
exits = stem_utils.only_relays_with_bandwidth(
cb.controller,
exits,
min_bw=round(relay.bandwidth * 1.25),
max_bw=max(round(relay.bandwidth * 2.00), 100))
if len(exits) < 1:
log.warning('No available exits to help measure %s %s', relay.nickname,
relay.fingerprint[0:8])
# TODO: Return ResultError of some sort
return None
exit = rng.choice(exits)
# Build the circuit
log.debug('We selected exit %s %s (cw=%d) to help measure %s %s (cw=%d)',
exit.nickname, exit.fingerprint[0:8], exit.bandwidth,
relay.nickname, relay.fingerprint[0:8], relay.bandwidth)
our_nick = conf['scanner']['nickname']
circ_fps = [relay.fingerprint, exit.fingerprint]
circ_id = cb.build_circuit(circ_fps)
if not circ_id:
log.warning('Could not build circuit involving %s', relay.nickname)
msg = 'Unable to complete circuit'
return [
ResultErrorCircuit(relay, circ_fps, dest.url, our_nick, msg=msg),
]
log.debug('Built circ %s %s for relay %s %s', circ_id,
stem_utils.circuit_str(cb.controller, circ_id), relay.nickname,
relay.fingerprint[0:8])
# Make a connection to the destionation webserver and make sure it can
# still help us measure
is_usable, usable_data = dest.is_usable(circ_id, s, cb.controller)
if not is_usable:
log.warning(
'When measuring %s %s the destination seemed to have '
'stopped being usable: %s', relay.nickname, relay.fingerprint[0:8],
usable_data)
cb.close_circuit(circ_id)
# TODO: Return a different/new type of ResultError?
msg = 'The destination seemed to have stopped being usable'
return [
ResultErrorStream(relay, circ_fps, dest.url, our_nick, msg=msg),
]
assert is_usable
assert 'content_length' in usable_data
# FIRST: measure RTT
rtts = measure_rtt_to_server(s, conf, dest, usable_data['content_length'])
if rtts is None:
log.warning('Unable to measure RTT to %s via relay %s %s', dest.url,
relay.nickname, relay.fingerprint[0:8])
cb.close_circuit(circ_id)
# TODO: Return a different/new type of ResultError?
msg = 'Something bad happened while measuring RTTs'
return [
ResultErrorStream(relay, circ_fps, dest.url, our_nick, msg=msg),
]
# SECOND: measure bandwidth
bw_results = measure_bandwidth_to_server(s, conf, dest,
usable_data['content_length'])
if bw_results is None:
log.warning('Unable to measure bandwidth to %s via relay %s %s',
dest.url, relay.nickname, relay.fingerprint[0:8])
cb.close_circuit(circ_id)
# TODO: Return a different/new type of ResultError?
msg = 'Something bad happened while measuring bandwidth'
return [
ResultErrorStream(relay, circ_fps, dest.url, our_nick, msg=msg),
]
cb.close_circuit(circ_id)
# Finally: store result
return [
ResultSuccess(rtts, bw_results, relay, circ_fps, dest.url, our_nick),
]
def measure_relay(args, conf, destinations, cb, rl, relay):
s = requests_utils.make_session(cb.controller,
conf.getfloat('general', 'http_timeout'))
# Pick a destionation
dest = destinations.next()
if not dest:
log.warning('Unable to get destination to measure %s %s',
relay.nickname, relay.fingerprint[0:8])
return None
# Pick a relay to help us measure the given relay. If the given relay is an
# exit, then pick a non-exit. Otherwise pick an exit.
helper = None
circ_fps = None
if relay.can_exit_to(dest.hostname, dest.port) and \
relay not in rl.bad_exits:
helper = _pick_ideal_second_hop(relay,
dest,
rl,
cb.controller,
is_exit=False)
if helper:
circ_fps = [helper.fingerprint, relay.fingerprint]
else:
helper = _pick_ideal_second_hop(relay,
dest,
rl,
cb.controller,
is_exit=True)
if helper:
circ_fps = [relay.fingerprint, helper.fingerprint]
if not helper:
# TODO: Return ResultError of some sort
log.warning('Unable to pick a 2nd hop to help measure %s %s',
relay.nickname, relay.fingerprint[0:8])
return None
assert helper
assert circ_fps is not None and len(circ_fps) == 2
# Build the circuit
our_nick = conf['scanner']['nickname']
circ_id = cb.build_circuit(circ_fps)
if not circ_id:
log.warning('Could not build circuit involving %s', relay.nickname)
msg = 'Unable to complete circuit'
return [
ResultErrorCircuit(relay, circ_fps, dest.url, our_nick, msg=msg),
]
log.debug('Built circ %s %s for relay %s %s', circ_id,
stem_utils.circuit_str(cb.controller, circ_id), relay.nickname,
relay.fingerprint[0:8])
# Make a connection to the destionation webserver and make sure it can
# still help us measure
is_usable, usable_data = dest.is_usable(circ_id, s, cb.controller)
if not is_usable:
log.warning(
'When measuring %s %s the destination seemed to have '
'stopped being usable: %s', relay.nickname, relay.fingerprint[0:8],
usable_data)
cb.close_circuit(circ_id)
# TODO: Return a different/new type of ResultError?
msg = 'The destination seemed to have stopped being usable'
return [
ResultErrorStream(relay, circ_fps, dest.url, our_nick, msg=msg),
]
assert is_usable
assert 'content_length' in usable_data
# FIRST: measure RTT
rtts = measure_rtt_to_server(s, conf, dest, usable_data['content_length'])
if rtts is None:
log.warning('Unable to measure RTT to %s via relay %s %s', dest.url,
relay.nickname, relay.fingerprint[0:8])
cb.close_circuit(circ_id)
# TODO: Return a different/new type of ResultError?
msg = 'Something bad happened while measuring RTTs'
return [
ResultErrorStream(relay, circ_fps, dest.url, our_nick, msg=msg),
]
# SECOND: measure bandwidth
bw_results = measure_bandwidth_to_server(s, conf, dest,
usable_data['content_length'])
if bw_results is None:
log.warning('Unable to measure bandwidth to %s via relay %s %s',
dest.url, relay.nickname, relay.fingerprint[0:8])
cb.close_circuit(circ_id)
# TODO: Return a different/new type of ResultError?
msg = 'Something bad happened while measuring bandwidth'
return [
ResultErrorStream(relay, circ_fps, dest.url, our_nick, msg=msg),
]
cb.close_circuit(circ_id)
# Finally: store result
return [
ResultSuccess(rtts, bw_results, relay, circ_fps, dest.url, our_nick),
]
def measure_relay(args, conf, destinations, cb, rl, relay):
"""
Select a Web server, a relay to build the circuit,
build the circuit and measure the bandwidth of the given relay.
:return Result: a measurement Result object
"""
log.debug('Measuring %s %s', relay.nickname, relay.fingerprint)
our_nick = conf['scanner']['nickname']
s = requests_utils.make_session(
cb.controller, conf.getfloat('general', 'http_timeout'))
# Probably because the scanner is stopping.
if s is None:
if settings.end_event.is_set():
return None
else:
# In future refactor this should be returned from the make_session
reason = "Unable to get proxies."
log.debug(reason + ' to measure %s %s',
relay.nickname, relay.fingerprint)
return [
ResultError(relay, [], '', our_nick,
msg=reason),
]
# Pick a destionation
dest = destinations.next()
# When there're no any functional destinations.
if not dest:
# NOTE: When there're still functional destinations but only one of
# them fail, the error will be included in `ResultErrorStream`.
# Since this is being executed in a thread, the scanner can not
# be stop here, but the `end_event` signal can be set so that the
# main thread stop the scanner.
# It might be useful to store the fact that the destinations fail,
# so store here the error, and set the signal once the error is stored
# (in `resultump`).
log.critical("There are not any functional destinations.\n"
"It is recommended to set several destinations so that "
"the scanner can continue if one fails.")
reason = "No functional destinations"
# Resultdump will set end_event after storing the error
return [
ResultErrorDestination(relay, [], '', our_nick, msg=reason),
]
# Pick a relay to help us measure the given relay. If the given relay is an
# exit, then pick a non-exit. Otherwise pick an exit.
helper = None
circ_fps = None
if relay.is_exit_not_bad_allowing_port(dest.port):
helper = _pick_ideal_second_hop(
relay, dest, rl, cb.controller, is_exit=False)
if helper:
circ_fps = [helper.fingerprint, relay.fingerprint]
# stored for debugging
nicknames = [helper.nickname, relay.nickname]
else:
helper = _pick_ideal_second_hop(
relay, dest, rl, cb.controller, is_exit=True)
if helper:
circ_fps = [relay.fingerprint, helper.fingerprint]
nicknames = [relay.nickname, helper.nickname]
if not helper:
reason = 'Unable to select a second relay'
log.debug(reason + ' to help measure %s (%s)',
relay.fingerprint, relay.nickname)
return [
ResultErrorSecondRelay(relay, [], dest.url, our_nick,
msg=reason),
]
# Build the circuit
circ_id, reason = cb.build_circuit(circ_fps)
if not circ_id:
log.debug('Could not build circuit with path %s (%s): %s ',
circ_fps, nicknames, reason)
return [
ResultErrorCircuit(relay, circ_fps, dest.url, our_nick,
msg=reason),
]
log.debug('Built circuit with path %s (%s) to measure %s (%s)',
circ_fps, nicknames, relay.fingerprint, relay.nickname)
# Make a connection to the destination
is_usable, usable_data = connect_to_destination_over_circuit(
dest, circ_id, s, cb.controller, dest._max_dl)
if not is_usable:
log.debug('Destination %s unusable via circuit %s (%s), %s',
dest.url, circ_fps, nicknames, usable_data)
cb.close_circuit(circ_id)
return [
ResultErrorStream(relay, circ_fps, dest.url, our_nick,
msg=usable_data),
]
assert is_usable
assert 'content_length' in usable_data
# FIRST: measure RTT
rtts, reason = measure_rtt_to_server(s, conf, dest,
usable_data['content_length'])
if rtts is None:
log.debug('Unable to measure RTT for %s (%s) to %s via circuit '
'%s (%s): %s', relay.fingerprint, relay.nickname,
dest.url, circ_fps, nicknames, reason)
cb.close_circuit(circ_id)
return [
ResultErrorStream(relay, circ_fps, dest.url, our_nick,
msg=str(reason)),
]
# SECOND: measure bandwidth
bw_results, reason = measure_bandwidth_to_server(
s, conf, dest, usable_data['content_length'])
if bw_results is None:
log.debug('Unable to measure bandwidth for %s (%s) to %s via circuit '
'%s (%s): %s', relay.fingerprint, relay.nickname,
dest.url, circ_fps, nicknames, reason)
cb.close_circuit(circ_id)
return [
ResultErrorStream(relay, circ_fps, dest.url, our_nick,
msg=str(reason)),
]
cb.close_circuit(circ_id)
# Finally: store result
log.debug('Success measurement for %s (%s) via circuit %s (%s) to %s',
relay.fingerprint, relay.nickname, circ_fps, nicknames, dest.url)
return [
ResultSuccess(rtts, bw_results, relay, circ_fps, dest.url, our_nick),
]