def run_speedtest(args, conf):
"""Initializes all the data and threads needed to measure the relays.
It launches or connect to Tor in a thread.
It initializes the list of relays seen in the Tor network.
It starts a thread to read the previous measurements and wait for new
measurements to write them to the disk.
It initializes a class that will be used to order the relays depending
on their measurements age.
It initializes the list of destinations that will be used for the
measurements.
It initializes the thread pool that will launch the measurement threads.
The pool starts 3 other threads that are not the measurement (worker)
threads.
Finally, it calls the function that will manage the measurement threads.
"""
global rd, pool, controller
controller = stem_utils.launch_or_connect_to_tor(conf)
# When there will be a refactor where conf is global, this can be removed
# from here.
state = State(conf.getpath('paths', 'state_fname'))
# XXX: tech-debt: create new function to obtain the controller and to
# write the state, so that a unit test to check the state tor version can
# be created
# Store tor version whenever the scanner starts.
state['tor_version'] = str(controller.get_version())
# Call only once to initialize http_headers
settings.init_http_headers(conf.get('scanner', 'nickname'), state['uuid'],
state['tor_version'])
# To do not have to pass args and conf to RelayList, pass an extra
# argument with the data_period
measurements_period = conf.getint('general', 'data_period')
rl = RelayList(args, conf, controller, measurements_period, state)
cb = CB(args, conf, controller, rl)
rd = ResultDump(args, conf)
rp = RelayPrioritizer(args, conf, rl, rd)
destinations, error_msg = DestinationList.from_config(
conf, cb, rl, controller)
if not destinations:
fail_hard(error_msg)
max_pending_results = conf.getint('scanner', 'measurement_threads')
pool = Pool(max_pending_results)
try:
main_loop(args, conf, controller, rl, cb, rd, rp, destinations, pool)
except KeyboardInterrupt:
log.info("Interrupted by the user.")
stop_threads(signal.SIGINT, None)
# Any exception not catched at this point would make the scanner stall.
# Log it and exit gracefully.
except Exception as e:
log.critical(FILLUP_TICKET_MSG)
log.exception(e)
stop_threads(signal.SIGTERM, None, 1)
def _init_controller_port(port):
assert isinstance(port, int)
try:
c = Controller.from_port(port=port)
c.authenticate()
except (IncorrectSocketType, SocketError):
fail_hard("Unable to connect to control port %s.", port)
# TODO: Allow for auth via more than just CookieAuthentication
log.info("Connected to tor via port %s", port)
return c
def __enter__(self):
mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC
self._fd = os.open(self._lock_fname, mode)
log.debug('Going to lock %s', self._lock_fname)
try:
fcntl.flock(self._fd, fcntl.LOCK_EX)
except OSError as e:
fail_hard(
'We couldn\'t call flock. Are you on an unsupported '
'platform? Error: %s', e)
log.debug('Received lock %s', self._lock_fname)
def __init__(self, args, conf):
assert os.path.isdir(conf.getpath('paths', 'datadir'))
self.conf = conf
self.fresh_days = conf.getint('general', 'data_period')
self.datadir = conf.getpath('paths', 'datadir')
self.data = {}
self.data_lock = RLock()
self.thread = Thread(target=self.enter)
self.queue = Queue()
try:
self.thread.start()
except RuntimeError as e:
fail_hard(e)
def main(args, conf):
'''
Main entry point in to the cleanup command.
:param argparse.Namespace args: command line arguments
:param configparser.ConfigParser conf: parsed config files
'''
datadir = conf.getpath('paths', 'datadir')
if not os.path.isdir(datadir):
fail_hard('%s does not exist', datadir)
if not args.no_results:
_clean_result_files(args, conf)
if not args.no_v3bw:
_clean_v3bw_files(args, conf)
def _get_user_config(args, conf=None):
"""Get user configuration.
Search for user configuration in the default path or the path passed as
argument and extend the configuration if they are found.
"""
if not conf:
conf = ConfigParser(interpolation=ExtendedInterpolation(),
converters={'path': _expand_path})
else:
assert isinstance(conf, ConfigParser)
if args.config:
if not os.path.isfile(args.config):
fail_hard('Configuration file %s not found.', args.config)
return _extend_config(conf, args.config)
if os.path.isfile(USER_CONFIG_PATH):
return _extend_config(conf, USER_CONFIG_PATH)
log.debug('No user config found.')
return conf
def _clean_v3bw_files(args, conf):
v3bw_dname = conf['paths']['v3bw_dname']
if not os.path.isdir(v3bw_dname):
fail_hard('%s does not exist', v3bw_dname)
compress_after_days = conf.getint('cleanup',
'v3bw_files_compress_after_days')
delete_after_days = conf.getint('cleanup', 'v3bw_files_delete_after_days')
_check_validity_periods_v3bw(compress_after_days, delete_after_days)
# first delete so that the files to be deleted are not compressed first
files_to_delete = _get_files_mtime_older_than(v3bw_dname,
delete_after_days,
['.v3bw', '.gz'])
_delete_files(v3bw_dname, files_to_delete, dry_run=args.dry_run)
files_to_compress = _get_files_mtime_older_than(v3bw_dname,
compress_after_days,
['.v3bw'])
# when dry_run is true, compress will also show all the files that
# would have been deleted, since they are not really deleted
_compress_files(v3bw_dname, files_to_compress, dry_run=args.dry_run)
def run_speedtest(args, conf):
write_start_ts(conf)
controller, _ = stem_utils.init_controller(
path=conf['tor']['control_socket'])
if not controller:
controller = stem_utils.launch_tor(conf)
else:
log.warning(
'Is sbws already running? '
'We found an existing Tor process at %s. We are not going to '
'launch Tor, nor are we going to try to configure it to behave '
'like we expect. This might work okay, but it also might not. '
'If you experience problems, you should try letting sbws launch '
'Tor for itself. The ability to use an already running Tor only '
'exists for sbws developers. It is expected to be broken and may '
'even lead to messed up results.', conf['tor']['control_socket'])
time.sleep(15)
assert stem_utils.is_controller_okay(controller)
cb = CB(args, conf, controller)
rl = RelayList(args, conf, controller)
rd = ResultDump(args, conf, end_event)
rp = RelayPrioritizer(args, conf, rl, rd)
destinations, error_msg = DestinationList.from_config(
conf, cb, rl, controller)
if not destinations:
fail_hard(error_msg)
max_pending_results = conf.getint('scanner', 'measurement_threads')
pool = Pool(max_pending_results)
pending_results = []
while True:
for target in rp.best_priority():
log.debug('Measuring %s %s', target.nickname,
target.fingerprint[0:8])
callback = result_putter(rd)
callback_err = result_putter_error(target)
async_result = pool.apply_async(
dispatch_worker_thread,
[args, conf, destinations, cb, rl, target], {}, callback,
callback_err)
pending_results.append(async_result)
while len(pending_results) >= max_pending_results:
time.sleep(5)
pending_results = [r for r in pending_results if not r.ready()]
def _clean_result_files(args, conf):
datadir = conf.getpath('paths', 'datadir')
if not os.path.isdir(datadir):
fail_hard('%s does not exist', datadir)
compress_after_days = conf.getint('cleanup',
'data_files_compress_after_days')
delete_after_days = conf.getint('cleanup', 'data_files_delete_after_days')
# first delete so that the files to be deleted are not compressed first
files_to_delete = _get_files_mtime_older_than(datadir, delete_after_days,
['.txt', '.gz'])
_delete_files(datadir, files_to_delete, dry_run=args.dry_run)
# when dry_run is true, compress will also show all the files that
# would have been deleted, since they are not really deleted
files_to_compress = _get_files_mtime_older_than(datadir,
compress_after_days,
['.txt'])
_compress_files(datadir, files_to_compress, dry_run=args.dry_run)
def main(args, conf):
if conf.getint('scanner', 'measurement_threads') < 1:
fail_hard('Number of measurement threads must be larger than 1')
min_dl = conf.getint('scanner', 'min_download_size')
max_dl = conf.getint('scanner', 'max_download_size')
if max_dl < min_dl:
fail_hard('Max download size %d cannot be smaller than min %d', max_dl,
min_dl)
os.makedirs(conf.getpath('paths', 'datadir'), exist_ok=True)
state = State(conf.getpath('paths', 'state_fname'))
state['scanner_started'] = now_isodt_str()
# Generate an unique identifier for each scanner
if 'uuid' not in state:
state['uuid'] = str(uuid.uuid4())
run_speedtest(args, conf)
def main(args, conf):
'''
Main entry point into the stats command.
:param argparse.Namespace args: command line arguments
:param configparser.ConfigParser conf: parsed config files
'''
datadir = conf.getpath('paths', 'datadir')
if not os.path.isdir(datadir):
fail_hard('%s does not exist', datadir)
fresh_days = conf.getint('general', 'data_period')
results = load_recent_results_in_datadir(fresh_days,
datadir,
success_only=False)
if len(results) < 1:
log.warning('No fresh results')
return
print_stats(args, results)
def _check_validity_periods_results(data_period, compress_after_days,
delete_after_days):
if compress_after_days - 2 < data_period:
fail_hard(
'For safetly, cleanup/data_files_compress_after_days (%d) must be '
'at least 2 days larger than general/data_period (%d)',
compress_after_days, data_period)
if delete_after_days < compress_after_days:
fail_hard(
'cleanup/data_files_delete_after_days (%d) must be the same or '
'larger than cleanup/data_files_compress_after_days (%d)',
delete_after_days, compress_after_days)
if compress_after_days / 2 < data_period:
log.warning(
'cleanup/data_files_compress_after_days (%d) is less than twice '
'general/data_period (%d). For ease of parsing older results '
'if necessary, it is recommended to make '
'data_files_compress_after_days at least twice the data_period.',
compress_after_days, data_period)
return True
def launch_tor(conf):
assert isinstance(conf, ConfigParser)
os.makedirs(conf.getpath('tor', 'datadir'), mode=0o700, exist_ok=True)
os.makedirs(conf.getpath('tor', 'log'), exist_ok=True)
os.makedirs(conf.getpath('tor', 'run_dpath'), mode=0o700, exist_ok=True)
# Bare minimum things, more or less
torrc = copy.deepcopy(TORRC_STARTING_POINT)
# Very important and/or common settings that we don't know until runtime
# The rest of the settings are in globals.py
torrc.update({
'DataDirectory': conf.getpath('tor', 'datadir'),
'PidFile': conf.getpath('tor', 'pid'),
'ControlSocket': conf.getpath('tor', 'control_socket'),
'Log': [
'NOTICE file {}'.format(os.path.join(conf.getpath('tor', 'log'),
'notice.log')),
],
# Things needed to make circuits fail a little faster. We get the
# circuit_timeout as a string instead of an int on purpose: stem only
# accepts strings.
'LearnCircuitBuildTimeout': '0',
'CircuitBuildTimeout': conf['general']['circuit_timeout'],
})
torrc = parse_user_torrc_config(torrc, conf['tor']['extra_lines'])
# Finally launch Tor
try:
stem.process.launch_tor_with_config(
torrc, init_msg_handler=log.debug, take_ownership=True)
except Exception as e:
fail_hard('Error trying to launch tor: %s', e)
# And return a controller to it
cont = _init_controller_socket(conf.getpath('tor', 'control_socket'))
# Set options that can fail at runtime
set_torrc_options_can_fail(cont)
# Set runtime options
set_torrc_runtime_options(cont)
log.info('Started and connected to Tor %s via %s', cont.get_version(),
conf.getpath('tor', 'control_socket'))
return cont
def main(args, conf):
if conf.getint('scanner', 'measurement_threads') < 1:
fail_hard('Number of measurement threads must be larger than 1')
min_dl = conf.getint('scanner', 'min_download_size')
max_dl = conf.getint('scanner', 'max_download_size')
if max_dl < min_dl:
fail_hard('Max download size %d cannot be smaller than min %d', max_dl,
min_dl)
os.makedirs(conf.getpath('paths', 'datadir'), exist_ok=True)
state = State(conf.getpath('paths', 'state_fname'))
state['scanner_started'] = now_isodt_str()
try:
run_speedtest(args, conf)
except KeyboardInterrupt as e:
raise e
finally:
end_event.set()
def main(args, conf):
if is_initted(args.directory):
fail_hard('Directory already seems to be initted')
if not os.path.isdir(args.directory):
log.info('Creating %s', args.directory)
os.makedirs(args.directory, exist_ok=False)
# Create config.log.ini ####
touch_file(os.path.join(args.directory, 'config.log.ini'))
# Create config.ini ####
fname = os.path.join(args.directory, 'config.ini')
if os.path.exists(fname) and not os.path.isfile(fname):
fail_hard('Don\'t know how to handle %s existing as a non-file', fname)
if os.path.isfile(fname) and not query_yes_no(
'Is it okay to overwrite {}?'.format(fname), default=None):
fail_hard('Cannot continue')
c = get_user_example_config()
if 'paths' not in c:
c['paths'] = {}
c['paths']['sbws_home'] = args.directory
log.info('Creating %s based on example config', fname)
with open(fname, 'wt') as fd:
c.write(fd)
def launch_tor(conf):
os.makedirs(conf.getpath('tor', 'datadir'), mode=0o700, exist_ok=True)
os.makedirs(conf.getpath('tor', 'log'), exist_ok=True)
os.makedirs(conf.getpath('tor', 'run_dpath'), mode=0o700, exist_ok=True)
# Bare minimum things, more or less
torrc = copy.deepcopy(TORRC_STARTING_POINT)
# Very important and/or common settings that we don't know until runtime
# The rest of the settings are in globals.py
torrc.update({
'DataDirectory':
conf.getpath('tor', 'datadir'),
'PidFile':
conf.getpath('tor', 'pid'),
'ControlSocket':
conf.getpath('tor', 'control_socket'),
'Log': [
'NOTICE file {}'.format(
os.path.join(conf.getpath('tor', 'log'), 'notice.log')),
],
'CircuitBuildTimeout':
conf['general']['circuit_timeout'],
})
torrc = parse_user_torrc_config(torrc, conf['tor']['extra_lines'])
# Finally launch Tor
try:
# If there is already a tor process running with the same control
# socket, this will exit here.
stem.process.launch_tor_with_config(torrc,
init_msg_handler=log.debug,
take_ownership=True)
except Exception as e:
fail_hard('Error trying to launch tor: %s', e)
log.info("Started own tor.")
# And return a controller to it
cont = _init_controller_socket(conf.getpath('tor', 'control_socket'))
# In the case it was not possible to connect to own tor socket.
if not cont:
fail_hard('Could not connect to own tor control socket.')
return cont
def main(args, conf):
os.makedirs(conf.getpath('paths', 'v3bw_dname'), exist_ok=True)
datadir = conf.getpath('paths', 'datadir')
if not os.path.isdir(datadir):
fail_hard('%s does not exist', datadir)
if args.scale_constant < 1:
fail_hard('--scale-constant must be positive')
if args.torflow_bw_margin < 0:
fail_hard('toflow-bw-margin must be major than 0.')
if args.scale_sbws:
scaling_method = SBWS_SCALING
elif args.raw:
scaling_method = None
else:
# sbws will scale as torflow until we have a better algorithm for
# scaling (#XXX)
scaling_method = TORFLOW_SCALING
if args.secs_recent:
fresh_days = ceil(args.secs_recent / 24 / 60 / 60)
else:
fresh_days = conf.getint('general', 'data_period')
reset_bw_ipv4_changes = conf.getboolean('general', 'reset_bw_ipv4_changes')
reset_bw_ipv6_changes = conf.getboolean('general', 'reset_bw_ipv6_changes')
results = load_recent_results_in_datadir(
fresh_days,
datadir,
on_changed_ipv4=reset_bw_ipv4_changes,
on_changed_ipv6=reset_bw_ipv6_changes)
if len(results) < 1:
log.warning('No recent results, so not generating anything. (Have you '
'ran sbws scanner recently?)')
return
state_fpath = conf.getpath('paths', 'state_fname')
consensus_path = os.path.join(conf.getpath('tor', 'datadir'),
"cached-consensus")
# Accept None as scanner_country to be compatible with older versions.
scanner_country = conf['scanner'].get('country')
destinations_countries = destination.parse_destinations_countries(conf)
bw_file = V3BWFile.from_results(results,
scanner_country,
destinations_countries,
state_fpath,
args.scale_constant,
scaling_method,
torflow_cap=args.torflow_bw_margin,
round_digs=args.round_digs,
secs_recent=args.secs_recent,
secs_away=args.secs_away,
min_num=args.min_num,
consensus_path=consensus_path)
output = args.output or \
conf.getpath('paths', 'v3bw_fname').format(now_fname())
bw_file.write(output)
bw_file.info_stats
def main(args, conf):
'''
Main entry point in to the cleanup command.
:param argparse.Namespace args: command line arguments
:param configparser.ConfigParser conf: parsed config files
'''
if not is_initted(args.directory):
fail_hard('Sbws isn\'t initialized. Try sbws init')
datadir = conf['paths']['datadir']
if not os.path.isdir(datadir):
fail_hard('%s does not exist', datadir)
fresh_days = conf.getint('general', 'data_period')
stale_days = conf.getint('cleanup', 'stale_days')
rotten_days = conf.getint('cleanup', 'rotten_days')
if stale_days - 2 < fresh_days:
fail_hard(
'For safetly, cleanup/stale_days (%d) must be at least 2 '
'days larger than general/data_period (%d)', stale_days,
fresh_days)
if rotten_days < stale_days:
fail_hard(
'cleanup/rotten_days (%d) must be the same or larger than '
'cleanup/stale_days (%d)', rotten_days, stale_days)
if stale_days / 2 < fresh_days:
log.warning(
'cleanup/stale_days (%d) is less than twice '
'general/data_period (%d). For ease of parsing older results '
'if necessary, it is recommended to make stale_days at least '
'twice the data_period.', stale_days, fresh_days)
_remove_rotten_files(datadir, rotten_days, dry_run=args.dry_run)
_compress_stale_files(datadir, stale_days, dry_run=args.dry_run)
def main(args, conf):
os.makedirs(conf.getpath('paths', 'v3bw_dname'), exist_ok=True)
datadir = conf.getpath('paths', 'datadir')
if not os.path.isdir(datadir):
fail_hard('%s does not exist', datadir)
if args.scale_constant < 1:
fail_hard('--scale-constant must be positive')
if args.torflow_bw_margin < 0:
fail_hard('toflow-bw-margin must be major than 0.')
if args.scale_sbws:
scaling_method = SBWS_SCALING
elif args.raw:
scaling_method = None
else:
scaling_method = TORFLOW_SCALING
fresh_days = conf.getint('general', 'data_period')
reset_bw_ipv4_changes = conf.getboolean('general', 'reset_bw_ipv4_changes')
reset_bw_ipv6_changes = conf.getboolean('general', 'reset_bw_ipv6_changes')
results = load_recent_results_in_datadir(
fresh_days,
datadir,
success_only=True,
on_changed_ipv4=reset_bw_ipv4_changes,
on_changed_ipv6=reset_bw_ipv6_changes)
if len(results) < 1:
log.warning('No recent results, so not generating anything. (Have you '
'ran sbws scanner recently?)')
return
state_fpath = conf.getpath('paths', 'state_fname')
bw_file = V3BWFile.from_results(results,
state_fpath,
args.scale_constant,
scaling_method,
torflow_cap=args.torflow_bw_margin,
torflow_round_digs=args.torflow_round_digs,
secs_recent=args.secs_recent,
secs_away=args.secs_away,
min_num=args.min_num)
output = args.output or \
conf.getpath('paths', 'v3bw_fname').format(now_fname())
bw_file.write(output)
bw_file.info_stats
def main(args, conf):
if not is_initted(args.directory):
fail_hard('Sbws isn\'t initialized. Try sbws init')
datadir = conf['paths']['datadir']
if not os.path.isdir(datadir):
fail_hard('%s does not exist', datadir)
if args.scale_constant < 1:
fail_hard('--scale-constant must be positive')
fresh_days = conf.getint('general', 'data_period')
results = load_recent_results_in_datadir(fresh_days,
datadir,
success_only=True)
if results:
# Using naive datetime object without timezone, assumed utc
# Not using .isoformat() since that does not include 'T'
earliest_bandwidth = datetime.utcfromtimestamp(
min([r.time for fp in results
for r in results[fp]])) \
.strftime(TIMESTAMP_DT_FRMT)
if len(results) < 1:
log.warning('No recent results, so not generating anything. (Have you '
'ran sbws scanner recently?)')
return
data_lines = [result_data_to_v3bw_line(results, fp) for fp in results]
data_lines = sorted(data_lines, key=lambda d: d.bw, reverse=True)
data_lines = scale_lines(args, data_lines)
generator_started = read_started_ts(conf)
if results:
header = V3BwHeader(earliest_bandwidth=earliest_bandwidth,
generator_started=generator_started)
else:
header = V3BwHeader(generator_started=generator_started)
log_stats(data_lines)
output = conf['paths']['v3bw_fname']
if args.output:
output = args.output
log.info('Writing v3bw file to %s', output)
with open(output, 'wt') as fd:
fd.write(str(header))
for line in data_lines:
fd.write('{}\n'.format(str(line)))
def main(args, conf):
if not is_initted(args.directory):
fail_hard('Sbws isn\'t initialized. Try sbws init')
if conf.getint('scanner', 'measurement_threads') < 1:
fail_hard('Number of measurement threads must be larger than 1')
min_dl = conf.getint('scanner', 'min_download_size')
max_dl = conf.getint('scanner', 'max_download_size')
if max_dl < min_dl:
fail_hard('Max download size %d cannot be smaller than min %d', max_dl,
min_dl)
os.makedirs(conf['paths']['datadir'], exist_ok=True)
try:
run_speedtest(args, conf)
except KeyboardInterrupt as e:
raise e
finally:
end_event.set()
def launch_tor(conf):
assert isinstance(conf, ConfigParser)
os.makedirs(conf.getpath('tor', 'datadir'), mode=0o700, exist_ok=True)
os.makedirs(conf.getpath('tor', 'log'), exist_ok=True)
os.makedirs(conf.getpath('tor', 'run_dpath'), mode=0o700, exist_ok=True)
# Bare minimum things, more or less
torrc = copy.deepcopy(TORRC_STARTING_POINT)
# Very important and/or common settings that we don't know until runtime
torrc.update({
'DataDirectory':
conf.getpath('tor', 'datadir'),
'PidFile':
conf.getpath('tor', 'pid'),
'ControlSocket':
conf.getpath('tor', 'control_socket'),
'Log': [
'NOTICE file {}'.format(
os.path.join(conf.getpath('tor', 'log'), 'notice.log')),
],
# Things needed to make circuits fail a little faster. We get the
# circuit_timeout as a string instead of an int on purpose: stem only
# accepts strings.
'LearnCircuitBuildTimeout':
'0',
'CircuitBuildTimeout':
conf['general']['circuit_timeout'],
})
# This block of code reads additional torrc lines from the user's
# config.ini so they can add arbitrary additional options.
#
# The user can't replace our options, only add to them. For example,
# there's no way to remove 'SocksPort auto' (if it is still in
# TORRC_STARTING_POINT). If you add a SocksPort in your config.ini, you'll
# open two socks ports.
#
# As an example, maybe the user hates their HDD and wants to fill it with
# debug logs, and wants to tell Tor to use only 1 CPU core.
#
# [tor]
# extra_lines =
# Log debug file /tmp/tor-debug.log
# NumCPUs 1
for line in conf['tor']['extra_lines'].split('\n'):
# Remove leading and trailing whitespace, if any
line = line.strip()
# Ignore blank lines
if len(line) < 1:
continue
# The way stem handles configuring Tor with a dictionary is the first
# word is a key and the remaining words are the value.
kv = line.split(None, 1)
if len(kv) < 2:
fail_hard(
'All torrc lines must have 2 or more words. "%s" has '
'fewer', line)
key, value = kv
log.info('Adding "%s %s" to torrc with which we are launching Tor',
key, value)
# It's really easy to add to the torrc if the key doesn't exist
if key not in torrc:
torrc.update({key: value})
# But if it does, we have to make a list of values. For example, say
# the user wants to add a SocksPort and we already have
# 'SocksPort auto' in the torrc. We'll go from
# torrc['SocksPort'] == 'auto'
# to
# torrc['SocksPort'] == ['auto', '9050']
else:
existing_val = torrc[key]
if isinstance(existing_val, str):
torrc.update({key: [existing_val, value]})
else:
assert isinstance(existing_val, list)
existing_val.append(value)
torrc.update({key: existing_val})
# Finally launch Tor
try:
stem.process.launch_tor_with_config(torrc,
init_msg_handler=log.debug,
take_ownership=True)
except Exception as e:
fail_hard('Error trying to launch tor: %s', e)
# And return a controller to it
cont = _init_controller_socket(conf.getpath('tor', 'control_socket'))
# Because we build things by hand and can't set these before Tor bootstraps
try:
cont.set_conf('__DisablePredictedCircuits', '1')
cont.set_conf('__LeaveStreamsUnattached', '1')
except (ControllerError, InvalidArguments, InvalidRequest) as e:
log.exception(
"Error trying to launch tor: %s. "
"Maybe the tor directory is being used by other "
"sbws instance?", e)
exit(1)
log.info('Started and connected to Tor %s via %s', cont.get_version(),
conf.getpath('tor', 'control_socket'))
return cont
def run_speedtest(args, conf):
"""Initializes all the data and threads needed to measure the relays.
It launches or connect to Tor in a thread.
It initializes the list of relays seen in the Tor network.
It starts a thread to read the previous measurements and wait for new
measurements to write them to the disk.
It initializes a class that will be used to order the relays depending
on their measurements age.
It initializes the list of destinations that will be used for the
measurements.
It initializes the thread pool that will launch the measurement threads.
The pool starts 3 other threads that are not the measurement (worker)
threads.
Finally, it calls the function that will manage the measurement threads.
"""
global rd, pool, controller
controller, _ = stem_utils.init_controller(
path=conf.getpath('tor', 'control_socket'))
if not controller:
controller = stem_utils.launch_tor(conf)
else:
log.warning(
'Is sbws already running? '
'We found an existing Tor process at %s. We are not going to '
'launch Tor, nor are we going to try to configure it to behave '
'like we expect. This might work okay, but it also might not. '
'If you experience problems, you should try letting sbws launch '
'Tor for itself. The ability to use an already running Tor only '
'exists for sbws developers. It is expected to be broken and may '
'even lead to messed up results.',
conf.getpath('tor', 'control_socket'))
time.sleep(15)
# When there will be a refactor where conf is global, this can be removed
# from here.
state = State(conf.getpath('paths', 'state_fname'))
# XXX: tech-debt: create new function to obtain the controller and to
# write the state, so that a unit test to check the state tor version can
# be created
# Store tor version whenever the scanner starts.
state['tor_version'] = str(controller.get_version())
# Call only once to initialize http_headers
settings.init_http_headers(conf.get('scanner', 'nickname'), state['uuid'],
state['tor_version'])
# To do not have to pass args and conf to RelayList, pass an extra
# argument with the data_period
measurements_period = conf.getint('general', 'data_period')
rl = RelayList(args, conf, controller, measurements_period, state)
cb = CB(args, conf, controller, rl)
rd = ResultDump(args, conf)
rp = RelayPrioritizer(args, conf, rl, rd)
destinations, error_msg = DestinationList.from_config(
conf, cb, rl, controller)
if not destinations:
fail_hard(error_msg)
max_pending_results = conf.getint('scanner', 'measurement_threads')
pool = Pool(max_pending_results)
try:
main_loop(args, conf, controller, rl, cb, rd, rp, destinations, pool)
except KeyboardInterrupt:
log.info("Interrupted by the user.")
stop_threads(signal.SIGINT, None)
# Any exception not catched at this point would make the scanner stall.
# Log it and exit gracefully.
except Exception as e:
log.critical(FILLUP_TICKET_MSG)
log.exception(e)
stop_threads(signal.SIGTERM, None, 1)
def _check_validity_periods_v3bw(compress_after_days, delete_after_days):
if 1 <= compress_after_days and compress_after_days < delete_after_days:
return True
fail_hard("v3bw files should only be compressed after 1 day and deleted "
"after a bigger number of days.")
