def test_set_under_min_report(mock_consensus, conf, datadir):
# The number of relays (1) is the same as the ones in the consensus,
# therefore there is no any relay excluded and under_min_report is not set.
mock_consensus.return_value = 1
state_fpath = conf['paths']['state_fpath']
results = load_result_file(str(datadir.join("results.txt")))
v3bwfile = V3BWFile.from_results(results, '', '', state_fpath)
bwl = v3bwfile.bw_lines[0]
assert not hasattr(bwl, "vote")
assert not hasattr(bwl, "under_min_report")
assert bwl.bw != 1
# The number of relays is the same as the ones in the consensus,
# but after filtering there's no any, under_min_report is set to 1
# and unmeasured was also set to 1.
# After filtering the relay is excluded because there's only 1 success
# result and it should have at least 2 (min_num)
v3bwfile = V3BWFile.from_results(results, '', '', state_fpath, min_num=2)
bwl = v3bwfile.bw_lines[0]
assert bwl.vote == 0
assert bwl.under_min_report == 1
assert bwl.unmeasured == 1
assert bwl.bw == 1
# The number of relays after scaling is than the 60% in the network,
# therefore the relays are excluded and under_min_report is set to 1.
mock_consensus.return_value = 3
v3bwfile = V3BWFile.from_results(results, '', '', state_fpath)
bwl = v3bwfile.bw_lines[0]
assert bwl.vote == 0
assert bwl.under_min_report == 1
assert bwl.bw != 1
def test_torflow_scale(datadir):
results = load_result_file(str(datadir.join("results.txt")))
v3bwfile = V3BWFile.from_results(results, scaling_method=TORFLOW_SCALING)
assert v3bwfile.bw_lines[0].bw == 1000
v3bwfile = V3BWFile.from_results(results,
scaling_method=TORFLOW_SCALING,
torflow_cap=0.0001)
assert v3bwfile.bw_lines[0].bw == 1000
v3bwfile = V3BWFile.from_results(results,
scaling_method=TORFLOW_SCALING,
torflow_cap=1,
torflow_round_digs=0)
assert v3bwfile.bw_lines[0].bw == 524
def test_torflow_scale_no_consensus_bw(datadir, conf, caplog):
state_fpath = conf['paths']['state_fpath']
results = load_result_file(str(
datadir.join("results_no_consensus_bw.txt")))
caplog.set_level(logging.DEBUG)
v3bwfile = V3BWFile.from_results(results, '', '', state_fpath)
assert v3bwfile.bw_lines[0].bw == 26
def test_from_arg_results_write_read(datadir, tmpdir, conf, args):
results = load_result_file(str(datadir.join("results.txt")))
v3bwfile = V3BWFile.from_results(results)
output = os.path.join(args.output, now_fname())
v3bwfile.write(output)
with open(output) as fd:
v3bw = fd.read()
assert v3bw == str(v3bwfile)
def main(args, conf):
os.makedirs(conf.getpath('paths', 'v3bw_dname'), exist_ok=True)
datadir = conf.getpath('paths', 'datadir')
if not os.path.isdir(datadir):
fail_hard('%s does not exist', datadir)
if args.scale_constant < 1:
fail_hard('--scale-constant must be positive')
if args.torflow_bw_margin < 0:
fail_hard('toflow-bw-margin must be major than 0.')
if args.scale_sbws:
scaling_method = SBWS_SCALING
elif args.raw:
scaling_method = None
else:
# sbws will scale as torflow until we have a better algorithm for
# scaling (#XXX)
scaling_method = TORFLOW_SCALING
if args.secs_recent:
fresh_days = ceil(args.secs_recent / 24 / 60 / 60)
else:
fresh_days = conf.getint('general', 'data_period')
reset_bw_ipv4_changes = conf.getboolean('general', 'reset_bw_ipv4_changes')
reset_bw_ipv6_changes = conf.getboolean('general', 'reset_bw_ipv6_changes')
results = load_recent_results_in_datadir(
fresh_days,
datadir,
on_changed_ipv4=reset_bw_ipv4_changes,
on_changed_ipv6=reset_bw_ipv6_changes)
if len(results) < 1:
log.warning('No recent results, so not generating anything. (Have you '
'ran sbws scanner recently?)')
return
state_fpath = conf.getpath('paths', 'state_fname')
consensus_path = os.path.join(conf.getpath('tor', 'datadir'),
"cached-consensus")
# Accept None as scanner_country to be compatible with older versions.
scanner_country = conf['scanner'].get('country')
destinations_countries = destination.parse_destinations_countries(conf)
bw_file = V3BWFile.from_results(results,
scanner_country,
destinations_countries,
state_fpath,
args.scale_constant,
scaling_method,
torflow_cap=args.torflow_bw_margin,
round_digs=args.round_digs,
secs_recent=args.secs_recent,
secs_away=args.secs_away,
min_num=args.min_num,
consensus_path=consensus_path)
output = args.output or \
conf.getpath('paths', 'v3bw_fname').format(now_fname())
bw_file.write(output)
bw_file.info_stats
def test_torflow_scale(mock_consensus, datadir, tmpdir, conf):
mock_consensus.return_value = 1
# state_fpath = str(tmpdir.join('.sbws', 'state.dat'))
state_fpath = conf['paths']['state_fpath']
results = load_result_file(str(datadir.join("results.txt")))
# Since v1.1.0, it'll write bw=1 if the minimum percent of measured relays
# wasn't reached. Therefore mock the consensus number.
# Because the consensus number is mocked, it'll try to read the sate path.
# Obtain it from conf, so that the root directory exists.
v3bwfile = V3BWFile.from_results(results,
'',
'',
state_fpath,
scaling_method=TORFLOW_SCALING,
round_digs=TORFLOW_ROUND_DIG)
assert v3bwfile.bw_lines[0].bw == 6
v3bwfile = V3BWFile.from_results(results,
'',
'',
state_fpath,
scaling_method=TORFLOW_SCALING,
torflow_cap=0.0001,
round_digs=TORFLOW_ROUND_DIG)
# Wrong because it should be rounded after clipping
assert v3bwfile.bw_lines[0].bw == 1
v3bwfile = V3BWFile.from_results(results,
'',
'',
state_fpath,
scaling_method=TORFLOW_SCALING,
torflow_cap=1,
round_digs=TORFLOW_ROUND_DIG)
assert v3bwfile.bw_lines[0].bw == 123
v3bwfile = V3BWFile.from_results(results,
'',
'',
state_fpath,
scaling_method=TORFLOW_SCALING,
torflow_cap=1,
round_digs=PROP276_ROUND_DIG)
assert v3bwfile.bw_lines[0].bw == 120
def test_torflow_scale(datadir):
results = load_result_file(str(datadir.join("results.txt")))
v3bwfile = V3BWFile.from_results(results,
scaling_method=TORFLOW_SCALING,
round_digs=TORFLOW_ROUND_DIG)
assert v3bwfile.bw_lines[0].bw == 123
v3bwfile = V3BWFile.from_results(results,
scaling_method=TORFLOW_SCALING,
torflow_cap=0.0001,
round_digs=TORFLOW_ROUND_DIG)
assert v3bwfile.bw_lines[0].bw == 123
v3bwfile = V3BWFile.from_results(results,
scaling_method=TORFLOW_SCALING,
torflow_cap=1,
round_digs=TORFLOW_ROUND_DIG)
assert v3bwfile.bw_lines[0].bw == 123
v3bwfile = V3BWFile.from_results(results,
scaling_method=TORFLOW_SCALING,
torflow_cap=1,
round_digs=PROP276_ROUND_DIG)
assert v3bwfile.bw_lines[0].bw == 120
def test_from_results_read(datadir, tmpdir, conf, args):
results = load_result_file(str(datadir.join("results.txt")))
expected_header = V3BWHeader(timestamp_l,
earliest_bandwidth=earliest_bandwidth,
latest_bandwidth=latest_bandwidth)
expected_bwls = [V3BWLine.from_results(results[fp]) for fp in results]
# bw store now B, not KB
expected_bwls[0].bw = round(expected_bwls[0].bw / 1000)
expected_f = V3BWFile(expected_header, expected_bwls)
# This way is going to convert bw to KB
v3bwfile = V3BWFile.from_results(results)
assert str(expected_f)[1:] == str(v3bwfile)[1:]
output = os.path.join(args.output, now_fname())
v3bwfile.write(output)
def main(args, conf):
os.makedirs(conf.getpath('paths', 'v3bw_dname'), exist_ok=True)
datadir = conf.getpath('paths', 'datadir')
if not os.path.isdir(datadir):
fail_hard('%s does not exist', datadir)
if args.scale_constant < 1:
fail_hard('--scale-constant must be positive')
if args.torflow_bw_margin < 0:
fail_hard('toflow-bw-margin must be major than 0.')
if args.scale_sbws:
scaling_method = SBWS_SCALING
elif args.raw:
scaling_method = None
else:
scaling_method = TORFLOW_SCALING
fresh_days = conf.getint('general', 'data_period')
reset_bw_ipv4_changes = conf.getboolean('general', 'reset_bw_ipv4_changes')
reset_bw_ipv6_changes = conf.getboolean('general', 'reset_bw_ipv6_changes')
results = load_recent_results_in_datadir(
fresh_days,
datadir,
success_only=True,
on_changed_ipv4=reset_bw_ipv4_changes,
on_changed_ipv6=reset_bw_ipv6_changes)
if len(results) < 1:
log.warning('No recent results, so not generating anything. (Have you '
'ran sbws scanner recently?)')
return
state_fpath = conf.getpath('paths', 'state_fname')
bw_file = V3BWFile.from_results(results,
state_fpath,
args.scale_constant,
scaling_method,
torflow_cap=args.torflow_bw_margin,
torflow_round_digs=args.torflow_round_digs,
secs_recent=args.secs_recent,
secs_away=args.secs_away,
min_num=args.min_num)
output = args.output or \
conf.getpath('paths', 'v3bw_fname').format(now_fname())
bw_file.write(output)
bw_file.info_stats
def test_from_results_read(datadir, tmpdir, conf, args):
results = load_result_file(str(datadir.join("results.txt")))
expected_header = V3BWHeader(timestamp_l,
earliest_bandwidth=earliest_bandwidth,
latest_bandwidth=latest_bandwidth)
exclusion_dict = dict([
(k, 0) for k in BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED
])
expected_header.add_relays_excluded_counters(exclusion_dict)
raw_bwls = [V3BWLine.from_results(results[fp])[0] for fp in results]
# Scale BWLines using torflow method, since it's the default and BWLines
# bandwidth is the raw bandwidth.
expected_bwls = V3BWFile.bw_torflow_scale(raw_bwls)
expected_f = V3BWFile(expected_header, expected_bwls)
# This way is going to convert bw to KB
v3bwfile = V3BWFile.from_results(results)
assert str(expected_f)[1:] == str(v3bwfile)[1:]
output = os.path.join(args.output, now_fname())
v3bwfile.write(output)
def test_sbws_scale(datadir):
results = load_result_file(str(datadir.join("results.txt")))
v3bwfile = V3BWFile.from_results(results, scaling_method=SBWS_SCALING)
assert v3bwfile.bw_lines[0].bw == 8
def test_from_arg_results_write(datadir, tmpdir, conf, args):
results = load_result_file(str(datadir.join("results.txt")))
v3bwfile = V3BWFile.from_results(results)
output = os.path.join(args.output, now_fname())
v3bwfile.write(output)
assert os.path.isfile(output)
