def test_update_progress(datadir, tmpdir):
bw_lines_raw = []
number_consensus_relays = 6
state = {}
header = V3BWHeader(str(now_unixts()))
results = load_result_file(str(datadir.join("results_away.txt")))
for fp, values in results.items():
# log.debug("Relay fp %s", fp)
line = V3BWLine.from_results(values)
if line is not None:
bw_lines_raw.append(line)
bwfile = V3BWFile(header, [])
bwfile.update_progress(len(bw_lines_raw), header, number_consensus_relays,
state)
assert header.percent_eligible_relays == '50'
assert state.get('min_perc_reached') is None
# Test that the headers are also included when there are enough eligible
# relays
number_consensus_relays = 3
header = V3BWHeader(str(now_unixts()))
bwfile.update_progress(len(bw_lines_raw), header, number_consensus_relays,
state)
assert state.get('min_perc_reached') == now_isodt_str()
assert header.minimum_number_eligible_relays == '2'
assert header.minimum_percent_eligible_relays == str(MIN_REPORT)
assert header.number_consensus_relays == '3'
assert header.number_eligible_relays == '3'
assert header.percent_eligible_relays == '100'
def test_torflow_scale_no_consensus_bw(datadir, conf, caplog):
state_fpath = conf['paths']['state_fpath']
results = load_result_file(str(
datadir.join("results_no_consensus_bw.txt")))
caplog.set_level(logging.DEBUG)
v3bwfile = V3BWFile.from_results(results, '', '', state_fpath)
assert v3bwfile.bw_lines[0].bw == 26
def test_set_under_min_report(mock_consensus, conf, datadir):
# The number of relays (1) is the same as the ones in the consensus,
# therefore there is no any relay excluded and under_min_report is not set.
mock_consensus.return_value = 1
state_fpath = conf['paths']['state_fpath']
results = load_result_file(str(datadir.join("results.txt")))
v3bwfile = V3BWFile.from_results(results, '', '', state_fpath)
bwl = v3bwfile.bw_lines[0]
assert not hasattr(bwl, "vote")
assert not hasattr(bwl, "under_min_report")
assert bwl.bw != 1
# The number of relays is the same as the ones in the consensus,
# but after filtering there's no any, under_min_report is set to 1
# and unmeasured was also set to 1.
# After filtering the relay is excluded because there's only 1 success
# result and it should have at least 2 (min_num)
v3bwfile = V3BWFile.from_results(results, '', '', state_fpath, min_num=2)
bwl = v3bwfile.bw_lines[0]
assert bwl.vote == 0
assert bwl.under_min_report == 1
assert bwl.unmeasured == 1
assert bwl.bw == 1
# The number of relays after scaling is than the 60% in the network,
# therefore the relays are excluded and under_min_report is set to 1.
mock_consensus.return_value = 3
v3bwfile = V3BWFile.from_results(results, '', '', state_fpath)
bwl = v3bwfile.bw_lines[0]
assert bwl.vote == 0
assert bwl.under_min_report == 1
assert bwl.bw != 1
def test_from_arg_results_write_read(datadir, tmpdir, conf, args):
results = load_result_file(str(datadir.join("results.txt")))
v3bwfile = V3BWFile.from_results(results)
output = os.path.join(args.output, now_fname())
v3bwfile.write(output)
with open(output) as fd:
v3bw = fd.read()
assert v3bw == str(v3bwfile)
def test_recent_measurement_attempt_count(root_data_path, datadir):
state_fpath = os.path.join(root_data_path, '.sbws/state.dat')
assert 15 == V3BWHeader.recent_measurement_attempt_count_from_file(
state_fpath)
# `results` does not matter here, using them to not have an empty list.
results = load_result_file(str(datadir.join("results.txt")))
header = V3BWHeader.from_results(results, '', '', state_fpath)
assert "15" == header.recent_measurement_attempt_count
def test_recent_priority_relay_count(root_data_path, datadir):
# This state has recent_priority_relay_count
state_fpath = os.path.join(root_data_path, '.sbws/state.dat')
assert 15 == V3BWHeader.recent_priority_relay_count_from_file(state_fpath)
# `results` does not matter here, using them to don't have an empty list.
results = load_result_file(str(datadir.join("results.txt")))
header = V3BWHeader.from_results(results, '', '', state_fpath)
assert "15" == header.recent_priority_relay_count
def test_generator_started(root_data_path, datadir):
state_fpath = os.path.join(root_data_path, '.sbws/state.dat')
# The method is correct
assert "2020-02-29T10:00:00" == V3BWHeader.generator_started_from_file(
state_fpath)
# `results` does not matter here, using them to not have an empty list.
results = load_result_file(str(datadir.join("results.txt")))
header = V3BWHeader.from_results(results, '', '', state_fpath)
# And the header is correct
assert "2020-02-29T10:00:00" == header.generator_started
def test_results_away_each_other(datadir):
min_num = 2
secs_away = 86400 # 1d
results = load_result_file(str(datadir.join("results_away.txt")))
# A has 4 results, 3 are success, 2 are 1 day away, 1 is 12h away
values = results["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"]
# There is one result excluded, but the relay is not excluded
bwl, reason = V3BWLine.from_results(values, secs_away=secs_away, min_num=2)
assert bwl.relay_recent_measurements_excluded_error_count == 1
assert reason is None
assert not hasattr(bwl, "vote")
assert not hasattr(bwl, "unmeasured")
success_results = [r for r in values if isinstance(r, ResultSuccess)]
assert len(success_results) >= min_num
results_away = V3BWLine.results_away_each_other(success_results, secs_away)
assert len(results_away) == 3
# B has 2 results, 12h away from each other
values = results["BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"]
# Two measurements are excluded and there were only 2,
# the relay is excluded
bwl, reason = V3BWLine.from_results(values, secs_away=secs_away, min_num=2)
assert bwl.relay_recent_measurements_excluded_near_count == 2
assert reason == 'recent_measurements_excluded_near_count'
assert bwl.vote == 0
assert bwl.unmeasured == 1
success_results = [r for r in values if isinstance(r, ResultSuccess)]
assert len(success_results) >= min_num
results_away = V3BWLine.results_away_each_other(success_results, secs_away)
assert not results_away
secs_away = 43200 # 12h
values = results["BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"]
success_results = [r for r in values if isinstance(r, ResultSuccess)]
assert len(success_results) >= min_num
results_away = V3BWLine.results_away_each_other(success_results, secs_away)
assert len(results_away) == 2
# C has 1 result
values = results["CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC"]
# There is only 1 result, the relay is excluded
bwl, reason = V3BWLine.from_results(values, min_num=2)
assert bwl.relay_recent_measurements_excluded_few_count == 1
assert reason == 'recent_measurements_excluded_few_count'
assert bwl.vote == 0
assert bwl.unmeasured == 1
success_results = [r for r in values if isinstance(r, ResultSuccess)]
assert len(success_results) < min_num
def test_torflow_scale(datadir):
results = load_result_file(str(datadir.join("results.txt")))
v3bwfile = V3BWFile.from_results(results, scaling_method=TORFLOW_SCALING)
assert v3bwfile.bw_lines[0].bw == 1000
v3bwfile = V3BWFile.from_results(results,
scaling_method=TORFLOW_SCALING,
torflow_cap=0.0001)
assert v3bwfile.bw_lines[0].bw == 1000
v3bwfile = V3BWFile.from_results(results,
scaling_method=TORFLOW_SCALING,
torflow_cap=1,
torflow_round_digs=0)
assert v3bwfile.bw_lines[0].bw == 524
def test_from_results_read(datadir, tmpdir, conf, args):
results = load_result_file(str(datadir.join("results.txt")))
expected_header = V3BWHeader(timestamp_l,
earliest_bandwidth=earliest_bandwidth,
latest_bandwidth=latest_bandwidth)
expected_bwls = [V3BWLine.from_results(results[fp]) for fp in results]
# bw store now B, not KB
expected_bwls[0].bw = round(expected_bwls[0].bw / 1000)
expected_f = V3BWFile(expected_header, expected_bwls)
# This way is going to convert bw to KB
v3bwfile = V3BWFile.from_results(results)
assert str(expected_f)[1:] == str(v3bwfile)[1:]
output = os.path.join(args.output, now_fname())
v3bwfile.write(output)
def test_load(datadir):
results = load_result_file(str(datadir.join("results.txt")))
results = [v for values in results.values() for v in values]
r1 = results[1]
assert isinstance(r1, ResultSuccess)
assert isinstance(r1.relay_recent_measurement_attempt[0],
datetime.datetime)
assert 2 == len(r1.relay_recent_measurement_attempt)
assert 3 == len(r1.relay_recent_priority_list)
assert 3 == len(r1.relay_in_recent_consensus)
r2 = results[2]
assert isinstance(r2, ResultErrorStream)
assert isinstance(r2.relay_recent_measurement_attempt[0],
datetime.datetime)
assert 2 == len(r2.relay_recent_measurement_attempt)
assert 3 == len(r2.relay_recent_priority_list)
assert 3 == len(r2.relay_in_recent_consensus)
def test_from_results_read(datadir, tmpdir, conf, args):
results = load_result_file(str(datadir.join("results.txt")))
expected_header = V3BWHeader(timestamp_l,
earliest_bandwidth=earliest_bandwidth,
latest_bandwidth=latest_bandwidth)
exclusion_dict = dict([
(k, 0) for k in BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED
])
expected_header.add_relays_excluded_counters(exclusion_dict)
raw_bwls = [V3BWLine.from_results(results[fp])[0] for fp in results]
# Scale BWLines using torflow method, since it's the default and BWLines
# bandwidth is the raw bandwidth.
expected_bwls = V3BWFile.bw_torflow_scale(raw_bwls)
expected_f = V3BWFile(expected_header, expected_bwls)
# This way is going to convert bw to KB
v3bwfile = V3BWFile.from_results(results)
assert str(expected_f)[1:] == str(v3bwfile)[1:]
output = os.path.join(args.output, now_fname())
v3bwfile.write(output)
def test_torflow_scale(mock_consensus, datadir, tmpdir, conf):
mock_consensus.return_value = 1
# state_fpath = str(tmpdir.join('.sbws', 'state.dat'))
state_fpath = conf['paths']['state_fpath']
results = load_result_file(str(datadir.join("results.txt")))
# Since v1.1.0, it'll write bw=1 if the minimum percent of measured relays
# wasn't reached. Therefore mock the consensus number.
# Because the consensus number is mocked, it'll try to read the sate path.
# Obtain it from conf, so that the root directory exists.
v3bwfile = V3BWFile.from_results(results,
'',
'',
state_fpath,
scaling_method=TORFLOW_SCALING,
round_digs=TORFLOW_ROUND_DIG)
assert v3bwfile.bw_lines[0].bw == 6
v3bwfile = V3BWFile.from_results(results,
'',
'',
state_fpath,
scaling_method=TORFLOW_SCALING,
torflow_cap=0.0001,
round_digs=TORFLOW_ROUND_DIG)
# Wrong because it should be rounded after clipping
assert v3bwfile.bw_lines[0].bw == 1
v3bwfile = V3BWFile.from_results(results,
'',
'',
state_fpath,
scaling_method=TORFLOW_SCALING,
torflow_cap=1,
round_digs=TORFLOW_ROUND_DIG)
assert v3bwfile.bw_lines[0].bw == 123
v3bwfile = V3BWFile.from_results(results,
'',
'',
state_fpath,
scaling_method=TORFLOW_SCALING,
torflow_cap=1,
round_digs=PROP276_ROUND_DIG)
assert v3bwfile.bw_lines[0].bw == 120
def test_torflow_scale(datadir):
results = load_result_file(str(datadir.join("results.txt")))
v3bwfile = V3BWFile.from_results(results,
scaling_method=TORFLOW_SCALING,
round_digs=TORFLOW_ROUND_DIG)
assert v3bwfile.bw_lines[0].bw == 123
v3bwfile = V3BWFile.from_results(results,
scaling_method=TORFLOW_SCALING,
torflow_cap=0.0001,
round_digs=TORFLOW_ROUND_DIG)
assert v3bwfile.bw_lines[0].bw == 123
v3bwfile = V3BWFile.from_results(results,
scaling_method=TORFLOW_SCALING,
torflow_cap=1,
round_digs=TORFLOW_ROUND_DIG)
assert v3bwfile.bw_lines[0].bw == 123
v3bwfile = V3BWFile.from_results(results,
scaling_method=TORFLOW_SCALING,
torflow_cap=1,
round_digs=PROP276_ROUND_DIG)
assert v3bwfile.bw_lines[0].bw == 120
def test_bw_filt_from_results(root_data_path):
results_file = os.path.join(root_data_path, ".sbws", "datadir",
"2019-03-25.txt")
results = load_result_file(results_file)
bw_filts = {}
for fp, values in results.items():
success_results = [r for r in values if isinstance(r, ResultSuccess)]
if success_results:
bw_measurements = scaling.bw_measurements_from_results(
success_results)
mu = round(mean(bw_measurements))
muf = scaling.bw_filt(bw_measurements)
bw_filts[fp] = (mu, muf)
for fp, values in bw_filts.items():
assert bw_filts[fp][0] <= bw_filts[fp][1]
assert 5526756 == bw_filts['117A456C911114076BEB4E757AC48B16CC0CCC5F'][0]
assert 5643086 == bw_filts['117A456C911114076BEB4E757AC48B16CC0CCC5F'][1]
assert 5664965 == bw_filts['693F73187624BE760AAD2A12C5ED89DB1DE044F5'][0]
assert 5774274 == bw_filts['693F73187624BE760AAD2A12C5ED89DB1DE044F5'][1]
assert 5508279 == bw_filts['270A861ABED22EC2B625198BCCD7B2B9DBFFC93C'][0]
assert 5583737 == bw_filts['270A861ABED22EC2B625198BCCD7B2B9DBFFC93C'][1]
assert 5379911 == bw_filts['E894C65997F8EC96558B554176EEEA39C6A43EF6'][0]
assert 5485088 == bw_filts['E894C65997F8EC96558B554176EEEA39C6A43EF6'][1]
def test_measured_progress_stats(datadir):
number_consensus_relays = 3
bw_lines_raw = []
statsd_exp = {
'percent_eligible_relays': 100,
'minimum_percent_eligible_relays': 60,
'number_consensus_relays': 3,
'minimum_number_eligible_relays': 2,
'number_eligible_relays': 3
}
min_perc_reached_before = None
results = load_result_file(str(datadir.join("results_away.txt")))
for fp, values in results.items():
# log.debug("Relay fp %s", fp)
line, _ = V3BWLine.from_results(values)
if line is not None:
bw_lines_raw.append(line)
assert len(bw_lines_raw) == 3
bw_lines = V3BWFile.bw_torflow_scale(bw_lines_raw)
assert len(bw_lines) == 3
statsd, success = V3BWFile.measured_progress_stats(
len(bw_lines), number_consensus_relays, min_perc_reached_before)
assert success
assert statsd == statsd_exp
number_consensus_relays = 6
statsd, success = V3BWFile.measured_progress_stats(
len(bw_lines), number_consensus_relays, min_perc_reached_before)
assert not success
statsd_exp = {
'percent_eligible_relays': 50,
'minimum_percent_eligible_relays': 60,
'number_consensus_relays': 6,
'minimum_number_eligible_relays': 4,
'number_eligible_relays': 3
}
assert statsd_exp == statsd
def test_relay_in_recent_consensus_count(root_data_path, datadir):
results = load_result_file(str(datadir.join("results.txt")))
for fp, values in results.items():
line = V3BWLine.from_results(values)
assert "3" == line[0].relay_in_recent_consensus_count
def test_from_arg_results_write(datadir, tmpdir, conf, args):
results = load_result_file(str(datadir.join("results.txt")))
v3bwfile = V3BWFile.from_results(results)
output = os.path.join(args.output, now_fname())
v3bwfile.write(output)
assert os.path.isfile(output)
def test_sbws_scale(datadir):
results = load_result_file(str(datadir.join("results.txt")))
v3bwfile = V3BWFile.from_results(results, scaling_method=SBWS_SCALING)
assert v3bwfile.bw_lines[0].bw == 8