def test_cleanup_delete_barely_rotten(time_mock, empty_dotsbws_datadir, caplog,
parser):
caplog.set_level(logging.DEBUG)
dotsbws = empty_dotsbws_datadir
args = parser.parse_args('-d {} --log-level DEBUG cleanup'.format(
dotsbws.name).split())
conf = get_config(args)
conf['general']['data_period'] = '1'
conf['cleanup']['stale_days'] = '5'
conf['cleanup']['rotten_days'] = '20'
now = 1443571200 # 1,443,571,200 is 30 Sep 2015 00:00:00 UTC
time_mock.side_effect = monotonic_time(start=now)
dd = os.path.join(dotsbws.name, 'datadir')
fname_rotten1 = os.path.join(dd, '2015-09-09.txt')
fname_rotten2 = os.path.join(dd, '2015-09-10.txt')
fname_leave = os.path.join(dd, '2015-09-11.txt')
touch_file(fname_rotten1)
touch_file(fname_rotten2)
touch_file(fname_leave)
sbws.core.cleanup.main(args, conf)
expected_fnames = [
fname_leave + '.gz',
os.path.join(dd, '.lockfile'),
]
existing_fnames = []
for root, dirs, files in os.walk(dd):
for fname in files:
existing_fnames.append(os.path.join(root, fname))
expected_fnames.sort()
existing_fnames.sort()
assert expected_fnames == existing_fnames
def test_generate_single_success_noscale(dotsbws_success_result, caplog,
parser, capfd):
dotsbws = dotsbws_success_result
args = parser.parse_args(
'-d {} --log-level DEBUG generate --output /dev/stdout'.format(
dotsbws.name).split())
conf = get_config(args)
sbws.core.generate.main(args, conf)
dd = conf['paths']['datadir']
# Here results is a dict
results = load_recent_results_in_datadir(1, dd, success_only=False)
assert isinstance(results, dict)
res_len = sum([len(results[fp]) for fp in results])
assert res_len == 1, 'There should be one result in the datadir'
# And here we change it to a list
results = [r for fp in results for r in results[fp]]
result = results[0]
assert isinstance(result, ResultSuccess), 'The one existing result '\
'should be a success'
captured = capfd.readouterr()
stdout_lines = captured.out.strip().split('\n')
assert len(stdout_lines) == 1 + NUM_LINES_HEADER
bw = round(
median(
[dl['amount'] / dl['duration'] / 1024 for dl in result.downloads]))
rtt = median([round(r * 1000) for r in result.rtts])
bw_line = 'node_id=${} bw={} nick={} rtt={} time={}'.format(
result.fingerprint, bw, result.nickname, rtt, round(result.time))
assert stdout_lines[NUM_LINES_HEADER] == bw_line
def test_stats_fresh_result(tmpdir, capsys, caplog):
'''
An initialized .sbws directory with a fresh error result should have some
boring stats and exit cleanly
'''
caplog.set_level(logging.DEBUG)
init_directory(tmpdir)
add_single_fresh_result(tmpdir)
p = create_parser()
args = p.parse_args(
'-d {} --log-level DEBUG stats --error-types'.format(tmpdir).split())
conf = get_config(args)
sbws.core.stats.main(args, conf)
captured = capsys.readouterr()
lines = captured.out.strip().split('\n')
needed_output_lines = [
'1 relays have recent results',
'Mean 0.00 successful measurements per relay',
'0 success results and 1 error results',
]
for needed_line in needed_output_lines:
assert needed_line in lines
lines = [l.getMessage() for l in caplog.records]
needed_log_lines = [
'Keeping 1/1 read lines from {}/{}/{}.txt'.format(
tmpdir, 'datadir',
datetime.utcfromtimestamp(time.time()).date()),
'Keeping 1/1 results after removing old ones',
]
for needed_line in needed_log_lines:
assert needed_line in lines
def dotsbws_success_result(empty_dotsbws_datadir):
'''
Creates an ~/.sbws with a single fresh ResultSuccess in it
'''
fp1 = 'A' * 40
fp2 = 'B' * 40
circ = [fp1, fp2]
nick = 'CowSayWhat'
relay_ip = '169.254.100.1'
server_ip = '169.254.100.2'
scanner_nick = 'SBWSscanner'
rtts = [4.242]
downloads = [{'duration': 4, 'amount': 40 * 1024}]
t = time.time()
relay = Result.Relay(fp1, nick, relay_ip)
result = ResultSuccess(rtts,
downloads,
relay,
circ,
server_ip,
scanner_nick,
t=t)
args = _PseudoArguments(directory=empty_dotsbws_datadir.name)
conf = get_config(args)
dd = conf['paths']['datadir']
write_result_to_datadir(result, dd)
return empty_dotsbws_datadir
def main():
parser = create_parser()
args = parser.parse_args()
conf = get_config(args)
_adjust_log_level(args, conf)
conf_valid, conf_errors = validate_config(conf)
if not conf_valid:
for e in conf_errors:
log.critical(e)
exit(1)
configure_logging(conf)
def_args = [args, conf]
def_kwargs = {}
known_commands = {
'cleanup': {'f': sbws.core.cleanup.main,
'a': def_args, 'kw': def_kwargs},
'scanner': {'f': sbws.core.scanner.main,
'a': def_args, 'kw': def_kwargs},
'generate': {'f': sbws.core.generate.main,
'a': def_args, 'kw': def_kwargs},
'init': {'f': sbws.core.init.main,
'a': def_args, 'kw': def_kwargs},
'stats': {'f': sbws.core.stats.main,
'a': def_args, 'kw': def_kwargs},
}
try:
if args.command not in known_commands:
parser.print_help()
else:
comm = known_commands[args.command]
exit(comm['f'](*comm['a'], **comm['kw']))
except KeyboardInterrupt:
print('')
def test_generate_empty_datadir(empty_dotsbws_datadir, caplog, parser):
dotsbws = empty_dotsbws_datadir
args = parser.parse_args(
'-d {} --log-level DEBUG generate --output /dev/stdout'.format(
dotsbws.name).split())
conf = get_config(args)
sbws.core.generate.main(args, conf)
assert 'No recent results' in caplog.records[-1].getMessage()
def empty_dotsbws_datadir(empty_dotsbws):
'''
Creates a ~/.sbws with nothing in it but a config.ini and an empty datadir
'''
args = _PseudoArguments(directory=empty_dotsbws.name)
conf = get_config(args)
dd = conf['paths']['datadir']
os.makedirs(dd, exist_ok=False)
return empty_dotsbws
def empty_dotsbws(parser):
'''
Creates a ~/.sbws with nothing in it but a config.ini
'''
d = TemporaryDirectory()
args = parser.parse_args('-d {} --log-level DEBUG init'.format(
d.name).split())
conf = get_config(args)
sbws.core.init.main(args, conf)
return d
def test_cleanup_nothing_to_do(empty_dotsbws_datadir, caplog, parser):
caplog.set_level(logging.DEBUG)
dotsbws = empty_dotsbws_datadir
args = parser.parse_args('-d {} --log-level DEBUG cleanup'.format(
dotsbws.name).split())
conf = get_config(args)
try:
sbws.core.cleanup.main(args, conf)
except Exception as e:
assert None, 'Nothing bad should have happened, but this did: {}'\
.format(e)
def test_cleanup_medium_stale(empty_dotsbws_datadir, caplog, parser):
dotsbws = empty_dotsbws_datadir
args = parser.parse_args('-d {} --log-level DEBUG cleanup'.format(
dotsbws.name).split())
conf = get_config(args)
conf['general']['data_period'] = '10'
conf['cleanup']['stale_days'] = '19'
conf['cleanup']['rotten_days'] = '50'
sbws.core.cleanup.main(args, conf)
assert 'cleanup/stale_days (19) is less than twice ' +\
'general/data_period (10).' in caplog.records[-1].getMessage()
def test_stats_stale_result(tmpdir, caplog):
'''
An initialized .sbws directory with no fresh results should say so and
exit cleanly
'''
init_directory(tmpdir)
add_single_stale_result(tmpdir)
p = create_parser()
args = p.parse_args('-d {} --log-level DEBUG stats'.format(tmpdir).split())
conf = get_config(args)
sbws.core.stats.main(args, conf)
assert 'No fresh results' == caplog.records[-1].getMessage()
def test_generate_two_relays_success_noscale(dotsbws_success_result_two_relays,
parser, capfd):
dotsbws = dotsbws_success_result_two_relays
args = parser.parse_args(
'-d {} --log-level DEBUG generate --output /dev/stdout'.format(
dotsbws.name).split())
conf = get_config(args)
sbws.core.generate.main(args, conf)
dd = conf['paths']['datadir']
# Here results is a dict
results = load_recent_results_in_datadir(1, dd, success_only=False)
assert isinstance(results, dict)
res_len = sum([len(results[fp]) for fp in results])
assert res_len == 4, 'There should be 4 results in the datadir'
# And here we change it to a list
results = [r for fp in results for r in results[fp]]
for result in results:
assert isinstance(result, ResultSuccess), 'All existing results '\
'should be a success'
captured = capfd.readouterr()
stdout_lines = captured.out.strip().split('\n')
assert len(stdout_lines) == 2 + NUM_LINES_HEADER
r1_results = [r for r in results if r.fingerprint == 'A' * 40]
r1_time = round(max([r.time for r in r1_results]))
r1_name = r1_results[0].nickname
r1_fingerprint = r1_results[0].fingerprint
r1_speeds = [
dl['amount'] / dl['duration'] / 1024 for r in r1_results
for dl in r.downloads
]
r1_speed = round(median(r1_speeds))
r1_rtt = round(
median([round(rtt * 1000) for r in r1_results for rtt in r.rtts]))
bw_line = 'node_id=${} bw={} nick={} rtt={} time={}'.format(
r1_fingerprint, r1_speed, r1_name, r1_rtt, r1_time)
assert stdout_lines[1 + NUM_LINES_HEADER] == bw_line
r2_results = [r for r in results if r.fingerprint == 'B' * 40]
r2_time = round(max([r.time for r in r2_results]))
r2_name = r2_results[0].nickname
r2_fingerprint = r2_results[0].fingerprint
r2_speeds = [
dl['amount'] / dl['duration'] / 1024 for r in r2_results
for dl in r.downloads
]
r2_speed = round(median(r2_speeds))
r2_rtt = round(
median([round(rtt * 1000) for r in r2_results for rtt in r.rtts]))
bw_line = 'node_id=${} bw={} nick={} rtt={} time={}'.format(
r2_fingerprint, r2_speed, r2_name, r2_rtt, r2_time)
assert stdout_lines[NUM_LINES_HEADER] == bw_line
def test_cleanup_no_datadir(empty_dotsbws, caplog, parser):
dotsbws = empty_dotsbws
args = parser.parse_args('-d {} --log-level DEBUG cleanup'.format(
dotsbws.name).split())
conf = get_config(args)
try:
sbws.core.cleanup.main(args, conf)
except SystemExit as e:
assert e.code == 1
else:
assert None, 'Should have failed'
dd = conf['paths']['datadir']
assert '{} does not exist'.format(dd) in caplog.records[-1].getMessage()
def test_cleanup_no_dotsbws(tmpdir, caplog, parser):
caplog.set_level(logging.DEBUG)
dotsbws = tmpdir
args = parser.parse_args(
'-d {} --log-level DEBUG cleanup'.format(dotsbws).split())
conf = get_config(args)
try:
sbws.core.cleanup.main(args, conf)
except SystemExit as e:
assert e.code == 1
else:
assert None, 'Should have failed'
assert 'Try sbws init' in caplog.records[-1].getMessage()
def test_generate_bad_scale_constant(empty_dotsbws_datadir, caplog, parser):
dotsbws = empty_dotsbws_datadir
args = parser.parse_args(
'-d {} --log-level DEBUG generate --scale-constant -1 '
'--output /dev/stdout'.format(dotsbws.name).split())
conf = get_config(args)
try:
sbws.core.generate.main(args, conf)
except SystemExit as e:
assert e.code == 1
else:
assert None, 'Should have failed'
assert '--scale-constant must be positive' in \
caplog.records[-1].getMessage()
def test_stats_uninitted(tmpdir, caplog):
'''
An un-initialized .sbws directory should fail hard and exit immediately
'''
p = create_parser()
args = p.parse_args('-d {} --log-level DEBUG stats'.format(tmpdir).split())
conf = get_config(args)
try:
sbws.core.stats.main(args, conf)
except SystemExit as e:
assert e.code == 1
else:
assert None, 'Should have failed'
assert 'Sbws isn\'t initialized. Try sbws init' == \
caplog.records[-1].getMessage()
def test_generate_single_error(dotsbws_error_result, caplog, parser):
caplog.set_level(logging.DEBUG)
dotsbws = dotsbws_error_result
args = parser.parse_args(
'-d {} --log-level DEBUG generate --output /dev/stdout'.format(
dotsbws.name).split())
conf = get_config(args)
sbws.core.generate.main(args, conf)
dd = conf['paths']['datadir']
for record in caplog.records:
if 'Keeping 0/1 read lines from {}'.format(dd) in record.getMessage():
break
else:
assert None, 'Unable to find log line indicating 0 success results '\
'in data file'
assert 'No recent results' in caplog.records[-1].getMessage()
def test_cleanup_small_rotten(empty_dotsbws_datadir, caplog, parser):
dotsbws = empty_dotsbws_datadir
args = parser.parse_args('-d {} --log-level DEBUG cleanup'.format(
dotsbws.name).split())
conf = get_config(args)
conf['general']['data_period'] = '1'
conf['cleanup']['stale_days'] = '5'
conf['cleanup']['rotten_days'] = '4'
try:
sbws.core.cleanup.main(args, conf)
except SystemExit as e:
assert e.code == 1
else:
assert None, 'Should have failed'
assert 'cleanup/rotten_days (4) must be the same or larger than ' +\
'cleanup/stale_days (5)' in caplog.records[-1].getMessage()
def test_cleanup_only_delete_rotten(time_mock, empty_dotsbws_datadir, caplog,
parser):
caplog.set_level(logging.DEBUG)
dotsbws = empty_dotsbws_datadir
args = parser.parse_args('-d {} --log-level DEBUG cleanup'.format(
dotsbws.name).split())
conf = get_config(args)
conf['general']['data_period'] = '1'
conf['cleanup']['stale_days'] = '10'
conf['cleanup']['rotten_days'] = str(365 * 2)
now = 1041379200 # 1,041,379,200 is 1 Jan 2003 00:00:00 UTC
time_mock.side_effect = monotonic_time(start=now)
j = os.path.join
dd = j(dotsbws.name, 'datadir')
sub_a = j(dd, 'a')
sub_b = j(dd, 'b')
sub_ab = j(dd, 'a', 'b')
for dname in [sub_a, sub_b, sub_ab]:
os.makedirs(dname, exist_ok=True)
should_delete_fnames = [
j(dd, '2000-01-01aaaa.txt'),
j(sub_a, '2000-10-01bbbb.txt'),
j(sub_b, '2000-10-10-cccc.txt'),
j(sub_a, '2000-10-10.dddd.txt'),
j(sub_a, '2000-10-11.eeee.txt.gz'),
j(dd, '2000-10-12.txt.gz'),
j(sub_ab, '2000-11-30.txt'),
]
should_ignore_fnames = [
j(dd, '2002-12-31.txt'), # too new, should be ignored
j(dd, '2003-01-01.txt'), # today, should be ignored
j(dd, '2003-02-10.txt'), # in the future, should be ignored
j(sub_b, '2000-10-10.nottxt'), # wrong ext, should be ignored
j(sub_a, '200j-10-10.txt'), # not YYYY-MM-DD*.txt, should be ignored
j(dd, '1999-1*-11.txt.gz'), # not YYYY-MM-DD*.txt.gz, should ignore
]
for fname in should_ignore_fnames + should_delete_fnames:
touch_file(fname)
sbws.core.cleanup.main(args, conf)
expected_fnames = should_ignore_fnames + [os.path.join(dd, '.lockfile')]
existing_fnames = []
for root, dirs, files in os.walk(dd):
for fname in files:
existing_fnames.append(os.path.join(root, fname))
expected_fnames.sort()
existing_fnames.sort()
assert expected_fnames == existing_fnames
def test_stats_initted(tmpdir, caplog):
'''
An initialized but rather empty .sbws directory should fail about missing
~/.sbws/datadir
'''
init_directory(tmpdir)
p = create_parser()
args = p.parse_args('-d {} --log-level DEBUG stats'.format(tmpdir).split())
conf = get_config(args)
try:
sbws.core.stats.main(args, conf)
except SystemExit as e:
assert e.code == 1
else:
assert None, 'Should have failed'
assert '{}/datadir does not exist'.format(tmpdir) == \
caplog.records[-1].getMessage()
def dotsbws_error_result(empty_dotsbws_datadir):
'''
Creates an ~/.sbws with a single fresh ResultError in it
'''
fp1 = 'A' * 40
fp2 = 'B' * 40
circ = [fp1, fp2]
nick = 'CowSayWhat'
relay_ip = '169.254.100.1'
server_ip = '169.254.100.2'
scanner_nick = 'SBWSscanner'
msg = 'UnitTest error message'
t = time.time()
relay = Result.Relay(fp1, nick, relay_ip)
result = ResultError(relay, circ, server_ip, scanner_nick, t=t, msg=msg)
args = _PseudoArguments(directory=empty_dotsbws_datadir.name)
conf = get_config(args)
dd = conf['paths']['datadir']
write_result_to_datadir(result, dd)
return empty_dotsbws_datadir
def test_stats_fresh_results(time_mock, tmpdir, capsys, caplog):
'''
An initialized .sbws directory with a fresh error and fresh success should
have some exciting stats and exit cleanly
'''
caplog.set_level(logging.DEBUG)
init_directory(tmpdir)
start = 1524769441
time_mock.side_effect = monotonic_time(start=start)
add_two_fresh_results(tmpdir, start - 1)
p = create_parser()
args = p.parse_args(
'-d {} --log-level DEBUG stats --error-types'.format(tmpdir).split())
conf = get_config(args)
sbws.core.stats.main(args, conf)
needed_output_lines = [
'1 relays have recent results',
'1 success results and 1 error results',
'Mean 1.00 successful measurements per relay',
'1/2 (50.00%) results were error-misc',
]
captured = capsys.readouterr()
lines = captured.out.strip().split('\n')
for needed_line in needed_output_lines:
assert needed_line in lines
lines = [l.getMessage() for l in caplog.records]
needed_log_lines = [
'Keeping 2/2 read lines from {}/{}/{}.txt'.format(
tmpdir, 'datadir',
datetime.utcfromtimestamp(time.time()).date()),
'Keeping 2/2 results after removing old ones',
'Found a _ResultType.Error for the first time',
'Found a _ResultType.Success for the first time',
]
for needed_line in needed_log_lines:
assert needed_line in lines
def init_directory(dname):
p = create_parser()
args = p.parse_args('-d {} --log-level DEBUG init'.format(dname).split())
conf = get_config(args)
sbws.core.init.main(args, conf)
def dotsbws_success_result_two_relays(empty_dotsbws_datadir):
'''
Creates an ~/.sbws with a a couple of fresh ResultSuccess for a couple or
relays
'''
args = _PseudoArguments(directory=empty_dotsbws_datadir.name)
conf = get_config(args)
dd = conf['paths']['datadir']
fp1 = 'A' * 40
fp2 = 'C' * 40
circ = [fp1, fp2]
nick = 'CowSayWhat1'
relay_ip = '169.254.100.1'
server_ip = '169.254.100.3'
scanner_nick = 'SBWSscanner'
rtts = [5, 25]
downloads = [{'duration': 4, 'amount': 40 * 1024}]
t = time.time()
relay = Result.Relay(fp1, nick, relay_ip)
result = ResultSuccess(rtts,
downloads,
relay,
circ,
server_ip,
scanner_nick,
t=t)
write_result_to_datadir(result, dd)
rtts = [10, 20]
downloads = [{'duration': 4, 'amount': 80 * 1024}]
t = time.time()
result = ResultSuccess(rtts,
downloads,
relay,
circ,
server_ip,
scanner_nick,
t=t)
write_result_to_datadir(result, dd)
fp1 = 'B' * 40
circ = [fp1, fp2]
nick = 'CowSayWhat2'
relay_ip = '169.254.100.2'
rtts = [50, 250]
downloads = [{'duration': 4, 'amount': 400 * 1024}]
t = time.time()
relay = Result.Relay(fp1, nick, relay_ip)
result = ResultSuccess(rtts,
downloads,
relay,
circ,
server_ip,
scanner_nick,
t=t)
write_result_to_datadir(result, dd)
rtts = [100, 200]
downloads = [{'duration': 4, 'amount': 800 * 1024}]
t = time.time()
result = ResultSuccess(rtts,
downloads,
relay,
circ,
server_ip,
scanner_nick,
t=t)
write_result_to_datadir(result, dd)
return empty_dotsbws_datadir
