def test_epsc_3D_vuln():
dev_file = pkg_resources.resource_filename(
"bob.bio.base.test", "data/vuln/scores-dev.csv"
)
eval_file = pkg_resources.resource_filename(
"bob.bio.base.test", "data/vuln/scores-eval.csv"
)
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(
vuln_commands.epsc,
[
"--output",
"epsc.pdf",
"-D",
dev_file,
eval_file,
],
)
assert_click_runner_result(result)
result = runner.invoke(
vuln_commands.epsc,
[
"--output",
"epsc.pdf",
"-D",
"-I",
"--no-wer",
dev_file,
eval_file,
],
)
assert_click_runner_result(result)
def test_fmr_iapmr_vuln():
dev_file = pkg_resources.resource_filename(
"bob.bio.base.test", "data/vuln/scores-dev.csv"
)
eval_file = pkg_resources.resource_filename(
"bob.bio.base.test", "data/vuln/scores-eval.csv"
)
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(
vuln_commands.fmr_iapmr,
[
"--output",
"FMRIAPMR.pdf",
dev_file,
eval_file,
],
)
assert_click_runner_result(result)
result = runner.invoke(
vuln_commands.fmr_iapmr,
[
"--output",
"FMRIAPMR.pdf",
dev_file,
eval_file,
"-G",
"-L",
"1e-7,1,0,1",
],
)
assert_click_runner_result(result)
def test_epc():
dev1 = _F("dev-1.txt")
test1 = _F("test-1.txt")
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.epc, [dev1, test1])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
dev2 = _F("dev-2.txt")
test2 = _F("test-2.txt")
with runner.isolated_filesystem():
result = runner.invoke(
commands.epc,
[
"--output",
"test.pdf",
"--legends",
"A,B",
"--titles",
"TA,TB",
"-ll",
"upper-right",
dev1,
test1,
dev2,
test2,
],
)
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def test_dir():
dev1 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/scores-nonorm-openset-dev"
)
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.dir, [dev1, "--rank", 2])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
test1 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/scores-nonorm-openset-dev"
)
with runner.isolated_filesystem():
result = runner.invoke(
commands.dir,
[
"--output",
"test.pdf",
"--legends",
"A,B",
"-S",
"--min-far-value",
"1e-6",
"-e",
dev1,
test1,
dev1,
test1,
],
)
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def test_sort():
def sorted_scores(score_lines):
lines = []
floats = []
for line in score_lines:
lines.append(line)
floats.append(line[-1])
sort_idx = numpy.argsort(floats)
lines = [lines[i] for i in sort_idx]
return lines
dev1 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/scores-nonorm-dev"
)
runner = CliRunner()
with runner.isolated_filesystem():
# create a temporary sort file and sort it and check if it is sorted!
path = "scores.txt"
shutil.copy(dev1, path)
result = runner.invoke(sort.sort, [path])
assert_click_runner_result(result, exit_code=0)
# load dev1 and sort it and compare to path
dev1_sorted = list(sorted_scores(scores(dev1)))
path_scores = list(scores(path))
assert dev1_sorted == path_scores
def test_hist_legends():
dev1 = _F("dev-1.txt")
test1 = _F("test-1.txt")
dev2 = _F("dev-2.txt")
test2 = _F("test-2.txt")
runner = CliRunner()
# share same title for dev/eval of each system
with runner.isolated_filesystem():
result = runner.invoke(
commands.hist,
["-e", "-sp", 111, "-ts", "A,B", dev1, test1, dev2, test2],
)
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
# individual titles for dev and eval
with runner.isolated_filesystem():
result = runner.invoke(
commands.hist,
["-e", "-sp", 121, "-ts", "A,B,C,D", dev1, test1, dev2, test2],
)
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def test_det_vuln():
dev_file = pkg_resources.resource_filename(
"bob.bio.base.test", "data/vuln/scores-dev.csv"
)
eval_file = pkg_resources.resource_filename(
"bob.bio.base.test", "data/vuln/scores-eval.csv"
)
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(
vuln_commands.det,
["-fnmr", "0.2", "-o", "DET.pdf", dev_file, eval_file],
)
assert_click_runner_result(result)
def test_metrics():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
runner = CliRunner()
result = runner.invoke(commands.metrics, [dev1])
with runner.isolated_filesystem():
with open('tmp', 'w') as f:
f.write(result.output)
test_ref = bob.io.base.test_utils.datafile('test_m1.txt', 'bob.measure')
assert_click_runner_result(result)
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-e', dev1, test1, dev2, test2]
)
with open('tmp', 'w') as f:
f.write(result.output)
test_ref = bob.io.base.test_utils.datafile('test_m2.txt', 'bob.measure')
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-e', '-l', 'tmp', dev1, test1, dev2, test2, '-lg',
'A,B']
)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', dev1, dev2]
)
assert_click_runner_result(result)
def test_metrics():
dev1 = _F("dev-1.txt")
runner = CliRunner()
result = runner.invoke(commands.metrics, [dev1])
with runner.isolated_filesystem():
with open("tmp", "w") as f:
f.write(result.output)
assert_click_runner_result(result)
dev2 = _F("dev-2.txt")
test1 = _F("test-1.txt")
test2 = _F("test-2.txt")
with runner.isolated_filesystem():
result = runner.invoke(commands.metrics,
["-e", dev1, test1, dev2, test2])
with open("tmp", "w") as f:
f.write(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics,
["-e", "-l", "tmp", dev1, test1, dev2, test2, "-lg", "A,B"],
)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(commands.metrics, ["-l", "tmp", dev1, dev2])
assert_click_runner_result(result)
def test_dir():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-openset-dev')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.dir, [dev1, '--rank', 2])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-openset-dev')
with runner.isolated_filesystem():
result = runner.invoke(commands.dir, ['--output', 'test.pdf',
'--legends', 'A,B', '-S',
'--min-far-value', '1e-6', '-e',
dev1, test1, dev1, test1])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def test_dir():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-openset-dev')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.dir, [dev1, '--rank', 2])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-openset-dev')
with runner.isolated_filesystem():
result = runner.invoke(commands.dir, [
'--output', 'test.pdf', '--legends', 'A,B', '-S',
'--min-far-value', '1e-6', '-e', dev1, test1, dev1, test1
])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def test_hist_vuln():
dev_file = pkg_resources.resource_filename(
"bob.bio.base.test", "data/vuln/scores-dev.csv"
)
eval_file = pkg_resources.resource_filename(
"bob.bio.base.test", "data/vuln/scores-eval.csv"
)
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(
vuln_commands.hist,
[
"--criterion",
"eer",
"--output",
"HISTO.pdf",
"-b",
"30",
"-ts",
"A,B",
dev_file,
eval_file,
],
)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
vuln_commands.hist,
[
"--criterion",
"eer",
"--output",
"HISTO.pdf",
"-b",
"2,20,30",
"-e",
dev_file,
eval_file,
],
)
assert_click_runner_result(result)
def test_epc():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.epc, [dev1, test1])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
result = runner.invoke(commands.epc, ['--output', 'test.pdf',
'--legends', 'A,B',
'--titles', 'TA,TB',
'-ll', 'upper-right',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def test_annotate_samples():
try:
tmp_dir = tempfile.mkdtemp(prefix="bobtest_")
runner = CliRunner()
result = runner.invoke(
annotate_samples,
args=("dummy_samples", "-a", "dummy", "-o", tmp_dir),
)
assert_click_runner_result(result)
# test if annotations exist
for dirpath, dirnames, filenames in os.walk(tmp_dir):
for filename in filenames:
path = os.path.join(dirpath, filename)
annot = read_annotation_file(path, "json")
assert annot["topleft"] == [0, 0]
# size of atnt images
assert annot["bottomright"] == [112, 92]
finally:
shutil.rmtree(tmp_dir)
def test_roc():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--output', 'test.pdf',dev1])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--split', '--output',
'test.pdf', '-e',
'-ts', 'A,',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['-e', '--output',
'test.pdf', '--legends', 'A,B',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def test_hist():
dev1 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/dev-4col.txt"
)
dev2 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/dev-5col.txt"
)
test1 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/test-4col.txt"
)
test2 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/test-5col.txt"
)
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, [dev1])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.hist,
[
"--criterion",
"min-hter",
"--output",
"HISTO.pdf",
"-b",
"30,auto",
dev1,
dev2,
],
)
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.hist,
[
"--criterion",
"eer",
"--output",
"HISTO.pdf",
"-b",
"30",
"-e",
"-ts",
"A,B",
dev1,
test1,
dev2,
test2,
],
)
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def test_hist():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, [dev1])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--criterion',
'min-hter', '--no-line',
'--output', 'HISTO.pdf', '-b',
'30,100', dev1, dev2])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['-e', '--criterion', 'eer','--output',
'HISTO.pdf', '-b', '30,20',
'-sp', 111, '-ts', 'A,B',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def test_cmc():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-cmc-5col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.cmc, [dev1])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-cmc-4col.txt')
with runner.isolated_filesystem():
result = runner.invoke(commands.cmc, ['--output', 'test.pdf',
'--legends', 'A,B', '-S',
'-ts', 'TA,TB', '-e',
dev1, test1, dev1, test1])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
dev_nonorm = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-dev')
dev_ztnorm = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-ztnorm-dev')
with runner.isolated_filesystem():
result = runner.invoke(commands.cmc, [
'-v', '--legends', 'A', '-e',
dev_nonorm, dev_ztnorm
])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def test_hist():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-5col.txt')
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-4col.txt')
test2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-5col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, [dev1])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--criterion', 'min-hter', '--output',
'HISTO.pdf', '-b',
'30,auto', dev1, dev2])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['--criterion', 'eer', '--output',
'HISTO.pdf', '-b', '30', '-e',
'-ts', 'A,B', dev1, test1, dev2,
test2])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def test_hist():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-5col.txt')
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-4col.txt')
test2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-5col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, [dev1])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, [
'--criterion', 'min-hter', '--output', 'HISTO.pdf', '-b',
'30,auto', dev1, dev2
])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, [
'--criterion', 'eer', '--output', 'HISTO.pdf', '-b', '30', '-e',
'-ts', 'A,B', dev1, test1, dev2, test2
])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def test_cmc():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-cmc-5col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.cmc, [dev1])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-cmc-4col.txt')
with runner.isolated_filesystem():
result = runner.invoke(commands.cmc, [
'--output', 'test.pdf', '--legends', 'A,B', '-S', '-ts', 'TA,TB',
'-e', dev1, test1, dev1, test1
])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
dev_nonorm = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-dev')
dev_ztnorm = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-ztnorm-dev')
with runner.isolated_filesystem():
result = runner.invoke(
commands.cmc,
['-v', '--legends', 'A', '-e', dev_nonorm, dev_ztnorm])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def test_roc():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--output',
'test.pdf',dev1])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-5col.txt')
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-4col.txt')
test2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-5col.txt')
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--split', '--output',
'test.pdf', '-S', '-ll',
'lower-left', '-e',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--output',
'test.pdf',
'-e', '--legends', 'A,B',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
dev_nonorm = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-dev')
dev_ztnorm = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-ztnorm-dev')
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, [
'--min-far-value', '1e-6',
'--lines-at', '1e-5',
'-v', '--legends', 'A', '-e',
dev_nonorm, dev_ztnorm
])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def _test_pipeline_click_cli(
cli, options, expected_outputs=("results/scores-dev.csv",)
):
runner = CliRunner()
with runner.isolated_filesystem():
_create_test_config("config.py")
result = runner.invoke(
cli,
[
"-vv",
"config.py",
]
+ options,
)
assert_click_runner_result(result)
# check for expected_output
output_files = glob.glob("results/**", recursive=True)
err_msg = "Found only:\n{output_files}\nin output directory given the options: {options}".format(
output_files="\n".join(output_files), options=options
)
for out in expected_outputs:
assert os.path.isfile(out), err_msg
def test_hist_legends():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
runner = CliRunner()
# share same title for dev/eval of each system
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['-e', '-sp', 111, '-ts', 'A,B',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
# individual titles for dev and eval
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, ['-e', '-sp', 121, '-ts',
'A,B,C,D',
dev1, test1, dev2, test2])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def test_hist():
dev1 = _F("dev-1.txt")
test1 = _F("test-1.txt")
dev2 = _F("dev-2.txt")
test2 = _F("test-2.txt")
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.hist, [dev1])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.hist,
[
"--criterion",
"min-hter",
"--no-line",
"--output",
"HISTO.pdf",
"-b",
"30,100",
dev1,
dev2,
],
)
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.hist,
[
"-e",
"--criterion",
"eer",
"--output",
"HISTO.pdf",
"-b",
"30,20",
"-sp",
111,
"-ts",
"A,B",
dev1,
test1,
dev2,
test2,
],
)
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def test_roc():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ['--output', 'test.pdf', dev1])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-5col.txt')
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-4col.txt')
test2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-5col.txt')
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, [
'--split', '--output', 'test.pdf', '-S', '-lc', 'lower-left', '-e',
dev1, test1, dev2, test2
])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, [
'--output', 'test.pdf', '-e', '--legends', 'A,B', dev1, test1,
dev2, test2
])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
dev_nonorm = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-nonorm-dev')
dev_ztnorm = pkg_resources.resource_filename('bob.bio.base.test',
'data/scores-ztnorm-dev')
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, [
'--min-far-value', '1e-6', '--lines-at', '1e-5', '-v', '--legends',
'A', '-e', dev_nonorm, dev_ztnorm
])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def test_epc():
dev1 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/dev-4col.txt"
)
test1 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/test-4col.txt"
)
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.epc, [dev1, test1])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
dev2 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/dev-4col.tar.gz"
)
test2 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/test-5col.txt"
)
with runner.isolated_filesystem():
result = runner.invoke(
commands.epc,
[
"--output",
"test.pdf",
"--legends",
"A,B",
"-S",
dev1,
test1,
dev2,
test2,
],
)
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
dev_nonorm = pkg_resources.resource_filename(
"bob.bio.base.test", "data/scores-nonorm-dev"
)
dev_ztnorm = pkg_resources.resource_filename(
"bob.bio.base.test", "data/scores-ztnorm-dev"
)
with runner.isolated_filesystem():
result = runner.invoke(
commands.epc, ["-v", "--legends", "A", dev_nonorm, dev_ztnorm]
)
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def test_roc():
dev1 = _F("dev-1.txt")
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.roc, ["--output", "test.pdf", dev1])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
dev2 = _F("dev-2.txt")
test1 = _F("test-1.txt")
test2 = _F("test-2.txt")
with runner.isolated_filesystem():
result = runner.invoke(
commands.roc,
[
"--split",
"--output",
"test.pdf",
"-e",
"-ts",
"A,",
dev1,
test1,
dev2,
test2,
],
)
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.roc,
[
"-e",
"--output",
"test.pdf",
"--legends",
"A,B",
dev1,
test1,
dev2,
test2,
],
)
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def test_metrics_vuln():
dev1 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/vuln/scores-dev.csv"
)
runner = CliRunner()
result = runner.invoke(vuln_commands.metrics, [dev1])
with runner.isolated_filesystem():
with open("tmp", "w") as f:
f.write(result.output)
assert_click_runner_result(result)
dev2 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/vuln/scores-dev-med.csv"
)
test1 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/vuln/scores-eval.csv"
)
test2 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/vuln/scores-eval-med.csv"
)
with runner.isolated_filesystem():
result = runner.invoke(
vuln_commands.metrics, ["-e", dev1, test1, dev2, test2]
)
with open("tmp", "w") as f:
f.write(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
vuln_commands.metrics,
["-e", "-l", "tmp", "-lg", "A,B", dev1, test1, dev2, test2],
)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
vuln_commands.metrics, ["-e", "-l", "tmp", dev1, test2]
)
assert_click_runner_result(result)
def test_evaluate():
dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure')
test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure')
dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure')
test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure')
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.evaluate, [dev1])
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.evaluate, ['--output', 'my_plots.pdf',
'-n', 300, dev1, dev2])
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.evaluate, ['-e', dev1, test1, dev2, test2])
assert_click_runner_result(result)
def test_evaluate():
dev1 = _F("dev-1.txt")
test1 = _F("test-1.txt")
dev2 = _F("dev-2.txt")
test2 = _F("test-2.txt")
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.evaluate, [dev1])
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.evaluate,
["--output", "my_plots.pdf", "-n", 300, dev1, dev2],
)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(commands.evaluate,
["-e", dev1, test1, dev2, test2])
assert_click_runner_result(result)
def test_det():
dev1 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/dev-4col.txt"
)
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(commands.det, [dev1, "-S"])
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
dev2 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/dev-5col.txt"
)
test1 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/test-4col.txt"
)
test2 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/test-5col.txt"
)
with runner.isolated_filesystem():
result = runner.invoke(
commands.det,
[
"--split",
"--output",
"test.pdf",
"--legends",
"A,B",
"-e",
dev1,
test1,
dev2,
test2,
],
)
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.det,
["--output", "test.pdf", "-e", dev1, test1, dev2, test2],
)
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
dev_nonorm = pkg_resources.resource_filename(
"bob.bio.base.test", "data/scores-nonorm-dev"
)
dev_ztnorm = pkg_resources.resource_filename(
"bob.bio.base.test", "data/scores-ztnorm-dev"
)
with runner.isolated_filesystem():
result = runner.invoke(
commands.det,
[
"--min-far-value",
"1e-6",
"--lines-at",
"1e-5",
"-e",
"-v",
"--legends",
"A",
dev_nonorm,
dev_ztnorm,
],
)
if result.output:
click.echo(result.output)
assert_click_runner_result(result)
def gen_case(
n_subjects,
n_probes_per_subject,
n_unknown_subjects,
n_pos=None,
n_neg=None,
n_unk=None,
):
"""Tests one case of the gen command"""
n_pos = n_subjects * n_probes_per_subject if n_pos is None else n_pos
n_neg = (n_subjects * (n_subjects - 1) *
n_probes_per_subject if n_neg is None else n_neg)
n_unk = (n_unknown_subjects * n_subjects *
n_probes_per_subject if n_unk is None else n_unk)
runner = CliRunner()
with runner.isolated_filesystem():
temp_path = "./gen_test_temp_dir/"
logger.info("Calling 'gen' with a specific amount of scores.")
result = runner.invoke(
gen,
args=[
"-mm",
"10",
"-mnm",
"-10",
"-sp",
"1",
"-sn",
"1",
"-p",
f"{n_probes_per_subject}",
"-s",
f"{n_subjects}",
"-u",
f"{n_unknown_subjects}",
"-f",
"--n-pos",
f"{n_pos}",
"--n-neg",
f"{n_neg}",
"--n-unk",
f"{n_unk}",
f"{temp_path}",
],
)
assert_click_runner_result(result)
assert os.path.exists(os.path.join(
temp_path, "scores-dev.csv")), "dev scores file not created."
line_count = 0
with open(os.path.join(temp_path, "scores-dev.csv")) as f:
for _ in f:
line_count += 1
assert line_count == 1 + n_pos + n_neg + n_unk
assert os.path.exists(os.path.join(
temp_path, "scores-eval.csv")), "eval scores file not created."
line_count = 0
with open(os.path.join(temp_path, "scores-eval.csv")) as f:
for _ in f:
line_count += 1
assert line_count == 1 + n_pos + n_neg + n_unk
def test_metrics():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
runner = CliRunner()
result = runner.invoke(commands.metrics, [dev1])
with runner.isolated_filesystem():
with open('tmp', 'w') as f:
f.write(result.output)
assert_click_runner_result(result)
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-5col.txt')
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-4col.txt')
test2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-5col.txt')
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-e', dev1, test1, dev2, test2]
)
with open('tmp', 'w') as f:
f.write(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-e', '-l', 'tmp', '-lg', 'A,B',
dev1, test1, dev2, test2]
)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-e', '-l', 'tmp', dev1, test2]
)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-e', '-l', 'tmp', '-T', '0.1',
'--criterion', 'mindcf', '--cost', 0.9,
dev1, test2]
)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp',
'--criterion', 'mindcf', '--cost', 0.9,
dev1]
)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-e', '--criterion', 'cllr', dev1, test2]
)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', '--criterion', 'cllr',
'--cost', 0.9, dev1]
)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-e', '--criterion', 'rr', '-T',
'0.1', dev1, test2]
)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ['-l', 'tmp', '--criterion', 'rr',
dev1, dev2]
)
assert_click_runner_result(result)
def test_metrics():
dev1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-4col.txt')
runner = CliRunner()
result = runner.invoke(commands.metrics, [dev1])
with runner.isolated_filesystem():
with open('tmp', 'w') as f:
f.write(result.output)
assert_click_runner_result(result)
dev2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/dev-5col.txt')
test1 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-4col.txt')
test2 = pkg_resources.resource_filename('bob.bio.base.test',
'data/test-5col.txt')
with runner.isolated_filesystem():
result = runner.invoke(commands.metrics,
['-e', dev1, test1, dev2, test2])
with open('tmp', 'w') as f:
f.write(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics,
['-e', '-l', 'tmp', '-lg', 'A,B', dev1, test1, dev2, test2])
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(commands.metrics,
['-e', '-l', 'tmp', dev1, test2])
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(commands.metrics, [
'-e', '-l', 'tmp', '-T', '0.1', '--criterion', 'mindcf', '--cost',
0.9, dev1, test2
])
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics,
['-l', 'tmp', '--criterion', 'mindcf', '--cost', 0.9, dev1])
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(commands.metrics,
['-e', '--criterion', 'cllr', dev1, test2])
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics,
['-l', 'tmp', '--criterion', 'cllr', '--cost', 0.9, dev1])
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics,
['-e', '--criterion', 'rr', '-T', '0.1', dev1, test2])
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(commands.metrics,
['-l', 'tmp', '--criterion', 'rr', dev1, dev2])
assert_click_runner_result(result)
def test_metrics():
dev1 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/dev-4col.txt"
)
runner = CliRunner()
result = runner.invoke(commands.metrics, [dev1])
with runner.isolated_filesystem():
with open("tmp", "w") as f:
f.write(result.output)
assert_click_runner_result(result)
dev2 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/dev-5col.txt"
)
test1 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/test-4col.txt"
)
test2 = pkg_resources.resource_filename(
"bob.bio.base.test", "data/test-5col.txt"
)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ["-e", dev1, test1, dev2, test2]
)
with open("tmp", "w") as f:
f.write(result.output)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics,
["-e", "-l", "tmp", "-lg", "A,B", dev1, test1, dev2, test2],
)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ["-e", "-l", "tmp", dev1, test2]
)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics,
[
"-e",
"-l",
"tmp",
"-T",
"0.1",
"--criterion",
"mindcf",
"--cost",
0.9,
dev1,
test2,
],
)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics,
["-l", "tmp", "--criterion", "mindcf", "--cost", 0.9, dev1],
)
assert_click_runner_result(result)
# with runner.isolated_filesystem():
# result = runner.invoke(
# commands.metrics, ["-e", "--criterion", "cllr", dev1, test2]
# )
# assert_click_runner_result(result)
# with runner.isolated_filesystem():
# result = runner.invoke(
# commands.metrics, ["-l", "tmp", "--criterion", "cllr", "--cost", 0.9, dev1]
# )
# assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics,
["-e", "--criterion", "rr", "-T", "0.1", dev1, test2],
)
assert_click_runner_result(result)
with runner.isolated_filesystem():
result = runner.invoke(
commands.metrics, ["-l", "tmp", "--criterion", "rr", dev1, dev2]
)
assert_click_runner_result(result)
