def test_epsc_3D_vuln(): dev_file = pkg_resources.resource_filename( "bob.bio.base.test", "data/vuln/scores-dev.csv" ) eval_file = pkg_resources.resource_filename( "bob.bio.base.test", "data/vuln/scores-eval.csv" ) runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke( vuln_commands.epsc, [ "--output", "epsc.pdf", "-D", dev_file, eval_file, ], ) assert_click_runner_result(result) result = runner.invoke( vuln_commands.epsc, [ "--output", "epsc.pdf", "-D", "-I", "--no-wer", dev_file, eval_file, ], ) assert_click_runner_result(result)
def test_fmr_iapmr_vuln(): dev_file = pkg_resources.resource_filename( "bob.bio.base.test", "data/vuln/scores-dev.csv" ) eval_file = pkg_resources.resource_filename( "bob.bio.base.test", "data/vuln/scores-eval.csv" ) runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke( vuln_commands.fmr_iapmr, [ "--output", "FMRIAPMR.pdf", dev_file, eval_file, ], ) assert_click_runner_result(result) result = runner.invoke( vuln_commands.fmr_iapmr, [ "--output", "FMRIAPMR.pdf", dev_file, eval_file, "-G", "-L", "1e-7,1,0,1", ], ) assert_click_runner_result(result)
def test_epc(): dev1 = _F("dev-1.txt") test1 = _F("test-1.txt") runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.epc, [dev1, test1]) if result.output: click.echo(result.output) assert_click_runner_result(result) dev2 = _F("dev-2.txt") test2 = _F("test-2.txt") with runner.isolated_filesystem(): result = runner.invoke( commands.epc, [ "--output", "test.pdf", "--legends", "A,B", "--titles", "TA,TB", "-ll", "upper-right", dev1, test1, dev2, test2, ], ) if result.output: click.echo(result.output) assert_click_runner_result(result)
def test_dir(): dev1 = pkg_resources.resource_filename( "bob.bio.base.test", "data/scores-nonorm-openset-dev" ) runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.dir, [dev1, "--rank", 2]) if result.output: click.echo(result.output) assert_click_runner_result(result) test1 = pkg_resources.resource_filename( "bob.bio.base.test", "data/scores-nonorm-openset-dev" ) with runner.isolated_filesystem(): result = runner.invoke( commands.dir, [ "--output", "test.pdf", "--legends", "A,B", "-S", "--min-far-value", "1e-6", "-e", dev1, test1, dev1, test1, ], ) if result.output: click.echo(result.output) assert_click_runner_result(result)
def test_sort(): def sorted_scores(score_lines): lines = [] floats = [] for line in score_lines: lines.append(line) floats.append(line[-1]) sort_idx = numpy.argsort(floats) lines = [lines[i] for i in sort_idx] return lines dev1 = pkg_resources.resource_filename( "bob.bio.base.test", "data/scores-nonorm-dev" ) runner = CliRunner() with runner.isolated_filesystem(): # create a temporary sort file and sort it and check if it is sorted! path = "scores.txt" shutil.copy(dev1, path) result = runner.invoke(sort.sort, [path]) assert_click_runner_result(result, exit_code=0) # load dev1 and sort it and compare to path dev1_sorted = list(sorted_scores(scores(dev1))) path_scores = list(scores(path)) assert dev1_sorted == path_scores
def test_hist_legends(): dev1 = _F("dev-1.txt") test1 = _F("test-1.txt") dev2 = _F("dev-2.txt") test2 = _F("test-2.txt") runner = CliRunner() # share same title for dev/eval of each system with runner.isolated_filesystem(): result = runner.invoke( commands.hist, ["-e", "-sp", 111, "-ts", "A,B", dev1, test1, dev2, test2], ) if result.output: click.echo(result.output) assert_click_runner_result(result) # individual titles for dev and eval with runner.isolated_filesystem(): result = runner.invoke( commands.hist, ["-e", "-sp", 121, "-ts", "A,B,C,D", dev1, test1, dev2, test2], ) if result.output: click.echo(result.output) assert_click_runner_result(result)
def test_det_vuln(): dev_file = pkg_resources.resource_filename( "bob.bio.base.test", "data/vuln/scores-dev.csv" ) eval_file = pkg_resources.resource_filename( "bob.bio.base.test", "data/vuln/scores-eval.csv" ) runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke( vuln_commands.det, ["-fnmr", "0.2", "-o", "DET.pdf", dev_file, eval_file], ) assert_click_runner_result(result)
def test_metrics(): dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure') runner = CliRunner() result = runner.invoke(commands.metrics, [dev1]) with runner.isolated_filesystem(): with open('tmp', 'w') as f: f.write(result.output) test_ref = bob.io.base.test_utils.datafile('test_m1.txt', 'bob.measure') assert_click_runner_result(result) dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure') test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure') test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure') with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ['-e', dev1, test1, dev2, test2] ) with open('tmp', 'w') as f: f.write(result.output) test_ref = bob.io.base.test_utils.datafile('test_m2.txt', 'bob.measure') assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ['-e', '-l', 'tmp', dev1, test1, dev2, test2, '-lg', 'A,B'] ) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ['-l', 'tmp', dev1, dev2] ) assert_click_runner_result(result)
def test_metrics(): dev1 = _F("dev-1.txt") runner = CliRunner() result = runner.invoke(commands.metrics, [dev1]) with runner.isolated_filesystem(): with open("tmp", "w") as f: f.write(result.output) assert_click_runner_result(result) dev2 = _F("dev-2.txt") test1 = _F("test-1.txt") test2 = _F("test-2.txt") with runner.isolated_filesystem(): result = runner.invoke(commands.metrics, ["-e", dev1, test1, dev2, test2]) with open("tmp", "w") as f: f.write(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ["-e", "-l", "tmp", dev1, test1, dev2, test2, "-lg", "A,B"], ) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke(commands.metrics, ["-l", "tmp", dev1, dev2]) assert_click_runner_result(result)
def test_dir(): dev1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/scores-nonorm-openset-dev') runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.dir, [dev1, '--rank', 2]) if result.output: click.echo(result.output) assert_click_runner_result(result) test1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/scores-nonorm-openset-dev') with runner.isolated_filesystem(): result = runner.invoke(commands.dir, ['--output', 'test.pdf', '--legends', 'A,B', '-S', '--min-far-value', '1e-6', '-e', dev1, test1, dev1, test1]) if result.output: click.echo(result.output) assert_click_runner_result(result)
def test_dir(): dev1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/scores-nonorm-openset-dev') runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.dir, [dev1, '--rank', 2]) if result.output: click.echo(result.output) assert_click_runner_result(result) test1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/scores-nonorm-openset-dev') with runner.isolated_filesystem(): result = runner.invoke(commands.dir, [ '--output', 'test.pdf', '--legends', 'A,B', '-S', '--min-far-value', '1e-6', '-e', dev1, test1, dev1, test1 ]) if result.output: click.echo(result.output) assert_click_runner_result(result)
def test_hist_vuln(): dev_file = pkg_resources.resource_filename( "bob.bio.base.test", "data/vuln/scores-dev.csv" ) eval_file = pkg_resources.resource_filename( "bob.bio.base.test", "data/vuln/scores-eval.csv" ) runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke( vuln_commands.hist, [ "--criterion", "eer", "--output", "HISTO.pdf", "-b", "30", "-ts", "A,B", dev_file, eval_file, ], ) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( vuln_commands.hist, [ "--criterion", "eer", "--output", "HISTO.pdf", "-b", "2,20,30", "-e", dev_file, eval_file, ], ) assert_click_runner_result(result)
def test_epc(): dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure') test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure') runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.epc, [dev1, test1]) if result.output: click.echo(result.output) assert_click_runner_result(result) dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure') test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure') with runner.isolated_filesystem(): result = runner.invoke(commands.epc, ['--output', 'test.pdf', '--legends', 'A,B', '--titles', 'TA,TB', '-ll', 'upper-right', dev1, test1, dev2, test2]) if result.output: click.echo(result.output) assert_click_runner_result(result)
def test_annotate_samples(): try: tmp_dir = tempfile.mkdtemp(prefix="bobtest_") runner = CliRunner() result = runner.invoke( annotate_samples, args=("dummy_samples", "-a", "dummy", "-o", tmp_dir), ) assert_click_runner_result(result) # test if annotations exist for dirpath, dirnames, filenames in os.walk(tmp_dir): for filename in filenames: path = os.path.join(dirpath, filename) annot = read_annotation_file(path, "json") assert annot["topleft"] == [0, 0] # size of atnt images assert annot["bottomright"] == [112, 92] finally: shutil.rmtree(tmp_dir)
def test_roc(): dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure') runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.roc, ['--output', 'test.pdf',dev1]) if result.output: click.echo(result.output) assert_click_runner_result(result) dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure') test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure') test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure') with runner.isolated_filesystem(): result = runner.invoke(commands.roc, ['--split', '--output', 'test.pdf', '-e', '-ts', 'A,', dev1, test1, dev2, test2]) if result.output: click.echo(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke(commands.roc, ['-e', '--output', 'test.pdf', '--legends', 'A,B', dev1, test1, dev2, test2]) if result.output: click.echo(result.output) assert_click_runner_result(result)
def test_hist(): dev1 = pkg_resources.resource_filename( "bob.bio.base.test", "data/dev-4col.txt" ) dev2 = pkg_resources.resource_filename( "bob.bio.base.test", "data/dev-5col.txt" ) test1 = pkg_resources.resource_filename( "bob.bio.base.test", "data/test-4col.txt" ) test2 = pkg_resources.resource_filename( "bob.bio.base.test", "data/test-5col.txt" ) runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.hist, [dev1]) if result.output: click.echo(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.hist, [ "--criterion", "min-hter", "--output", "HISTO.pdf", "-b", "30,auto", dev1, dev2, ], ) if result.output: click.echo(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.hist, [ "--criterion", "eer", "--output", "HISTO.pdf", "-b", "30", "-e", "-ts", "A,B", dev1, test1, dev2, test2, ], ) if result.output: click.echo(result.output) assert_click_runner_result(result)
def test_hist(): dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure') test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure') dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure') test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure') runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.hist, [dev1]) if result.output: click.echo(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke(commands.hist, ['--criterion', 'min-hter', '--no-line', '--output', 'HISTO.pdf', '-b', '30,100', dev1, dev2]) if result.output: click.echo(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke(commands.hist, ['-e', '--criterion', 'eer','--output', 'HISTO.pdf', '-b', '30,20', '-sp', 111, '-ts', 'A,B', dev1, test1, dev2, test2]) if result.output: click.echo(result.output) assert_click_runner_result(result)
def test_cmc(): dev1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/scores-cmc-5col.txt') runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.cmc, [dev1]) if result.output: click.echo(result.output) assert_click_runner_result(result) test1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/scores-cmc-4col.txt') with runner.isolated_filesystem(): result = runner.invoke(commands.cmc, ['--output', 'test.pdf', '--legends', 'A,B', '-S', '-ts', 'TA,TB', '-e', dev1, test1, dev1, test1]) if result.output: click.echo(result.output) assert_click_runner_result(result) dev_nonorm = pkg_resources.resource_filename('bob.bio.base.test', 'data/scores-nonorm-dev') dev_ztnorm = pkg_resources.resource_filename('bob.bio.base.test', 'data/scores-ztnorm-dev') with runner.isolated_filesystem(): result = runner.invoke(commands.cmc, [ '-v', '--legends', 'A', '-e', dev_nonorm, dev_ztnorm ]) if result.output: click.echo(result.output) assert_click_runner_result(result)
def test_hist(): dev1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/dev-4col.txt') dev2 = pkg_resources.resource_filename('bob.bio.base.test', 'data/dev-5col.txt') test1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/test-4col.txt') test2 = pkg_resources.resource_filename('bob.bio.base.test', 'data/test-5col.txt') runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.hist, [dev1]) if result.output: click.echo(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke(commands.hist, ['--criterion', 'min-hter', '--output', 'HISTO.pdf', '-b', '30,auto', dev1, dev2]) if result.output: click.echo(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke(commands.hist, ['--criterion', 'eer', '--output', 'HISTO.pdf', '-b', '30', '-e', '-ts', 'A,B', dev1, test1, dev2, test2]) if result.output: click.echo(result.output) assert_click_runner_result(result)
def test_hist(): dev1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/dev-4col.txt') dev2 = pkg_resources.resource_filename('bob.bio.base.test', 'data/dev-5col.txt') test1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/test-4col.txt') test2 = pkg_resources.resource_filename('bob.bio.base.test', 'data/test-5col.txt') runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.hist, [dev1]) if result.output: click.echo(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke(commands.hist, [ '--criterion', 'min-hter', '--output', 'HISTO.pdf', '-b', '30,auto', dev1, dev2 ]) if result.output: click.echo(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke(commands.hist, [ '--criterion', 'eer', '--output', 'HISTO.pdf', '-b', '30', '-e', '-ts', 'A,B', dev1, test1, dev2, test2 ]) if result.output: click.echo(result.output) assert_click_runner_result(result)
def test_cmc(): dev1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/scores-cmc-5col.txt') runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.cmc, [dev1]) if result.output: click.echo(result.output) assert_click_runner_result(result) test1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/scores-cmc-4col.txt') with runner.isolated_filesystem(): result = runner.invoke(commands.cmc, [ '--output', 'test.pdf', '--legends', 'A,B', '-S', '-ts', 'TA,TB', '-e', dev1, test1, dev1, test1 ]) if result.output: click.echo(result.output) assert_click_runner_result(result) dev_nonorm = pkg_resources.resource_filename('bob.bio.base.test', 'data/scores-nonorm-dev') dev_ztnorm = pkg_resources.resource_filename('bob.bio.base.test', 'data/scores-ztnorm-dev') with runner.isolated_filesystem(): result = runner.invoke( commands.cmc, ['-v', '--legends', 'A', '-e', dev_nonorm, dev_ztnorm]) if result.output: click.echo(result.output) assert_click_runner_result(result)
def test_roc(): dev1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/dev-4col.txt') runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.roc, ['--output', 'test.pdf',dev1]) if result.output: click.echo(result.output) assert_click_runner_result(result) dev2 = pkg_resources.resource_filename('bob.bio.base.test', 'data/dev-5col.txt') test1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/test-4col.txt') test2 = pkg_resources.resource_filename('bob.bio.base.test', 'data/test-5col.txt') with runner.isolated_filesystem(): result = runner.invoke(commands.roc, ['--split', '--output', 'test.pdf', '-S', '-ll', 'lower-left', '-e', dev1, test1, dev2, test2]) if result.output: click.echo(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke(commands.roc, ['--output', 'test.pdf', '-e', '--legends', 'A,B', dev1, test1, dev2, test2]) if result.output: click.echo(result.output) assert_click_runner_result(result) dev_nonorm = pkg_resources.resource_filename('bob.bio.base.test', 'data/scores-nonorm-dev') dev_ztnorm = pkg_resources.resource_filename('bob.bio.base.test', 'data/scores-ztnorm-dev') with runner.isolated_filesystem(): result = runner.invoke(commands.roc, [ '--min-far-value', '1e-6', '--lines-at', '1e-5', '-v', '--legends', 'A', '-e', dev_nonorm, dev_ztnorm ]) if result.output: click.echo(result.output) assert_click_runner_result(result)
def _test_pipeline_click_cli( cli, options, expected_outputs=("results/scores-dev.csv",) ): runner = CliRunner() with runner.isolated_filesystem(): _create_test_config("config.py") result = runner.invoke( cli, [ "-vv", "config.py", ] + options, ) assert_click_runner_result(result) # check for expected_output output_files = glob.glob("results/**", recursive=True) err_msg = "Found only:\n{output_files}\nin output directory given the options: {options}".format( output_files="\n".join(output_files), options=options ) for out in expected_outputs: assert os.path.isfile(out), err_msg
def test_hist_legends(): dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure') test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure') dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure') test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure') runner = CliRunner() # share same title for dev/eval of each system with runner.isolated_filesystem(): result = runner.invoke(commands.hist, ['-e', '-sp', 111, '-ts', 'A,B', dev1, test1, dev2, test2]) if result.output: click.echo(result.output) assert_click_runner_result(result) # individual titles for dev and eval with runner.isolated_filesystem(): result = runner.invoke(commands.hist, ['-e', '-sp', 121, '-ts', 'A,B,C,D', dev1, test1, dev2, test2]) if result.output: click.echo(result.output) assert_click_runner_result(result)
def test_hist(): dev1 = _F("dev-1.txt") test1 = _F("test-1.txt") dev2 = _F("dev-2.txt") test2 = _F("test-2.txt") runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.hist, [dev1]) if result.output: click.echo(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.hist, [ "--criterion", "min-hter", "--no-line", "--output", "HISTO.pdf", "-b", "30,100", dev1, dev2, ], ) if result.output: click.echo(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.hist, [ "-e", "--criterion", "eer", "--output", "HISTO.pdf", "-b", "30,20", "-sp", 111, "-ts", "A,B", dev1, test1, dev2, test2, ], ) if result.output: click.echo(result.output) assert_click_runner_result(result)
def test_roc(): dev1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/dev-4col.txt') runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.roc, ['--output', 'test.pdf', dev1]) if result.output: click.echo(result.output) assert_click_runner_result(result) dev2 = pkg_resources.resource_filename('bob.bio.base.test', 'data/dev-5col.txt') test1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/test-4col.txt') test2 = pkg_resources.resource_filename('bob.bio.base.test', 'data/test-5col.txt') with runner.isolated_filesystem(): result = runner.invoke(commands.roc, [ '--split', '--output', 'test.pdf', '-S', '-lc', 'lower-left', '-e', dev1, test1, dev2, test2 ]) if result.output: click.echo(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke(commands.roc, [ '--output', 'test.pdf', '-e', '--legends', 'A,B', dev1, test1, dev2, test2 ]) if result.output: click.echo(result.output) assert_click_runner_result(result) dev_nonorm = pkg_resources.resource_filename('bob.bio.base.test', 'data/scores-nonorm-dev') dev_ztnorm = pkg_resources.resource_filename('bob.bio.base.test', 'data/scores-ztnorm-dev') with runner.isolated_filesystem(): result = runner.invoke(commands.roc, [ '--min-far-value', '1e-6', '--lines-at', '1e-5', '-v', '--legends', 'A', '-e', dev_nonorm, dev_ztnorm ]) if result.output: click.echo(result.output) assert_click_runner_result(result)
def test_epc(): dev1 = pkg_resources.resource_filename( "bob.bio.base.test", "data/dev-4col.txt" ) test1 = pkg_resources.resource_filename( "bob.bio.base.test", "data/test-4col.txt" ) runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.epc, [dev1, test1]) if result.output: click.echo(result.output) assert_click_runner_result(result) dev2 = pkg_resources.resource_filename( "bob.bio.base.test", "data/dev-4col.tar.gz" ) test2 = pkg_resources.resource_filename( "bob.bio.base.test", "data/test-5col.txt" ) with runner.isolated_filesystem(): result = runner.invoke( commands.epc, [ "--output", "test.pdf", "--legends", "A,B", "-S", dev1, test1, dev2, test2, ], ) if result.output: click.echo(result.output) assert_click_runner_result(result) dev_nonorm = pkg_resources.resource_filename( "bob.bio.base.test", "data/scores-nonorm-dev" ) dev_ztnorm = pkg_resources.resource_filename( "bob.bio.base.test", "data/scores-ztnorm-dev" ) with runner.isolated_filesystem(): result = runner.invoke( commands.epc, ["-v", "--legends", "A", dev_nonorm, dev_ztnorm] ) if result.output: click.echo(result.output) assert_click_runner_result(result)
def test_roc(): dev1 = _F("dev-1.txt") runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.roc, ["--output", "test.pdf", dev1]) if result.output: click.echo(result.output) assert_click_runner_result(result) dev2 = _F("dev-2.txt") test1 = _F("test-1.txt") test2 = _F("test-2.txt") with runner.isolated_filesystem(): result = runner.invoke( commands.roc, [ "--split", "--output", "test.pdf", "-e", "-ts", "A,", dev1, test1, dev2, test2, ], ) if result.output: click.echo(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.roc, [ "-e", "--output", "test.pdf", "--legends", "A,B", dev1, test1, dev2, test2, ], ) if result.output: click.echo(result.output) assert_click_runner_result(result)
def test_metrics_vuln(): dev1 = pkg_resources.resource_filename( "bob.bio.base.test", "data/vuln/scores-dev.csv" ) runner = CliRunner() result = runner.invoke(vuln_commands.metrics, [dev1]) with runner.isolated_filesystem(): with open("tmp", "w") as f: f.write(result.output) assert_click_runner_result(result) dev2 = pkg_resources.resource_filename( "bob.bio.base.test", "data/vuln/scores-dev-med.csv" ) test1 = pkg_resources.resource_filename( "bob.bio.base.test", "data/vuln/scores-eval.csv" ) test2 = pkg_resources.resource_filename( "bob.bio.base.test", "data/vuln/scores-eval-med.csv" ) with runner.isolated_filesystem(): result = runner.invoke( vuln_commands.metrics, ["-e", dev1, test1, dev2, test2] ) with open("tmp", "w") as f: f.write(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( vuln_commands.metrics, ["-e", "-l", "tmp", "-lg", "A,B", dev1, test1, dev2, test2], ) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( vuln_commands.metrics, ["-e", "-l", "tmp", dev1, test2] ) assert_click_runner_result(result)
def test_evaluate(): dev1 = bob.io.base.test_utils.datafile('dev-1.txt', 'bob.measure') test1 = bob.io.base.test_utils.datafile('test-1.txt', 'bob.measure') dev2 = bob.io.base.test_utils.datafile('dev-2.txt', 'bob.measure') test2 = bob.io.base.test_utils.datafile('test-2.txt', 'bob.measure') runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.evaluate, [dev1]) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.evaluate, ['--output', 'my_plots.pdf', '-n', 300, dev1, dev2]) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.evaluate, ['-e', dev1, test1, dev2, test2]) assert_click_runner_result(result)
def test_evaluate(): dev1 = _F("dev-1.txt") test1 = _F("test-1.txt") dev2 = _F("dev-2.txt") test2 = _F("test-2.txt") runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.evaluate, [dev1]) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.evaluate, ["--output", "my_plots.pdf", "-n", 300, dev1, dev2], ) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke(commands.evaluate, ["-e", dev1, test1, dev2, test2]) assert_click_runner_result(result)
def test_det(): dev1 = pkg_resources.resource_filename( "bob.bio.base.test", "data/dev-4col.txt" ) runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke(commands.det, [dev1, "-S"]) if result.output: click.echo(result.output) assert_click_runner_result(result) dev2 = pkg_resources.resource_filename( "bob.bio.base.test", "data/dev-5col.txt" ) test1 = pkg_resources.resource_filename( "bob.bio.base.test", "data/test-4col.txt" ) test2 = pkg_resources.resource_filename( "bob.bio.base.test", "data/test-5col.txt" ) with runner.isolated_filesystem(): result = runner.invoke( commands.det, [ "--split", "--output", "test.pdf", "--legends", "A,B", "-e", dev1, test1, dev2, test2, ], ) if result.output: click.echo(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.det, ["--output", "test.pdf", "-e", dev1, test1, dev2, test2], ) if result.output: click.echo(result.output) assert_click_runner_result(result) dev_nonorm = pkg_resources.resource_filename( "bob.bio.base.test", "data/scores-nonorm-dev" ) dev_ztnorm = pkg_resources.resource_filename( "bob.bio.base.test", "data/scores-ztnorm-dev" ) with runner.isolated_filesystem(): result = runner.invoke( commands.det, [ "--min-far-value", "1e-6", "--lines-at", "1e-5", "-e", "-v", "--legends", "A", dev_nonorm, dev_ztnorm, ], ) if result.output: click.echo(result.output) assert_click_runner_result(result)
def gen_case( n_subjects, n_probes_per_subject, n_unknown_subjects, n_pos=None, n_neg=None, n_unk=None, ): """Tests one case of the gen command""" n_pos = n_subjects * n_probes_per_subject if n_pos is None else n_pos n_neg = (n_subjects * (n_subjects - 1) * n_probes_per_subject if n_neg is None else n_neg) n_unk = (n_unknown_subjects * n_subjects * n_probes_per_subject if n_unk is None else n_unk) runner = CliRunner() with runner.isolated_filesystem(): temp_path = "./gen_test_temp_dir/" logger.info("Calling 'gen' with a specific amount of scores.") result = runner.invoke( gen, args=[ "-mm", "10", "-mnm", "-10", "-sp", "1", "-sn", "1", "-p", f"{n_probes_per_subject}", "-s", f"{n_subjects}", "-u", f"{n_unknown_subjects}", "-f", "--n-pos", f"{n_pos}", "--n-neg", f"{n_neg}", "--n-unk", f"{n_unk}", f"{temp_path}", ], ) assert_click_runner_result(result) assert os.path.exists(os.path.join( temp_path, "scores-dev.csv")), "dev scores file not created." line_count = 0 with open(os.path.join(temp_path, "scores-dev.csv")) as f: for _ in f: line_count += 1 assert line_count == 1 + n_pos + n_neg + n_unk assert os.path.exists(os.path.join( temp_path, "scores-eval.csv")), "eval scores file not created." line_count = 0 with open(os.path.join(temp_path, "scores-eval.csv")) as f: for _ in f: line_count += 1 assert line_count == 1 + n_pos + n_neg + n_unk
def test_metrics(): dev1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/dev-4col.txt') runner = CliRunner() result = runner.invoke(commands.metrics, [dev1]) with runner.isolated_filesystem(): with open('tmp', 'w') as f: f.write(result.output) assert_click_runner_result(result) dev2 = pkg_resources.resource_filename('bob.bio.base.test', 'data/dev-5col.txt') test1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/test-4col.txt') test2 = pkg_resources.resource_filename('bob.bio.base.test', 'data/test-5col.txt') with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ['-e', dev1, test1, dev2, test2] ) with open('tmp', 'w') as f: f.write(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ['-e', '-l', 'tmp', '-lg', 'A,B', dev1, test1, dev2, test2] ) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ['-e', '-l', 'tmp', dev1, test2] ) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ['-e', '-l', 'tmp', '-T', '0.1', '--criterion', 'mindcf', '--cost', 0.9, dev1, test2] ) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ['-l', 'tmp', '--criterion', 'mindcf', '--cost', 0.9, dev1] ) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ['-e', '--criterion', 'cllr', dev1, test2] ) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ['-l', 'tmp', '--criterion', 'cllr', '--cost', 0.9, dev1] ) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ['-e', '--criterion', 'rr', '-T', '0.1', dev1, test2] ) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ['-l', 'tmp', '--criterion', 'rr', dev1, dev2] ) assert_click_runner_result(result)
def test_metrics(): dev1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/dev-4col.txt') runner = CliRunner() result = runner.invoke(commands.metrics, [dev1]) with runner.isolated_filesystem(): with open('tmp', 'w') as f: f.write(result.output) assert_click_runner_result(result) dev2 = pkg_resources.resource_filename('bob.bio.base.test', 'data/dev-5col.txt') test1 = pkg_resources.resource_filename('bob.bio.base.test', 'data/test-4col.txt') test2 = pkg_resources.resource_filename('bob.bio.base.test', 'data/test-5col.txt') with runner.isolated_filesystem(): result = runner.invoke(commands.metrics, ['-e', dev1, test1, dev2, test2]) with open('tmp', 'w') as f: f.write(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ['-e', '-l', 'tmp', '-lg', 'A,B', dev1, test1, dev2, test2]) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke(commands.metrics, ['-e', '-l', 'tmp', dev1, test2]) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke(commands.metrics, [ '-e', '-l', 'tmp', '-T', '0.1', '--criterion', 'mindcf', '--cost', 0.9, dev1, test2 ]) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ['-l', 'tmp', '--criterion', 'mindcf', '--cost', 0.9, dev1]) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke(commands.metrics, ['-e', '--criterion', 'cllr', dev1, test2]) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ['-l', 'tmp', '--criterion', 'cllr', '--cost', 0.9, dev1]) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ['-e', '--criterion', 'rr', '-T', '0.1', dev1, test2]) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke(commands.metrics, ['-l', 'tmp', '--criterion', 'rr', dev1, dev2]) assert_click_runner_result(result)
def test_metrics(): dev1 = pkg_resources.resource_filename( "bob.bio.base.test", "data/dev-4col.txt" ) runner = CliRunner() result = runner.invoke(commands.metrics, [dev1]) with runner.isolated_filesystem(): with open("tmp", "w") as f: f.write(result.output) assert_click_runner_result(result) dev2 = pkg_resources.resource_filename( "bob.bio.base.test", "data/dev-5col.txt" ) test1 = pkg_resources.resource_filename( "bob.bio.base.test", "data/test-4col.txt" ) test2 = pkg_resources.resource_filename( "bob.bio.base.test", "data/test-5col.txt" ) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ["-e", dev1, test1, dev2, test2] ) with open("tmp", "w") as f: f.write(result.output) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ["-e", "-l", "tmp", "-lg", "A,B", dev1, test1, dev2, test2], ) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ["-e", "-l", "tmp", dev1, test2] ) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, [ "-e", "-l", "tmp", "-T", "0.1", "--criterion", "mindcf", "--cost", 0.9, dev1, test2, ], ) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ["-l", "tmp", "--criterion", "mindcf", "--cost", 0.9, dev1], ) assert_click_runner_result(result) # with runner.isolated_filesystem(): # result = runner.invoke( # commands.metrics, ["-e", "--criterion", "cllr", dev1, test2] # ) # assert_click_runner_result(result) # with runner.isolated_filesystem(): # result = runner.invoke( # commands.metrics, ["-l", "tmp", "--criterion", "cllr", "--cost", 0.9, dev1] # ) # assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ["-e", "--criterion", "rr", "-T", "0.1", dev1, test2], ) assert_click_runner_result(result) with runner.isolated_filesystem(): result = runner.invoke( commands.metrics, ["-l", "tmp", "--criterion", "rr", dev1, dev2] ) assert_click_runner_result(result)