def test_fusion():
# tests that the fuse_scores script is doing something useful
test_dir = tempfile.mkdtemp(prefix='bobtest_')
reference_files = [os.path.join(data_dir, s) for s in ('scores-nonorm-dev', 'scores-ztnorm-dev')]
output_files = [os.path.join(test_dir, s) for s in ("fused-dev", "fused-eval")]
parameters = [
'--dev-files', reference_files[0], reference_files[1],
'--eval-files', reference_files[0], reference_files[1],
'--fused-dev-file', output_files[0],
'--fused-eval-file', output_files[1],
'--max-iterations', '100',
'--convergence-threshold', '1e-4',
'-v'
]
# execute the script
from bob.bio.base.script.fuse_scores import main
try:
main(parameters)
# assert that we can read the two files, and that they contain the same number of lines as the original file
for i in (0,1):
assert os.path.exists(output_files[i])
r = score.four_column(reference_files[i])
o = score.four_column(output_files[i])
assert len(list(r)) == len(list(o))
finally:
shutil.rmtree(test_dir)
def test_fusion():
# tests that the fuse_scores script is doing something useful
test_dir = tempfile.mkdtemp(prefix='bobtest_')
reference_files = [
os.path.join(data_dir, s)
for s in ('scores-nonorm-dev', 'scores-ztnorm-dev')
]
output_files = [
os.path.join(test_dir, s) for s in ("fused-dev", "fused-eval")
]
parameters = [
'--dev-files', reference_files[0], reference_files[1], '--eval-files',
reference_files[0], reference_files[1], '--fused-dev-file',
output_files[0], '--fused-eval-file', output_files[1],
'--max-iterations', '100', '--convergence-threshold', '1e-4', '-v'
]
# execute the script
from bob.bio.base.script.fuse_scores import main
try:
main(parameters)
# assert that we can read the two files, and that they contain the same number of lines as the original file
for i in (0, 1):
assert os.path.exists(output_files[i])
r = score.four_column(reference_files[i])
o = score.four_column(output_files[i])
assert len(list(r)) == len(list(o))
finally:
shutil.rmtree(test_dir)
def test_evaluate():
# tests our 'evaluate' script using the reference files
test_dir = tempfile.mkdtemp(prefix='bobtest_')
reference_files = ('scores-nonorm-dev', 'scores-ztnorm-dev')
plots = [os.path.join(test_dir, '%s.pdf')%f for f in ['roc', 'cmc', 'det']]
parameters = [
'--dev-files', reference_files[0], reference_files[1],
'--eval-files', reference_files[0], reference_files[1],
'--directory', os.path.join(data_dir),
'--legends', 'no norm', 'ZT norm',
'--criterion', 'HTER',
'--roc', plots[0],
'--det', plots[1],
'--cmc', plots[2],
'--rr',
'--thresholds', '5000', '0',
'-v',
]
# execute the script
from bob.bio.base.script.evaluate import main
main(parameters)
for i in range(3):
assert os.path.exists(plots[i])
os.remove(plots[i])
os.rmdir(test_dir)
def test_verify_execute_only():
test_dir = tempfile.mkdtemp(prefix='bobtest_')
# define dummy parameters
parameters = [
'-d', 'dummy',
'-p', 'dummy',
'-e', 'dummy',
'-a', 'dummy',
'--zt-norm',
'--allow-missing-files',
'-vs', 'test_missing',
'--temp-directory', test_dir,
'--result-directory', test_dir,
'--preferred-package', 'bob.bio.base',
'--imports', 'bob.bio.base.test.dummy',
'--execute-only', 'preprocessing', 'score-computation',
'--dry-run'
]
try:
from bob.bio.base.script.verify import main
main(parameters)
finally:
if os.path.exists(test_dir):
shutil.rmtree(test_dir)
def test_verify_filelist():
try:
import bob.db.verification.filelist
except ImportError:
raise SkipTest(
"Skipping test since bob.db.verification.filelist is not available"
)
test_dir = tempfile.mkdtemp(prefix='bobtest_')
# define dummy parameters
parameters = [
'-d',
os.path.join(dummy_dir, 'filelist.py'), '-p', 'dummy', '-e', 'dummy',
'-a', 'dummy', '--zt-norm', '-vs', 'test_filelist', '--temp-directory',
test_dir, '--result-directory', test_dir, '--preferred-package',
'bob.bio.base'
]
print(bob.bio.base.tools.command_line(parameters))
try:
from bob.bio.base.script.verify import main
main(parameters)
# assert that the score file exists
score_files = [
os.path.join(test_dir, 'test_filelist', 'None', norm, 'scores-dev')
for norm in ('nonorm', 'ztnorm')
]
assert os.path.exists(
score_files[0]), "Score file %s does not exist" % score_files[0]
assert os.path.exists(
score_files[1]), "Score file %s does not exist" % score_files[1]
# assert that the scores are are identical (might be in a different order, though
reference_files = [
os.path.join(data_dir, 'scores-%s-dev' % norm)
for norm in ('nonorm', 'ztnorm')
]
for i in (0, 1):
# load scores
a1, b1 = bob.measure.load.split_four_column(score_files[i])
a2, b2 = bob.measure.load.split_four_column(reference_files[i])
# sort scores
a1 = sorted(a1)
a2 = sorted(a2)
b1 = sorted(b1)
b2 = sorted(b2)
# assert that scores are almost equal
for i in range(len(a1)):
abs(a1[i] - a2[i]) < 1e-6
for i in range(len(b1)):
abs(b1[i] - b2[i]) < 1e-6
finally:
shutil.rmtree(test_dir)
def test_collect_results():
# simply test that the collect_results script works
test_dir = tempfile.mkdtemp(prefix='bobtest_')
try:
from bob.bio.base.script.collect_results import main
main(['--directory', test_dir, '--sort', '--sort-key', 'dir', '--criterion', 'FAR', '--self-test'])
finally:
if os.path.exists(test_dir):
os.rmdir(test_dir)
def _verify(parameters,
test_dir,
sub_dir,
ref_modifier="",
score_modifier=('scores', ''),
counts=3,
check_zt=True):
from bob.bio.base.script.verify import main
try:
main(parameters)
Range = (0, 1) if check_zt else (0, )
# assert that the score file exists
score_files = [
os.path.join(test_dir, sub_dir, 'Default', norm,
'%s-dev%s' % score_modifier)
for norm in ('nonorm', 'ztnorm')
]
for i in Range:
assert os.path.exists(
score_files[i]
), "Score file %s does not exist" % score_files[i]
# also assert that the scores are still the same -- though they have no real meaning
reference_files = [
os.path.join(data_dir, 'scores-%s%s-dev' % (norm, ref_modifier))
for norm in ('nonorm', 'ztnorm')
]
if regenerate_reference:
for i in Range:
shutil.copy(score_files[i], reference_files[i])
for i in Range:
d = []
# read reference and new data
for score_file in (score_files[i], reference_files[i]):
f = score.open_file(score_file)
d_ = []
for line in f:
if isinstance(line, bytes): line = line.decode('utf-8')
d_.append(line.rstrip().split())
d.append(numpy.array(d_))
assert d[0].shape == d[1].shape
# assert that the data order is still correct
assert (d[0][:, 0:counts] == d[1][:, 0:counts]).all()
# assert that the values are OK
assert numpy.allclose(d[0][:, counts].astype(float),
d[1][:, counts].astype(float), 1e-5)
assert not os.path.exists(os.path.join(test_dir, 'submitted.sql3'))
finally:
shutil.rmtree(test_dir)
def test_collect_results():
# simply test that the collect_results script works
test_dir = tempfile.mkdtemp(prefix='bobtest_')
try:
from bob.bio.base.script.collect_results import main
main([
'--directory', test_dir, '--sort', '--sort-key', 'dir',
'--criterion', 'FAR', '--self-test'
])
finally:
if os.path.exists(test_dir):
os.rmdir(test_dir)
def test_verify_filelist():
try:
import bob.db.verification.filelist
except ImportError:
raise SkipTest("Skipping test since bob.db.verification.filelist is not available")
test_dir = tempfile.mkdtemp(prefix='bobtest_')
# define dummy parameters
parameters = [
'-d', os.path.join(dummy_dir, 'filelist.py'),
'-p', 'dummy',
'-e', 'dummy',
'-a', 'dummy',
'--zt-norm',
'-vs', 'test_filelist',
'--temp-directory', test_dir,
'--result-directory', test_dir,
'--preferred-package', 'bob.bio.base'
]
print (bob.bio.base.tools.command_line(parameters))
try:
from bob.bio.base.script.verify import main
main(parameters)
# assert that the score file exists
score_files = [os.path.join(test_dir, 'test_filelist', 'None', norm, 'scores-dev') for norm in ('nonorm', 'ztnorm')]
assert os.path.exists(score_files[0]), "Score file %s does not exist" % score_files[0]
assert os.path.exists(score_files[1]), "Score file %s does not exist" % score_files[1]
# assert that the scores are are identical (might be in a different order, though
reference_files = [os.path.join(data_dir, 'scores-%s-dev' % norm) for norm in ('nonorm', 'ztnorm')]
for i in (0,1):
# load scores
a1, b1 = bob.measure.load.split_four_column(score_files[i])
a2, b2 = bob.measure.load.split_four_column(reference_files[i])
# sort scores
a1 = sorted(a1); a2 = sorted(a2); b1 = sorted(b1); b2 = sorted(b2)
# assert that scores are almost equal
for i in range(len(a1)):
abs(a1[i] - a2[i]) < 1e-6
for i in range(len(b1)):
abs(b1[i] - b2[i]) < 1e-6
finally:
shutil.rmtree(test_dir)
def test_verify_filelist():
test_dir = tempfile.mkdtemp(prefix='bobtest_')
# define dummy parameters
parameters = [
'-d',
os.path.join(dummy_dir, 'filelist.py'), '-p', 'dummy', '-e', 'dummy',
'-a', 'dummy', '--zt-norm', '-vs', 'test_filelist', '--temp-directory',
test_dir, '--result-directory', test_dir, '--preferred-package',
'bob.bio.base'
]
from bob.bio.base.script.verify import main
try:
main(parameters)
# assert that the score file exists
score_files = [
os.path.join(test_dir, 'test_filelist', 'None', norm, 'scores-dev')
for norm in ('nonorm', 'ztnorm')
]
assert os.path.exists(
score_files[0]), "Score file %s does not exist" % score_files[0]
assert os.path.exists(
score_files[1]), "Score file %s does not exist" % score_files[1]
# assert that the scores are are identical (might be in a different order, though
reference_files = [
os.path.join(data_dir, 'scores-%s-dev' % norm)
for norm in ('nonorm', 'ztnorm')
]
for i in (0, 1):
# load scores
a1, b1 = score.split_four_column(score_files[i])
a2, b2 = score.split_four_column(reference_files[i])
# sort scores
a1 = sorted(a1)
a2 = sorted(a2)
b1 = sorted(b1)
b2 = sorted(b2)
# assert that scores are almost equal
assert all(abs(a1[j] - a2[j]) < 1e-6 for j in range(len(a1)))
assert all(abs(b1[j] - b2[j]) < 1e-6 for j in range(len(b1)))
finally:
shutil.rmtree(test_dir)
def test_verify_execute_only():
test_dir = tempfile.mkdtemp(prefix='bobtest_')
# define dummy parameters
parameters = [
'-d', 'dummy', '-p', 'dummy', '-e', 'dummy', '-a', 'dummy',
'--zt-norm', '--allow-missing-files', '-vs', 'test_missing',
'--temp-directory', test_dir, '--result-directory', test_dir,
'--preferred-package', 'bob.bio.base', '--imports',
'bob.bio.base.test.dummy', '--execute-only', 'preprocessing',
'score-computation', '--dry-run'
]
try:
from bob.bio.base.script.verify import main
main(parameters)
finally:
if os.path.exists(test_dir):
shutil.rmtree(test_dir)
def test_evaluate():
# tests our 'evaluate' script using the reference files
test_dir = tempfile.mkdtemp(prefix='bobtest_')
reference_files = ('scores-nonorm-dev', 'scores-ztnorm-dev')
plots = [
os.path.join(test_dir, '%s.pdf') % f for f in ['roc', 'cmc', 'det']
]
parameters = [
'--dev-files',
reference_files[0],
reference_files[1],
'--eval-files',
reference_files[0],
reference_files[1],
'--directory',
os.path.join(data_dir),
'--legends',
'no norm',
'ZT norm',
'--criterion',
'HTER',
'--roc',
plots[0],
'--det',
plots[1],
'--cmc',
plots[2],
'--rr',
'--thresholds',
'5000',
'0',
'-v',
]
# execute the script
from bob.bio.base.script.evaluate import main
main(parameters)
for i in range(3):
assert os.path.exists(plots[i])
os.remove(plots[i])
os.rmdir(test_dir)
def test_verify_filelist():
test_dir = tempfile.mkdtemp(prefix='bobtest_')
# define dummy parameters
parameters = [
'-d', os.path.join(dummy_dir, 'filelist.py'),
'-p', 'dummy',
'-e', 'dummy',
'-a', 'dummy',
'--zt-norm',
'-vs', 'test_filelist',
'--temp-directory', test_dir,
'--result-directory', test_dir,
'--preferred-package', 'bob.bio.base'
]
from bob.bio.base.script.verify import main
try:
main(parameters)
# assert that the score file exists
score_files = [os.path.join(test_dir, 'test_filelist', 'None', norm, 'scores-dev') for norm in ('nonorm', 'ztnorm')]
assert os.path.exists(score_files[0]), "Score file %s does not exist" % score_files[0]
assert os.path.exists(score_files[1]), "Score file %s does not exist" % score_files[1]
# assert that the scores are are identical (might be in a different order, though
reference_files = [os.path.join(data_dir, 'scores-%s-dev' % norm) for norm in ('nonorm', 'ztnorm')]
for i in (0,1):
# load scores
a1, b1 = score.split_four_column(score_files[i])
a2, b2 = score.split_four_column(reference_files[i])
# sort scores
a1 = sorted(a1); a2 = sorted(a2); b1 = sorted(b1); b2 = sorted(b2)
# assert that scores are almost equal
assert all(abs(a1[j] - a2[j]) < 1e-6 for j in range(len(a1)))
assert all(abs(b1[j] - b2[j]) < 1e-6 for j in range(len(b1)))
finally:
shutil.rmtree(test_dir)
def test_collect_results():
# simply test that the collect_results script works
from bob.bio.base.script.collect_results import main
# FAR criterion
main([
'-D', data_dir, '-d', 'scores-nonorm-dev', '-e',
'scores-nonorm-fivecol-dev', '-n', '.', '-z', '.', '--sort',
'--sort-key', 'dir', '--criterion', 'FAR', '--far-threshold', '0.1',
'--self-test', '-v'
])
# Recognition Rate
main([
'-D', data_dir, '-d', 'scores-nonorm-dev', '-e',
'scores-nonorm-fivecol-dev', '-n', '.', '-z', '.', '--sort',
'--sort-key', 'dir', '--criterion', 'RR', '--rank', '10',
'--self-test', '-v'
])
# DIR
main([
'-D', data_dir, '-d', 'scores-nonorm-openset-dev', '-n', '.', '-z',
'.', '--sort', '--sort-key', 'dir', '--criterion', 'DIR',
'--far-threshold', '0.1', '--self-test', '-v'
])
def _verify(parameters, test_dir, sub_dir, ref_modifier="", score_modifier=('scores',''), counts=3, check_zt=True):
from bob.bio.base.script.verify import main
try:
main(parameters)
Range = (0,1) if check_zt else (0,)
# assert that the score file exists
score_files = [os.path.join(test_dir, sub_dir, 'Default', norm, '%s-dev%s'%score_modifier) for norm in ('nonorm', 'ztnorm')]
for i in Range:
assert os.path.exists(score_files[i]), "Score file %s does not exist" % score_files[i]
# also assert that the scores are still the same -- though they have no real meaning
reference_files = [os.path.join(data_dir, 'scores-%s%s-dev'%(norm, ref_modifier)) for norm in ('nonorm', 'ztnorm')]
if regenerate_reference:
for i in Range:
shutil.copy(score_files[i], reference_files[i])
for i in Range:
d = []
# read reference and new data
for score_file in (score_files[i], reference_files[i]):
f = score.open_file(score_file)
d_ = []
for line in f:
if isinstance(line, bytes): line = line.decode('utf-8')
d_.append(line.rstrip().split())
d.append(numpy.array(d_))
assert d[0].shape == d[1].shape
# assert that the data order is still correct
assert (d[0][:,0:counts] == d[1][:, 0:counts]).all()
# assert that the values are OK
assert numpy.allclose(d[0][:,counts].astype(float), d[1][:,counts].astype(float), 1e-5)
assert not os.path.exists(os.path.join(test_dir, 'submitted.sql3'))
finally:
shutil.rmtree(test_dir)
def test_collect_results():
# simply test that the collect_results script works
from bob.bio.base.script.collect_results import main
# FAR criterion
main([
'-D', data_dir,
'-d', 'scores-nonorm-dev',
'-e', 'scores-nonorm-fivecol-dev',
'-n', '.', '-z', '.',
'--sort', '--sort-key', 'dir',
'--criterion', 'FAR', '--far-threshold', '0.1',
'--self-test', '-v'
])
# Recognition Rate
main([
'-D', data_dir,
'-d', 'scores-nonorm-dev',
'-e', 'scores-nonorm-fivecol-dev',
'-n', '.', '-z', '.',
'--sort', '--sort-key', 'dir',
'--criterion', 'RR', '--rank', '10',
'--self-test', '-v'
])
# DIR
main([
'-D', data_dir,
'-d', 'scores-nonorm-openset-dev',
'-n', '.', '-z', '.',
'--sort', '--sort-key', 'dir',
'--criterion', 'DIR', '--far-threshold', '0.1',
'--self-test', '-v'
])
def test_verify_missing():
test_dir = tempfile.mkdtemp(prefix='bobtest_')
# define dummy parameters
parameters = [
'-d', 'dummy', '-p',
'bob.bio.base.test.dummy.preprocessor.DummyPreprocessor(return_none=True)',
'-e', 'dummy', '-a', 'dummy', '--zt-norm', '--allow-missing-files',
'-vs', 'test_missing', '--temp-directory', test_dir,
'--result-directory', test_dir, '--preferred-package', 'bob.bio.base',
'--imports', 'bob.bio.base.test.dummy'
]
print(bob.bio.base.tools.command_line(parameters))
try:
from bob.bio.base.script.verify import main
main(parameters)
# assert that the score file exists
score_files = [
os.path.join(test_dir, 'test_missing', 'Default', norm,
'scores-dev') for norm in ('nonorm', 'ztnorm')
]
assert os.path.exists(
score_files[0]), "Score file %s does not exist" % score_files[0]
assert os.path.exists(
score_files[1]), "Score file %s does not exist" % score_files[1]
# assert that all scores are NaN
for i in (0, 1):
# load scores
a, b = bob.measure.load.split_four_column(score_files[i])
assert numpy.all(numpy.isnan(a))
assert numpy.all(numpy.isnan(b))
finally:
shutil.rmtree(test_dir)
def test_verify_missing():
test_dir = tempfile.mkdtemp(prefix='bobtest_')
# define dummy parameters
parameters = [
'-d', 'dummy',
'-p', 'bob.bio.base.test.dummy.preprocessor.DummyPreprocessor(return_none=True)',
'-e', 'dummy',
'-a', 'dummy',
'--zt-norm',
'--allow-missing-files',
'-vs', 'test_missing',
'--temp-directory', test_dir,
'--result-directory', test_dir,
'--preferred-package', 'bob.bio.base',
'--imports', 'bob.bio.base.test.dummy'
]
print (bob.bio.base.tools.command_line(parameters))
try:
from bob.bio.base.script.verify import main
main(parameters)
# assert that the score file exists
score_files = [os.path.join(test_dir, 'test_missing', 'Default', norm, 'scores-dev') for norm in ('nonorm', 'ztnorm')]
assert os.path.exists(score_files[0]), "Score file %s does not exist" % score_files[0]
assert os.path.exists(score_files[1]), "Score file %s does not exist" % score_files[1]
# assert that all scores are NaN
for i in (0,1):
# load scores
a, b = bob.measure.load.split_four_column(score_files[i])
assert numpy.all(numpy.isnan(a))
assert numpy.all(numpy.isnan(b))
finally:
shutil.rmtree(test_dir)
def test_grid_search():
test_dir = tempfile.mkdtemp(prefix='bobtest_')
# tests that the parameter_test.py script works properly
try:
# first test without grid option
parameters = [
'-c',
os.path.join(dummy_dir, 'grid_search.py'), '-d', 'dummy', '-e',
'dummy', '-s', 'test_grid_search', '-T', test_dir, '-R', test_dir,
'-v', '--', '--dry-run', '--preferred-package', 'bob.bio.base'
]
from bob.bio.base.script.grid_search import main
with utils.Quiet():
main(parameters)
# number of jobs should be 12
assert bob.bio.base.script.grid_search.task_count == 6
# but no job in the grid
assert bob.bio.base.script.grid_search.job_count == 0
# now, in the grid...
parameters = [
'-c',
os.path.join(dummy_dir, 'grid_search.py'), '-d', 'dummy', '-s',
'test_grid_search', '-i', '.', '-G', test_dir, '-T', test_dir,
'-R', test_dir, '-w', 'Job.txt', '-g', 'grid', '-v', '--',
'--dry-run', '--preferred-package', 'bob.bio.base'
]
with utils.Quiet():
main(parameters)
# number of jobs should be 12
assert bob.bio.base.script.grid_search.task_count == 6
# number of jobs in the grid: 36 (including best possible re-use of files; minus preprocessing)
assert bob.bio.base.script.grid_search.job_count == 30
# and now, finally run locally
parameters = [
'-c',
os.path.join(dummy_dir, 'grid_search.py'), '-d', 'dummy', '-s',
'test_grid_search', '-G', test_dir, '-T', test_dir, '-R', test_dir,
'-w', 'Job.txt', '-l', '4', '-L', '-1', '-v', '--', '--imports',
'bob.io.image', '--dry-run', '--preferred-package', 'bob.bio.base'
]
with utils.Quiet():
main(parameters)
# number of jobs should be 12
assert bob.bio.base.script.grid_search.task_count == 6
# number of jobs in the grid: 36 (including best possible re-use of files; minus preprocessing)
assert bob.bio.base.script.grid_search.job_count == 0
finally:
shutil.rmtree(test_dir)
#!/usr/bin/python
# -*- coding: utf-8 -*-
import re
import sys
from bob.bio.base.script.verify import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
sys.exit(main())
def test_grid_search():
test_dir = tempfile.mkdtemp(prefix='bobtest_')
# tests that the parameter_test.py script works properly
try:
# first test without grid option
parameters = [
'-c', os.path.join(dummy_dir, 'grid_search.py'),
'-d', 'dummy',
'-e', 'dummy',
'-s', 'test_grid_search',
'-T', test_dir,
'-R', test_dir,
'-v',
'--', '--dry-run',
]
from bob.bio.base.script.grid_search import main
with utils.Quiet():
main(parameters)
# number of jobs should be 12
assert bob.bio.base.script.grid_search.task_count == 6
# but no job in the grid
assert bob.bio.base.script.grid_search.job_count == 0
# now, in the grid...
parameters = [
'-c', os.path.join(dummy_dir, 'grid_search.py'),
'-d', 'dummy',
'-s', 'test_grid_search',
'-i', '.',
'-G', test_dir,
'-T', test_dir,
'-R', test_dir,
'-w', 'Job.txt',
'-g', 'grid',
'-v',
'--', '--dry-run',
]
with utils.Quiet():
main(parameters)
# number of jobs should be 12
assert bob.bio.base.script.grid_search.task_count == 6
# number of jobs in the grid: 36 (including best possible re-use of files; minus preprocessing)
assert bob.bio.base.script.grid_search.job_count == 30
# and now, finally run locally
parameters = [
'-c', os.path.join(dummy_dir, 'grid_search.py'),
'-d', 'dummy',
'-s', 'test_grid_search',
'-G', test_dir,
'-T', test_dir,
'-R', test_dir,
'-w', 'Job.txt',
'-l', '4', '-L', '-1', '-v',
'--', '--imports', 'bob.io.image',
'--dry-run',
]
with utils.Quiet():
main(parameters)
# number of jobs should be 12
assert bob.bio.base.script.grid_search.task_count == 6
# number of jobs in the grid: 36 (including best possible re-use of files; minus preprocessing)
assert bob.bio.base.script.grid_search.job_count == 0
finally:
shutil.rmtree(test_dir)
def test_grid_search():
test_dir = tempfile.mkdtemp(prefix='bobtest_')
# tests that the parameter_test.py script works properly
try:
# first test without grid option
parameters = [
'-c', os.path.join(dummy_dir, 'grid_search.py'),
'-d', 'dummy',
'-e', 'dummy',
'-s', 'test_grid_search',
'-T', test_dir,
'-R', test_dir,
'-v',
'--', '--dry-run',
'--preferred-package', 'bob.bio.base'
]
from bob.bio.base.script.grid_search import main
with utils.Quiet():
main(parameters)
# number of jobs should be 12
assert bob.bio.base.script.grid_search.task_count == 6
# but no job in the grid
assert bob.bio.base.script.grid_search.job_count == 0
# assert that the Experiment.info files are at the right location
for p in (1,2):
for f in (1,2):
for s in (1,2):
if 2*p>f:
assert os.path.exists(os.path.join(test_dir, "test_grid_search/Default/P%d/F%d/S%d/Experiment.info"%(p,f,s)))
# now, in the grid...
parameters = [
'-c', os.path.join(dummy_dir, 'grid_search.py'),
'-d', 'dummy',
'-s', 'test_grid_search',
'-i', '.',
'-G', test_dir,
'-T', test_dir,
'-R', test_dir,
'-g', 'grid',
'-v',
'--', '--dry-run',
'--preferred-package', 'bob.bio.base'
]
with utils.Quiet():
main(parameters)
# number of jobs should be 12
assert bob.bio.base.script.grid_search.task_count == 6
# number of jobs in the grid: 36 (including best possible re-use of files; minus preprocessing)
assert bob.bio.base.script.grid_search.job_count == 30
# and now, finally run locally
parameters = [
'-c', os.path.join(dummy_dir, 'grid_search.py'),
'-d', 'dummy',
'-s', 'test_grid_search',
'-G', test_dir,
'-T', test_dir,
'-R', test_dir,
'-l', '4', '-L', '-1', '-v',
'--', '--imports', 'bob.io.image',
'--dry-run',
'--preferred-package', 'bob.bio.base'
]
with utils.Quiet():
main(parameters)
# number of jobs should be 12
assert bob.bio.base.script.grid_search.task_count == 6
# number of jobs in the grid: 36 (including best possible re-use of files; minus preprocessing)
assert bob.bio.base.script.grid_search.job_count == 0
finally:
shutil.rmtree(test_dir)