def __face_verify__(self, parameters, test_dir, sub_dir, ref_modifier="", score_modifier=('scores','')):
from facereclib.script.faceverify import main
main([sys.argv[0]] + parameters)
# assert that the score file exists
score_files = (os.path.join(test_dir, sub_dir, 'scores', 'Default', 'nonorm', '%s-dev%s'%score_modifier), os.path.join(test_dir, sub_dir, 'scores', 'Default', 'ztnorm', '%s-dev%s'%score_modifier))
self.assertTrue(os.path.exists(score_files[0]))
self.assertTrue(os.path.exists(score_files[1]))
# also assert that the scores are still the same -- though they have no real meaning
reference_files = (os.path.join(base_dir, 'scripts', 'scores-nonorm%s-dev'%ref_modifier), os.path.join(base_dir, 'scripts', 'scores-ztnorm%s-dev'%ref_modifier))
for i in (0,1):
d = []
# read reference and new data
for score_file in (score_files[i], reference_files[i]):
f = bob.measure.load.open_file(score_files[i])
d_ = []
for line in f:
if isinstance(line, bytes): line = line.decode('utf-8')
d_.append(line.rstrip().split())
d.append(numpy.array(d_))
self.assertTrue(d[0].shape, d[1].shape)
# assert that the data order is still correct
self.assertTrue((d[0][:,0:3] == d[1][:, 0:3]).all())
# assert that the values are OK
self.assertTrue((numpy.abs(d[0][:,3].astype(float) - d[1][:,3].astype(float)) < 1e-5).all())
shutil.rmtree(test_dir)
def __face_verify__(self, parameters, test_dir, sub_dir, ref_modifier="", score_modifier=('scores','')):
from facereclib.script.faceverify import main
main([sys.argv[0]] + parameters)
# assert that the score file exists
score_files = (os.path.join(test_dir, sub_dir, 'scores', 'Default', 'nonorm', '%s-dev%s'%score_modifier), os.path.join(test_dir, sub_dir, 'scores', 'Default', 'ztnorm', '%s-dev%s'%score_modifier))
self.assertTrue(os.path.exists(score_files[0]))
self.assertTrue(os.path.exists(score_files[1]))
# also assert that the scores are still the same -- though they have no real meaning
reference_files = (os.path.join(base_dir, 'scripts', 'scores-nonorm%s-dev'%ref_modifier), os.path.join(base_dir, 'scripts', 'scores-ztnorm%s-dev'%ref_modifier))
for i in (0,1):
d = []
# read reference and new data
for score_file in (score_files[i], reference_files[i]):
f = bob.measure.load.open_file(score_files[i])
d_ = []
for line in f:
if isinstance(line, bytes): line = line.decode('utf-8')
d_.append(line.rstrip().split())
d.append(numpy.array(d_))
self.assertTrue(d[0].shape, d[1].shape)
# assert that the data order is still correct
self.assertTrue((d[0][:,0:3] == d[1][:, 0:3]).all())
# assert that the values are OK
self.assertTrue((numpy.abs(d[0][:,3].astype(float) - d[1][:,3].astype(float)) < 1e-5).all())
shutil.rmtree(test_dir)
def test06_faceverify_gbu_grid(self):
self.grid_available()
# try to import the gbu database
try:
facereclib.utils.resources.load_resource('gbu','database')
except Exception as e:
raise SkipTest("The resource for database 'gbu' could not be loaded; probably you didn't define the 'xbob.db.gbu' in your *buildout.cfg*. Here is the import error: '%s'" % e)
test_dir = tempfile.mkdtemp(prefix='frltest_')
# define dummy parameters
parameters = [
sys.argv[0],
'-p', 'face-crop',
'-f', 'eigenfaces',
'-t', os.path.join(config_dir, 'tools', 'dummy.py'),
'-g', 'grid',
'--dry-run',
'--user-directory', test_dir,
'-b', 'dummy'
]
print (' '.join(parameters))
# run the test; should not execute anything...
from facereclib.script.faceverify_gbu import main
main(parameters)
shutil.rmtree(test_dir)
def test06_faceverify_gbu_grid(self):
self.grid_available()
# try to import the gbu database
try:
facereclib.utils.resources.load_resource('gbu','database')
except Exception as e:
raise SkipTest("The resource for database 'gbu' could not be loaded; probably you didn't define the 'bob.db.gbu' in your *buildout.cfg*. Here is the import error: '%s'" % e)
test_dir = tempfile.mkdtemp(prefix='frltest_')
# define dummy parameters
parameters = [
sys.argv[0],
'-p', 'face-crop',
'-f', 'eigenfaces',
'-t', os.path.join(config_dir, 'tools', 'dummy.py'),
'-g', 'grid',
'--dry-run',
'--user-directory', test_dir,
'-b', 'dummy'
]
print (facereclib.utils.command_line(parameters))
# run the test; should not execute anything...
from facereclib.script.faceverify_gbu import main
main(parameters)
shutil.rmtree(test_dir)
def test21_parameter_script(self):
self.grid_available()
test_dir = tempfile.mkdtemp(prefix='frltest_')
# tests that the parameter_test.py script works properly
# first test without grid option
parameters = [
sys.argv[0],
'-c', os.path.join(base_dir, 'scripts', 'parameter_Test.py'),
'-d', os.path.join(base_dir, 'scripts', 'atnt_Test.py'),
'-f', 'lgbphs',
'-b', 'test_p',
'-s', '.',
'-T', test_dir,
'-R', test_dir,
'--', '--dry-run',
]
from facereclib.script.parameter_test import main
main(parameters)
# number of jobs should be 12
self.assertEqual(facereclib.script.parameter_test.task_count, 12)
# but no job in the grid
self.assertEqual(facereclib.script.parameter_test.job_count, 0)
# now, in the grid...
parameters = [
sys.argv[0],
'-c', os.path.join(base_dir, 'scripts', 'parameter_Test.py'),
'-d', os.path.join(base_dir, 'scripts', 'atnt_Test.py'),
'-f', 'lgbphs',
'-b', 'test_p',
'-i', '.',
'-s', '.',
'-T', test_dir,
'-R', test_dir,
'-g', 'grid',
'--', '--dry-run',
]
main(parameters)
# number of jobs should be 12
self.assertEqual(facereclib.script.parameter_test.task_count, 12)
# number of jobs in the grid: 36 (including best possible re-use of files; minus preprocessing)
self.assertEqual(facereclib.script.parameter_test.job_count, 36)
shutil.rmtree(test_dir)
def test01x_faceverify_filelist(self):
try:
import bob.db.verification.filelist
except ImportError:
raise SkipTest("Skipping test since bob.db.verification.filelist is not available")
test_dir = tempfile.mkdtemp(prefix='frltest_')
# define dummy parameters
parameters = [
'-d', os.path.join(base_dir, 'databases', 'atnt_fl', 'atnt_fl_database.py'),
# '--protocol', 'None',
'-p', os.path.join(config_dir, 'preprocessing', 'face_crop.py'),
'-f', os.path.join(config_dir, 'features', 'eigenfaces.py'),
'-t', os.path.join(config_dir, 'tools', 'dummy.py'),
'--zt-norm',
'-b', 'test_x',
'--temp-directory', test_dir,
'--user-directory', test_dir
]
print (facereclib.utils.command_line(parameters))
from facereclib.script.faceverify import main
main([sys.argv[0]] + parameters)
# assert that the score file exists
score_files = (os.path.join(test_dir, 'test_x', 'scores', 'nonorm', 'scores-dev'), os.path.join(test_dir, 'test_x', 'scores', 'ztnorm', 'scores-dev'))
self.assertTrue(os.path.exists(score_files[0]))
self.assertTrue(os.path.exists(score_files[1]))
# assert that the scores are are identical
reference_files = (os.path.join(base_dir, 'scripts', 'scores-nonorm-dev'), os.path.join(base_dir, 'scripts', 'scores-ztnorm-dev'))
for i in (0,1):
a1, b1 = bob.measure.load.split_four_column(score_files[i])
a2, b2 = bob.measure.load.split_four_column(reference_files[i])
a1 = sorted(a1); a2 = sorted(a2); b1 = sorted(b1); b2 = sorted(b2)
for i in range(len(a1)):
self.assertAlmostEqual(a1[i], a2[i], 6)
for i in range(len(b1)):
self.assertAlmostEqual(b1[i], b2[i], 6)
shutil.rmtree(test_dir)
def test01x_faceverify_filelist(self):
try:
import bob.db.verification.filelist
except ImportError:
raise SkipTest("Skipping test since bob.db.verification.filelist is not available")
test_dir = tempfile.mkdtemp(prefix='frltest_')
# define dummy parameters
parameters = [
'-d', os.path.join(base_dir, 'databases', 'atnt_fl', 'atnt_fl_database.py'),
# '--protocol', 'None',
'-p', os.path.join(config_dir, 'preprocessing', 'face_crop.py'),
'-f', os.path.join(config_dir, 'features', 'eigenfaces.py'),
'-t', os.path.join(config_dir, 'tools', 'dummy.py'),
'--zt-norm',
'-b', 'test_x',
'--temp-directory', test_dir,
'--user-directory', test_dir
]
print (facereclib.utils.command_line(parameters))
from facereclib.script.faceverify import main
main([sys.argv[0]] + parameters)
# assert that the score file exists
score_files = (os.path.join(test_dir, 'test_x', 'scores', 'nonorm', 'scores-dev'), os.path.join(test_dir, 'test_x', 'scores', 'ztnorm', 'scores-dev'))
self.assertTrue(os.path.exists(score_files[0]))
self.assertTrue(os.path.exists(score_files[1]))
# assert that the scores are are identical
reference_files = (os.path.join(base_dir, 'scripts', 'scores-nonorm-dev'), os.path.join(base_dir, 'scripts', 'scores-ztnorm-dev'))
for i in (0,1):
a1, b1 = bob.measure.load.split_four_column(score_files[i])
a2, b2 = bob.measure.load.split_four_column(reference_files[i])
a1 = sorted(a1); a2 = sorted(a2); b1 = sorted(b1); b2 = sorted(b2)
for i in range(len(a1)):
self.assertAlmostEqual(a1[i], a2[i], 6)
for i in range(len(b1)):
self.assertAlmostEqual(b1[i], b2[i], 6)
shutil.rmtree(test_dir)
def test15_evaluate(self):
# tests our 'evaluate' script using the reference files
test_dir = tempfile.mkdtemp(prefix='frltest_')
reference_files = ('scores-nonorm-dev', 'scores-ztnorm-dev')
plots = [os.path.join(test_dir, '%s.pdf')%f for f in ['roc', 'cmc', 'det']]
parameters = [
'--dev-files', reference_files[0], reference_files[1],
'--eval-files', reference_files[0], reference_files[1],
'--directory', os.path.join(base_dir, 'scripts'),
'--legends', 'no norm', 'ZT norm',
'--criterion', 'HTER',
'--roc', plots[0],
'--det', plots[1],
'--cmc', plots[2],
]
# execute the script
from facereclib.script.evaluate import main
main(parameters)
for i in range(3):
self.assertTrue(os.path.exists(plots[i]))
os.remove(plots[i])
os.rmdir(test_dir)
def test02_faceverify_grid(self):
self.grid_available()
test_dir = tempfile.mkdtemp(prefix='frltest_')
# define dummy parameters including the dry-run
parameters = [
sys.argv[0],
'-d', os.path.join(base_dir, 'scripts', 'atnt_Test.py'),
'-p', 'face-crop',
'-f', 'eigenfaces',
'-t', os.path.join(config_dir, 'tools', 'dummy.py'),
'-g', 'grid',
'--zt-norm',
'--dry-run',
'--user-directory', test_dir,
'-b', 'dummy'
]
print (' '.join(parameters))
# run the test; should not execute anything...
from facereclib.script.faceverify import main
main(parameters)
shutil.rmtree(test_dir)
def test15_evaluate(self):
# tests our 'evaluate' script using the reference files
test_dir = tempfile.mkdtemp(prefix='frltest_')
reference_files = ('scores-nonorm-dev', 'scores-ztnorm-dev')
plots = [os.path.join(test_dir, '%s.pdf')%f for f in ['roc', 'cmc', 'det']]
parameters = [
'--dev-files', reference_files[0], reference_files[1],
'--eval-files', reference_files[0], reference_files[1],
'--directory', os.path.join(base_dir, 'scripts'),
'--legends', 'no norm', 'ZT norm',
'--criterion', 'HTER',
'--roc', plots[0],
'--det', plots[1],
'--cmc', plots[2],
]
# execute the script
from facereclib.script.evaluate import main
main(parameters)
for i in range(3):
self.assertTrue(os.path.exists(plots[i]))
os.remove(plots[i])
os.rmdir(test_dir)
def test02_faceverify_grid(self):
self.grid_available()
test_dir = tempfile.mkdtemp(prefix='frltest_')
# define dummy parameters including the dry-run
parameters = [
sys.argv[0],
'-d', os.path.join(base_dir, 'scripts', 'atnt_Test.py'),
'-p', 'face-crop',
'-f', 'eigenfaces',
'-t', os.path.join(config_dir, 'tools', 'dummy.py'),
'-g', 'grid',
'--zt-norm',
'--dry-run',
'--user-directory', test_dir,
'-b', 'dummy'
]
print (facereclib.utils.command_line(parameters))
# run the test; should not execute anything...
from facereclib.script.faceverify import main
main(parameters)
shutil.rmtree(test_dir)
def test16_collect_results(self): # simply test that the collect_results script works test_dir = tempfile.mkdtemp(prefix='frltest_') from facereclib.script.collect_results import main main(['--directory', test_dir, '--sort', '--sort-key', 'dir', '--criterion', 'FAR', '--self-test']) os.rmdir(test_dir)
def test11_baselines_api(self):
self.grid_available()
# test that all of the baselines would execute
from facereclib.script.baselines import available_databases, all_algorithms, main
for database in available_databases:
parameters = [sys.argv[0], '-d', database, '--dry-run']
main(parameters)
parameters.append('-g')
main(parameters)
parameters.extend(['-e', 'HTER'])
main(parameters)
for algorithm in all_algorithms:
parameters = [sys.argv[0], '-a', algorithm, '--dry-run']
main(parameters)
parameters.append('-g')
main(parameters)
parameters.extend(['-e', 'HTER'])
main(parameters)
def test16_collect_results(self): # simply test that the collect_results script works test_dir = tempfile.mkdtemp(prefix='frltest_') from facereclib.script.collect_results import main main(['--directory', test_dir, '--sort', '--sort-key', 'dir', '--criterion', 'FAR', '--self-test']) os.rmdir(test_dir)
def test11_baselines_api(self):
self.grid_available()
# test that all of the baselines would execute
from facereclib.script.baselines import available_databases, all_algorithms, main
for database in available_databases:
parameters = [sys.argv[0], '-d', database, '--dry-run']
main(parameters)
parameters.append('-g')
main(parameters)
parameters.extend(['-e', 'HTER'])
main(parameters)
for algorithm in all_algorithms:
parameters = [sys.argv[0], '-a', algorithm, '--dry-run']
main(parameters)
parameters.append('-g')
main(parameters)
parameters.extend(['-e', 'HTER'])
main(parameters)
