def run_test_suite(verbose=False):
ts = generate_test_suite()
path = str(TempDirectory.create_new() / 'mb.py')
with open(path, 'wt') as f:
f.write(ts)
args = ['', path]
if verbose:
args.append('-v')
tests_passed = nose.run(argv=args)
TempDirectory.delete_all()
if tests_passed:
sys.exit(0)
else:
sys.exit(1)
def run_test_suite(verbose=False):
ts = generate_test_suite()
path = str(TempDirectory.create_new() / 'mb.py')
with open(path, 'wt') as f:
f.write(ts)
args = ['', path]
if verbose:
args.append('-v')
tests_passed = nose.run(argv=args)
TempDirectory.delete_all()
if tests_passed:
sys.exit(0)
else:
sys.exit(1)
def __call__(self, img_generator):
test_path = TempDirectory.create_new() / 'menpobench_test_images'
# Save images down to mat file
images_to_mat(img_generator, test_path)
# Call matlab bridge to test file - will drop out a result mat
invoke_matlab(
"addpath('{}'); menpobench_matlab_fit('{}', '{}');".format(
matlab_functions_dir(), self.method_path, test_path))
return load_matlab_results(test_path)
def __call__(self, img_generator):
test_path = TempDirectory.create_new() / "menpobench_test_images"
# Save images down to mat file
images_to_mat(img_generator, test_path)
# Call matlab bridge to test file - will drop out a result mat
invoke_matlab(
"addpath('{}'); menpobench_matlab_fit('{}', '{}');".format(
matlab_functions_dir(), self.method_path, test_path
)
)
return load_matlab_results(test_path)
def invoke_benchmark(
experiment_name, output_dir=None, overwrite=False, matlab=False, upload=False, force=False, force_upload=False
):
print("")
print(centre_str("- - - - M E N P O B E N C H - - - -"))
if upload:
if not can_upload():
raise MenpoCDNCredentialsMissingError(
"MENPO_CDN_S3_ACCESS_KEY and MENPO_CDN_S3_SECRECT_KEY both are" " needed to upload cached results"
)
print(centre_str("** MENPO CDN UPLOAD ENABLED **"))
if force_upload:
print(centre_str("** UPLOAD FORCED **"))
print(centre_str("v" + menpobench.__version__))
print(centre_str("config: {}".format(experiment_name)))
if output_dir is not None:
print(centre_str("output: {}".format(output_dir)))
print(centre_str("cache: {}".format(resolve_cache_dir())))
if force:
print(centre_str("FORCED RECOMPUTATION ENABLED"))
# Load the experiment and check it's schematically valid
ex = retrieve_experiment(experiment_name)
# Check if we have any dependency on matlab
if ex.depends_on_matlab:
print(centre_str("matlab: {}".format(resolve_matlab_bin_path())))
print("")
if output_dir is not None:
# Handle the creation of the output directory
output_dir = Path(norm_path(output_dir))
if output_dir.is_dir():
if not overwrite:
raise OutputDirExistsError(
"Output directory {} already exists.\n"
"Pass '--overwrite' if you want menpobench to delete this "
"directory automatically.".format(output_dir)
)
else:
print("--overwrite passed and output directory {} exists - " "deleting\n".format(output_dir))
shutil.rmtree(str(output_dir))
output_dir.mkdir()
errors_dir = output_dir / "errors"
results_dir = output_dir / "results"
errors_dir.mkdir()
results_dir.mkdir()
results_trainable_dir = results_dir / "trainable_methods"
results_untrainable_dir = results_dir / "untrainable_methods"
errors_trainable_dir = errors_dir / "trainable_methods"
errors_untrainable_dir = errors_dir / "untrainable_methods"
save_yaml(ex.config, str(output_dir / "experiment.yaml"))
run = partial(
run_method,
ex,
upload=upload,
force=force,
force_upload=force_upload,
matlab=matlab,
output=(output_dir is not None),
)
try:
if ex.n_trainable_methods > 0:
print(centre_str("I. TRAINABLE METHODS"))
if output_dir is not None:
results_trainable_dir.mkdir()
errors_trainable_dir.mkdir()
else:
results_trainable_dir = None
errors_trainable_dir = None
for i, train in enumerate(ex.trainable_methods, 1):
print(centre_str("{}/{} - {}".format(i, ex.n_trainable_methods, train), c="="))
run(train, trainable=True, errors_dir=errors_trainable_dir, results_dir=results_trainable_dir)
if ex.n_untrainable_methods > 0:
print(centre_str("II. UNTRAINABLE METHODS", c=" "))
if output_dir is not None:
results_untrainable_dir.mkdir()
errors_untrainable_dir.mkdir()
else:
results_untrainable_dir = None
errors_untrainable_dir = None
for i, test in enumerate(ex.untrainable_methods, 1):
print(centre_str("{}/{} - " "{}".format(i, ex.n_untrainable_methods, test), c="="))
run(test, trainable=False, errors_dir=errors_untrainable_dir, results_dir=results_untrainable_dir)
# We now have all the results computed - draw the CED curves.
if output_dir is not None:
plot_ceds(output_dir)
finally:
TempDirectory.delete_all()
def invoke_benchmark(experiment_name,
output_dir=None,
overwrite=False,
matlab=False,
upload=False,
force=False,
force_upload=False):
print('')
print(centre_str('- - - - M E N P O B E N C H - - - -'))
if upload:
if not can_upload():
raise MenpoCDNCredentialsMissingError(
'MENPO_CDN_S3_ACCESS_KEY and MENPO_CDN_S3_SECRECT_KEY both are'
' needed to upload cached results')
print(centre_str('** MENPO CDN UPLOAD ENABLED **'))
if force_upload:
print(centre_str('** UPLOAD FORCED **'))
print(centre_str('v' + menpobench.__version__))
print(centre_str('config: {}'.format(experiment_name)))
if output_dir is not None:
print(centre_str('output: {}'.format(output_dir)))
print(centre_str('cache: {}'.format(resolve_cache_dir())))
if force:
print(centre_str('FORCED RECOMPUTATION ENABLED'))
# Load the experiment and check it's schematically valid
ex = retrieve_experiment(experiment_name)
# Check if we have any dependency on matlab
if ex.depends_on_matlab:
print(centre_str('matlab: {}'.format(resolve_matlab_bin_path())))
print('')
if output_dir is not None:
# Handle the creation of the output directory
output_dir = Path(norm_path(output_dir))
if output_dir.is_dir():
if not overwrite:
raise OutputDirExistsError(
"Output directory {} already exists.\n"
"Pass '--overwrite' if you want menpobench to delete this "
"directory automatically.".format(output_dir))
else:
print('--overwrite passed and output directory {} exists - '
'deleting\n'.format(output_dir))
shutil.rmtree(str(output_dir))
output_dir.mkdir()
errors_dir = output_dir / 'errors'
results_dir = output_dir / 'results'
errors_dir.mkdir()
results_dir.mkdir()
results_trainable_dir = results_dir / 'trainable_methods'
results_untrainable_dir = results_dir / 'untrainable_methods'
errors_trainable_dir = errors_dir / 'trainable_methods'
errors_untrainable_dir = errors_dir / 'untrainable_methods'
save_yaml(ex.config, str(output_dir / 'experiment.yaml'))
run = partial(run_method,
ex,
upload=upload,
force=force,
force_upload=force_upload,
matlab=matlab,
output=(output_dir is not None))
try:
if ex.n_trainable_methods > 0:
print(centre_str('I. TRAINABLE METHODS'))
if output_dir is not None:
results_trainable_dir.mkdir()
errors_trainable_dir.mkdir()
else:
results_trainable_dir = None
errors_trainable_dir = None
for i, train in enumerate(ex.trainable_methods, 1):
print(
centre_str('{}/{} - {}'.format(i, ex.n_trainable_methods,
train),
c='='))
run(train,
trainable=True,
errors_dir=errors_trainable_dir,
results_dir=results_trainable_dir)
if ex.n_untrainable_methods > 0:
print(centre_str('II. UNTRAINABLE METHODS', c=' '))
if output_dir is not None:
results_untrainable_dir.mkdir()
errors_untrainable_dir.mkdir()
else:
results_untrainable_dir = None
errors_untrainable_dir = None
for i, test in enumerate(ex.untrainable_methods, 1):
print(
centre_str('{}/{} - '
'{}'.format(i, ex.n_untrainable_methods, test),
c='='))
run(test,
trainable=False,
errors_dir=errors_untrainable_dir,
results_dir=results_untrainable_dir)
# We now have all the results computed - draw the CED curves.
if output_dir is not None:
plot_ceds(output_dir)
finally:
TempDirectory.delete_all()
def _unpacked_cache_dir(self):
if self._unpacked_temp_dir is None:
self._unpacked_temp_dir = TempDirectory.create_new()
return self._unpacked_temp_dir
def _unpacked_cache_dir(self):
if self._unpacked_temp_dir is None:
self._unpacked_temp_dir = TempDirectory.create_new()
return self._unpacked_temp_dir