def save_custom_config(c, validate_schema=True):
from menpobench.utils import save_yaml, load_yaml
if custom_config_path().exists():
# Update existing config file with new information
config = load_yaml(custom_config_path())
config.update(c)
else:
config = c
if validate_schema:
# validate the config against the schema
s = config_schema()
if not schema_is_valid(s, config):
report = schema_error_report(s, config)
raise SchemaError('configuration', 'user', report)
save_yaml(config, custom_config_path())
def invoke_benchmark(
experiment_name, output_dir=None, overwrite=False, matlab=False, upload=False, force=False, force_upload=False
):
print("")
print(centre_str("- - - - M E N P O B E N C H - - - -"))
if upload:
if not can_upload():
raise MenpoCDNCredentialsMissingError(
"MENPO_CDN_S3_ACCESS_KEY and MENPO_CDN_S3_SECRECT_KEY both are" " needed to upload cached results"
)
print(centre_str("** MENPO CDN UPLOAD ENABLED **"))
if force_upload:
print(centre_str("** UPLOAD FORCED **"))
print(centre_str("v" + menpobench.__version__))
print(centre_str("config: {}".format(experiment_name)))
if output_dir is not None:
print(centre_str("output: {}".format(output_dir)))
print(centre_str("cache: {}".format(resolve_cache_dir())))
if force:
print(centre_str("FORCED RECOMPUTATION ENABLED"))
# Load the experiment and check it's schematically valid
ex = retrieve_experiment(experiment_name)
# Check if we have any dependency on matlab
if ex.depends_on_matlab:
print(centre_str("matlab: {}".format(resolve_matlab_bin_path())))
print("")
if output_dir is not None:
# Handle the creation of the output directory
output_dir = Path(norm_path(output_dir))
if output_dir.is_dir():
if not overwrite:
raise OutputDirExistsError(
"Output directory {} already exists.\n"
"Pass '--overwrite' if you want menpobench to delete this "
"directory automatically.".format(output_dir)
)
else:
print("--overwrite passed and output directory {} exists - " "deleting\n".format(output_dir))
shutil.rmtree(str(output_dir))
output_dir.mkdir()
errors_dir = output_dir / "errors"
results_dir = output_dir / "results"
errors_dir.mkdir()
results_dir.mkdir()
results_trainable_dir = results_dir / "trainable_methods"
results_untrainable_dir = results_dir / "untrainable_methods"
errors_trainable_dir = errors_dir / "trainable_methods"
errors_untrainable_dir = errors_dir / "untrainable_methods"
save_yaml(ex.config, str(output_dir / "experiment.yaml"))
run = partial(
run_method,
ex,
upload=upload,
force=force,
force_upload=force_upload,
matlab=matlab,
output=(output_dir is not None),
)
try:
if ex.n_trainable_methods > 0:
print(centre_str("I. TRAINABLE METHODS"))
if output_dir is not None:
results_trainable_dir.mkdir()
errors_trainable_dir.mkdir()
else:
results_trainable_dir = None
errors_trainable_dir = None
for i, train in enumerate(ex.trainable_methods, 1):
print(centre_str("{}/{} - {}".format(i, ex.n_trainable_methods, train), c="="))
run(train, trainable=True, errors_dir=errors_trainable_dir, results_dir=results_trainable_dir)
if ex.n_untrainable_methods > 0:
print(centre_str("II. UNTRAINABLE METHODS", c=" "))
if output_dir is not None:
results_untrainable_dir.mkdir()
errors_untrainable_dir.mkdir()
else:
results_untrainable_dir = None
errors_untrainable_dir = None
for i, test in enumerate(ex.untrainable_methods, 1):
print(centre_str("{}/{} - " "{}".format(i, ex.n_untrainable_methods, test), c="="))
run(test, trainable=False, errors_dir=errors_untrainable_dir, results_dir=results_untrainable_dir)
# We now have all the results computed - draw the CED curves.
if output_dir is not None:
plot_ceds(output_dir)
finally:
TempDirectory.delete_all()
def invoke_benchmark(experiment_name,
output_dir=None,
overwrite=False,
matlab=False,
upload=False,
force=False,
force_upload=False):
print('')
print(centre_str('- - - - M E N P O B E N C H - - - -'))
if upload:
if not can_upload():
raise MenpoCDNCredentialsMissingError(
'MENPO_CDN_S3_ACCESS_KEY and MENPO_CDN_S3_SECRECT_KEY both are'
' needed to upload cached results')
print(centre_str('** MENPO CDN UPLOAD ENABLED **'))
if force_upload:
print(centre_str('** UPLOAD FORCED **'))
print(centre_str('v' + menpobench.__version__))
print(centre_str('config: {}'.format(experiment_name)))
if output_dir is not None:
print(centre_str('output: {}'.format(output_dir)))
print(centre_str('cache: {}'.format(resolve_cache_dir())))
if force:
print(centre_str('FORCED RECOMPUTATION ENABLED'))
# Load the experiment and check it's schematically valid
ex = retrieve_experiment(experiment_name)
# Check if we have any dependency on matlab
if ex.depends_on_matlab:
print(centre_str('matlab: {}'.format(resolve_matlab_bin_path())))
print('')
if output_dir is not None:
# Handle the creation of the output directory
output_dir = Path(norm_path(output_dir))
if output_dir.is_dir():
if not overwrite:
raise OutputDirExistsError(
"Output directory {} already exists.\n"
"Pass '--overwrite' if you want menpobench to delete this "
"directory automatically.".format(output_dir))
else:
print('--overwrite passed and output directory {} exists - '
'deleting\n'.format(output_dir))
shutil.rmtree(str(output_dir))
output_dir.mkdir()
errors_dir = output_dir / 'errors'
results_dir = output_dir / 'results'
errors_dir.mkdir()
results_dir.mkdir()
results_trainable_dir = results_dir / 'trainable_methods'
results_untrainable_dir = results_dir / 'untrainable_methods'
errors_trainable_dir = errors_dir / 'trainable_methods'
errors_untrainable_dir = errors_dir / 'untrainable_methods'
save_yaml(ex.config, str(output_dir / 'experiment.yaml'))
run = partial(run_method,
ex,
upload=upload,
force=force,
force_upload=force_upload,
matlab=matlab,
output=(output_dir is not None))
try:
if ex.n_trainable_methods > 0:
print(centre_str('I. TRAINABLE METHODS'))
if output_dir is not None:
results_trainable_dir.mkdir()
errors_trainable_dir.mkdir()
else:
results_trainable_dir = None
errors_trainable_dir = None
for i, train in enumerate(ex.trainable_methods, 1):
print(
centre_str('{}/{} - {}'.format(i, ex.n_trainable_methods,
train),
c='='))
run(train,
trainable=True,
errors_dir=errors_trainable_dir,
results_dir=results_trainable_dir)
if ex.n_untrainable_methods > 0:
print(centre_str('II. UNTRAINABLE METHODS', c=' '))
if output_dir is not None:
results_untrainable_dir.mkdir()
errors_untrainable_dir.mkdir()
else:
results_untrainable_dir = None
errors_untrainable_dir = None
for i, test in enumerate(ex.untrainable_methods, 1):
print(
centre_str('{}/{} - '
'{}'.format(i, ex.n_untrainable_methods, test),
c='='))
run(test,
trainable=False,
errors_dir=errors_untrainable_dir,
results_dir=results_untrainable_dir)
# We now have all the results computed - draw the CED curves.
if output_dir is not None:
plot_ceds(output_dir)
finally:
TempDirectory.delete_all()