def invoke_train(train, training_f): print(centre_str("training", c="-")) print("Training '{}' with {}".format(train, training_f)) train_set = training_f() test = train(train_set) print("Training of '{}' completed.".format(train)) return test
def invoke_train(train, training_f): print(centre_str('training', c='-')) print("Training '{}' with {}".format(train, training_f)) train_set = training_f() test = train(train_set) print("Training of '{}' completed.".format(train)) return test
def invoke_test(test, testing_f): print(centre_str("testing", c="-")) print("Testing '{}' with {}".format(test, testing_f)) test_set = testing_f() results = test(test_set) print("Testing of '{}' completed.\n".format(test)) return { id_: {"gt": gt.points.tolist(), "result": r.tojson()} for (id_, r, gt) in zip(test_set.ids, results, test_set.gt_shapes) }
def run_method(ex, method, trainable=True, upload=False, force=False, force_upload=False, output=False, errors_dir=None, results_dir=None, matlab=False): id_f = ex.trainable_id if trainable else ex.untrainable_id run = (partial(invoke_train_and_test, ex.training) if trainable else invoke_test) cachable = ex.testing.predefined and method.predefined if trainable: # to cache a trainable method we also need the experiment training # to be predefined cachable = cachable and ex.training.predefined if not force: if cachable: id_ = id_f(method) id_hash = hash_of_id(id_)[:5] print(centre_str('[ cache id: {} ]'.format(id_hash))) if force_upload and upload: results = run(method, ex.testing) print('Uploading results for {} (forced)'.format(id_hash)) upload_results(results, id_) else: try: results = retrieve_results(id_) except CachedExperimentNotAvailable: print('No cached version of {}.'.format(id_hash)) results = run(method, ex.testing) if upload: print('Uploading results for {}'.format(id_hash)) upload_results(results, id_) else: if upload: print('Skipping upload of {} - already ' 'saved'.format(id_hash)) elif not output: print('Warning: cannot upload {} as it is' 'uncachable'.format(method)) else: results = run(method, ex.testing) else: results = run(method, ex.testing) if output: save_test_results(results, method.name, results_dir, matlab=matlab) compute_and_save_errors(results, ex.error_metrics, method.name, errors_dir) print("Results saved for '{}'.\n".format(method))
def run_method( ex, method, trainable=True, upload=False, force=False, force_upload=False, output=False, errors_dir=None, results_dir=None, matlab=False, ): id_f = ex.trainable_id if trainable else ex.untrainable_id run = partial(invoke_train_and_test, ex.training) if trainable else invoke_test cachable = ex.testing.predefined and method.predefined if trainable: # to cache a trainable method we also need the experiment training # to be predefined cachable = cachable and ex.training.predefined if not force: if cachable: id_ = id_f(method) id_hash = hash_of_id(id_)[:5] print(centre_str("[ cache id: {} ]".format(id_hash))) if force_upload and upload: results = run(method, ex.testing) print("Uploading results for {} (forced)".format(id_hash)) upload_results(results, id_) else: try: results = retrieve_results(id_) except CachedExperimentNotAvailable: print("No cached version of {}.".format(id_hash)) results = run(method, ex.testing) if upload: print("Uploading results for {}".format(id_hash)) upload_results(results, id_) else: if upload: print("Skipping upload of {} - already " "saved".format(id_hash)) elif not output: print("Warning: cannot upload {} as it is" "uncachable".format(method)) else: results = run(method, ex.testing) else: results = run(method, ex.testing) if output: save_test_results(results, method.name, results_dir, matlab=matlab) compute_and_save_errors(results, ex.error_metrics, method.name, errors_dir) print("Results saved for '{}'.\n".format(method))
def invoke_test(test, testing_f): print(centre_str('testing', c='-')) print("Testing '{}' with {}".format(test, testing_f)) test_set = testing_f() results = test(test_set) print("Testing of '{}' completed.\n".format(test)) return { id_: { 'gt': gt.points.tolist(), 'result': r.tojson() } for (id_, r, gt) in zip(test_set.ids, results, test_set.gt_shapes) }
def invoke_benchmark( experiment_name, output_dir=None, overwrite=False, matlab=False, upload=False, force=False, force_upload=False ): print("") print(centre_str("- - - - M E N P O B E N C H - - - -")) if upload: if not can_upload(): raise MenpoCDNCredentialsMissingError( "MENPO_CDN_S3_ACCESS_KEY and MENPO_CDN_S3_SECRECT_KEY both are" " needed to upload cached results" ) print(centre_str("** MENPO CDN UPLOAD ENABLED **")) if force_upload: print(centre_str("** UPLOAD FORCED **")) print(centre_str("v" + menpobench.__version__)) print(centre_str("config: {}".format(experiment_name))) if output_dir is not None: print(centre_str("output: {}".format(output_dir))) print(centre_str("cache: {}".format(resolve_cache_dir()))) if force: print(centre_str("FORCED RECOMPUTATION ENABLED")) # Load the experiment and check it's schematically valid ex = retrieve_experiment(experiment_name) # Check if we have any dependency on matlab if ex.depends_on_matlab: print(centre_str("matlab: {}".format(resolve_matlab_bin_path()))) print("") if output_dir is not None: # Handle the creation of the output directory output_dir = Path(norm_path(output_dir)) if output_dir.is_dir(): if not overwrite: raise OutputDirExistsError( "Output directory {} already exists.\n" "Pass '--overwrite' if you want menpobench to delete this " "directory automatically.".format(output_dir) ) else: print("--overwrite passed and output directory {} exists - " "deleting\n".format(output_dir)) shutil.rmtree(str(output_dir)) output_dir.mkdir() errors_dir = output_dir / "errors" results_dir = output_dir / "results" errors_dir.mkdir() results_dir.mkdir() results_trainable_dir = results_dir / "trainable_methods" results_untrainable_dir = results_dir / "untrainable_methods" errors_trainable_dir = errors_dir / "trainable_methods" errors_untrainable_dir = errors_dir / "untrainable_methods" save_yaml(ex.config, str(output_dir / "experiment.yaml")) run = partial( run_method, ex, upload=upload, force=force, force_upload=force_upload, matlab=matlab, output=(output_dir is not None), ) try: if ex.n_trainable_methods > 0: print(centre_str("I. TRAINABLE METHODS")) if output_dir is not None: results_trainable_dir.mkdir() errors_trainable_dir.mkdir() else: results_trainable_dir = None errors_trainable_dir = None for i, train in enumerate(ex.trainable_methods, 1): print(centre_str("{}/{} - {}".format(i, ex.n_trainable_methods, train), c="=")) run(train, trainable=True, errors_dir=errors_trainable_dir, results_dir=results_trainable_dir) if ex.n_untrainable_methods > 0: print(centre_str("II. UNTRAINABLE METHODS", c=" ")) if output_dir is not None: results_untrainable_dir.mkdir() errors_untrainable_dir.mkdir() else: results_untrainable_dir = None errors_untrainable_dir = None for i, test in enumerate(ex.untrainable_methods, 1): print(centre_str("{}/{} - " "{}".format(i, ex.n_untrainable_methods, test), c="=")) run(test, trainable=False, errors_dir=errors_untrainable_dir, results_dir=results_untrainable_dir) # We now have all the results computed - draw the CED curves. if output_dir is not None: plot_ceds(output_dir) finally: TempDirectory.delete_all()
def invoke_benchmark(experiment_name, output_dir=None, overwrite=False, matlab=False, upload=False, force=False, force_upload=False): print('') print(centre_str('- - - - M E N P O B E N C H - - - -')) if upload: if not can_upload(): raise MenpoCDNCredentialsMissingError( 'MENPO_CDN_S3_ACCESS_KEY and MENPO_CDN_S3_SECRECT_KEY both are' ' needed to upload cached results') print(centre_str('** MENPO CDN UPLOAD ENABLED **')) if force_upload: print(centre_str('** UPLOAD FORCED **')) print(centre_str('v' + menpobench.__version__)) print(centre_str('config: {}'.format(experiment_name))) if output_dir is not None: print(centre_str('output: {}'.format(output_dir))) print(centre_str('cache: {}'.format(resolve_cache_dir()))) if force: print(centre_str('FORCED RECOMPUTATION ENABLED')) # Load the experiment and check it's schematically valid ex = retrieve_experiment(experiment_name) # Check if we have any dependency on matlab if ex.depends_on_matlab: print(centre_str('matlab: {}'.format(resolve_matlab_bin_path()))) print('') if output_dir is not None: # Handle the creation of the output directory output_dir = Path(norm_path(output_dir)) if output_dir.is_dir(): if not overwrite: raise OutputDirExistsError( "Output directory {} already exists.\n" "Pass '--overwrite' if you want menpobench to delete this " "directory automatically.".format(output_dir)) else: print('--overwrite passed and output directory {} exists - ' 'deleting\n'.format(output_dir)) shutil.rmtree(str(output_dir)) output_dir.mkdir() errors_dir = output_dir / 'errors' results_dir = output_dir / 'results' errors_dir.mkdir() results_dir.mkdir() results_trainable_dir = results_dir / 'trainable_methods' results_untrainable_dir = results_dir / 'untrainable_methods' errors_trainable_dir = errors_dir / 'trainable_methods' errors_untrainable_dir = errors_dir / 'untrainable_methods' save_yaml(ex.config, str(output_dir / 'experiment.yaml')) run = partial(run_method, ex, upload=upload, force=force, force_upload=force_upload, matlab=matlab, output=(output_dir is not None)) try: if ex.n_trainable_methods > 0: print(centre_str('I. TRAINABLE METHODS')) if output_dir is not None: results_trainable_dir.mkdir() errors_trainable_dir.mkdir() else: results_trainable_dir = None errors_trainable_dir = None for i, train in enumerate(ex.trainable_methods, 1): print( centre_str('{}/{} - {}'.format(i, ex.n_trainable_methods, train), c='=')) run(train, trainable=True, errors_dir=errors_trainable_dir, results_dir=results_trainable_dir) if ex.n_untrainable_methods > 0: print(centre_str('II. UNTRAINABLE METHODS', c=' ')) if output_dir is not None: results_untrainable_dir.mkdir() errors_untrainable_dir.mkdir() else: results_untrainable_dir = None errors_untrainable_dir = None for i, test in enumerate(ex.untrainable_methods, 1): print( centre_str('{}/{} - ' '{}'.format(i, ex.n_untrainable_methods, test), c='=')) run(test, trainable=False, errors_dir=errors_untrainable_dir, results_dir=results_untrainable_dir) # We now have all the results computed - draw the CED curves. if output_dir is not None: plot_ceds(output_dir) finally: TempDirectory.delete_all()