Exemple #1
0
def run_test_suite(verbose=False):
    ts = generate_test_suite()
    path = str(TempDirectory.create_new() / 'mb.py')
    with open(path, 'wt') as f:
        f.write(ts)
    args = ['', path]
    if verbose:
        args.append('-v')
    tests_passed = nose.run(argv=args)
    TempDirectory.delete_all()
    if tests_passed:
        sys.exit(0)
    else:
        sys.exit(1)
Exemple #2
0
def run_test_suite(verbose=False):
    ts = generate_test_suite()
    path = str(TempDirectory.create_new() / 'mb.py')
    with open(path, 'wt') as f:
        f.write(ts)
    args = ['', path]
    if verbose:
        args.append('-v')
    tests_passed = nose.run(argv=args)
    TempDirectory.delete_all()
    if tests_passed:
        sys.exit(0)
    else:
        sys.exit(1)
Exemple #3
0
    def __call__(self, img_generator):
        test_path = TempDirectory.create_new() / 'menpobench_test_images'
        # Save images down to mat file
        images_to_mat(img_generator, test_path)

        # Call matlab bridge to test file - will drop out a result mat
        invoke_matlab(
            "addpath('{}'); menpobench_matlab_fit('{}', '{}');".format(
                matlab_functions_dir(), self.method_path, test_path))

        return load_matlab_results(test_path)
Exemple #4
0
    def __call__(self, img_generator):
        test_path = TempDirectory.create_new() / "menpobench_test_images"
        # Save images down to mat file
        images_to_mat(img_generator, test_path)

        # Call matlab bridge to test file - will drop out a result mat
        invoke_matlab(
            "addpath('{}'); menpobench_matlab_fit('{}', '{}');".format(
                matlab_functions_dir(), self.method_path, test_path
            )
        )

        return load_matlab_results(test_path)
Exemple #5
0
def invoke_benchmark(
    experiment_name, output_dir=None, overwrite=False, matlab=False, upload=False, force=False, force_upload=False
):
    print("")
    print(centre_str("- - - -  M E N P O B E N C H  - - - -"))
    if upload:
        if not can_upload():
            raise MenpoCDNCredentialsMissingError(
                "MENPO_CDN_S3_ACCESS_KEY and MENPO_CDN_S3_SECRECT_KEY both are" " needed to upload cached results"
            )
        print(centre_str("** MENPO CDN UPLOAD ENABLED **"))
        if force_upload:
            print(centre_str("** UPLOAD FORCED **"))
    print(centre_str("v" + menpobench.__version__))
    print(centre_str("config: {}".format(experiment_name)))
    if output_dir is not None:
        print(centre_str("output: {}".format(output_dir)))
    print(centre_str("cache: {}".format(resolve_cache_dir())))
    if force:
        print(centre_str("FORCED RECOMPUTATION ENABLED"))

    # Load the experiment and check it's schematically valid
    ex = retrieve_experiment(experiment_name)

    # Check if we have any dependency on matlab
    if ex.depends_on_matlab:
        print(centre_str("matlab: {}".format(resolve_matlab_bin_path())))

    print("")
    if output_dir is not None:
        # Handle the creation of the output directory
        output_dir = Path(norm_path(output_dir))
        if output_dir.is_dir():
            if not overwrite:
                raise OutputDirExistsError(
                    "Output directory {} already exists.\n"
                    "Pass '--overwrite' if you want menpobench to delete this "
                    "directory automatically.".format(output_dir)
                )
            else:
                print("--overwrite passed and output directory {} exists - " "deleting\n".format(output_dir))
                shutil.rmtree(str(output_dir))
        output_dir.mkdir()
        errors_dir = output_dir / "errors"
        results_dir = output_dir / "results"
        errors_dir.mkdir()
        results_dir.mkdir()
        results_trainable_dir = results_dir / "trainable_methods"
        results_untrainable_dir = results_dir / "untrainable_methods"
        errors_trainable_dir = errors_dir / "trainable_methods"
        errors_untrainable_dir = errors_dir / "untrainable_methods"
        save_yaml(ex.config, str(output_dir / "experiment.yaml"))

    run = partial(
        run_method,
        ex,
        upload=upload,
        force=force,
        force_upload=force_upload,
        matlab=matlab,
        output=(output_dir is not None),
    )
    try:
        if ex.n_trainable_methods > 0:
            print(centre_str("I. TRAINABLE METHODS"))
            if output_dir is not None:
                results_trainable_dir.mkdir()
                errors_trainable_dir.mkdir()
            else:
                results_trainable_dir = None
                errors_trainable_dir = None

            for i, train in enumerate(ex.trainable_methods, 1):
                print(centre_str("{}/{} - {}".format(i, ex.n_trainable_methods, train), c="="))
                run(train, trainable=True, errors_dir=errors_trainable_dir, results_dir=results_trainable_dir)

        if ex.n_untrainable_methods > 0:
            print(centre_str("II. UNTRAINABLE METHODS", c=" "))
            if output_dir is not None:
                results_untrainable_dir.mkdir()
                errors_untrainable_dir.mkdir()
            else:
                results_untrainable_dir = None
                errors_untrainable_dir = None

            for i, test in enumerate(ex.untrainable_methods, 1):
                print(centre_str("{}/{} - " "{}".format(i, ex.n_untrainable_methods, test), c="="))
                run(test, trainable=False, errors_dir=errors_untrainable_dir, results_dir=results_untrainable_dir)

        # We now have all the results computed - draw the CED curves.
        if output_dir is not None:
            plot_ceds(output_dir)
    finally:
        TempDirectory.delete_all()
Exemple #6
0
def invoke_benchmark(experiment_name,
                     output_dir=None,
                     overwrite=False,
                     matlab=False,
                     upload=False,
                     force=False,
                     force_upload=False):
    print('')
    print(centre_str('- - - -  M E N P O B E N C H  - - - -'))
    if upload:
        if not can_upload():
            raise MenpoCDNCredentialsMissingError(
                'MENPO_CDN_S3_ACCESS_KEY and MENPO_CDN_S3_SECRECT_KEY both are'
                ' needed to upload cached results')
        print(centre_str('** MENPO CDN UPLOAD ENABLED **'))
        if force_upload:
            print(centre_str('** UPLOAD FORCED **'))
    print(centre_str('v' + menpobench.__version__))
    print(centre_str('config: {}'.format(experiment_name)))
    if output_dir is not None:
        print(centre_str('output: {}'.format(output_dir)))
    print(centre_str('cache: {}'.format(resolve_cache_dir())))
    if force:
        print(centre_str('FORCED RECOMPUTATION ENABLED'))

    # Load the experiment and check it's schematically valid
    ex = retrieve_experiment(experiment_name)

    # Check if we have any dependency on matlab
    if ex.depends_on_matlab:
        print(centre_str('matlab: {}'.format(resolve_matlab_bin_path())))

    print('')
    if output_dir is not None:
        # Handle the creation of the output directory
        output_dir = Path(norm_path(output_dir))
        if output_dir.is_dir():
            if not overwrite:
                raise OutputDirExistsError(
                    "Output directory {} already exists.\n"
                    "Pass '--overwrite' if you want menpobench to delete this "
                    "directory automatically.".format(output_dir))
            else:
                print('--overwrite passed and output directory {} exists - '
                      'deleting\n'.format(output_dir))
                shutil.rmtree(str(output_dir))
        output_dir.mkdir()
        errors_dir = output_dir / 'errors'
        results_dir = output_dir / 'results'
        errors_dir.mkdir()
        results_dir.mkdir()
        results_trainable_dir = results_dir / 'trainable_methods'
        results_untrainable_dir = results_dir / 'untrainable_methods'
        errors_trainable_dir = errors_dir / 'trainable_methods'
        errors_untrainable_dir = errors_dir / 'untrainable_methods'
        save_yaml(ex.config, str(output_dir / 'experiment.yaml'))

    run = partial(run_method,
                  ex,
                  upload=upload,
                  force=force,
                  force_upload=force_upload,
                  matlab=matlab,
                  output=(output_dir is not None))
    try:
        if ex.n_trainable_methods > 0:
            print(centre_str('I. TRAINABLE METHODS'))
            if output_dir is not None:
                results_trainable_dir.mkdir()
                errors_trainable_dir.mkdir()
            else:
                results_trainable_dir = None
                errors_trainable_dir = None

            for i, train in enumerate(ex.trainable_methods, 1):
                print(
                    centre_str('{}/{} - {}'.format(i, ex.n_trainable_methods,
                                                   train),
                               c='='))
                run(train,
                    trainable=True,
                    errors_dir=errors_trainable_dir,
                    results_dir=results_trainable_dir)

        if ex.n_untrainable_methods > 0:
            print(centre_str('II. UNTRAINABLE METHODS', c=' '))
            if output_dir is not None:
                results_untrainable_dir.mkdir()
                errors_untrainable_dir.mkdir()
            else:
                results_untrainable_dir = None
                errors_untrainable_dir = None

            for i, test in enumerate(ex.untrainable_methods, 1):
                print(
                    centre_str('{}/{} - '
                               '{}'.format(i, ex.n_untrainable_methods, test),
                               c='='))
                run(test,
                    trainable=False,
                    errors_dir=errors_untrainable_dir,
                    results_dir=results_untrainable_dir)

        # We now have all the results computed - draw the CED curves.
        if output_dir is not None:
            plot_ceds(output_dir)
    finally:
        TempDirectory.delete_all()
 def _unpacked_cache_dir(self):
     if self._unpacked_temp_dir is None:
         self._unpacked_temp_dir = TempDirectory.create_new()
     return self._unpacked_temp_dir
Exemple #8
0
 def _unpacked_cache_dir(self):
     if self._unpacked_temp_dir is None:
         self._unpacked_temp_dir = TempDirectory.create_new()
     return self._unpacked_temp_dir