def clean(benchmark, token=None, filename='all'): benchmark = Benchmark(benchmark) # Delete result files output_folder = benchmark.get_output_folder() if output_folder.exists() and filename == 'all': print(f"rm -rf {output_folder}") rm_folder(output_folder) else: was_removed = False for ext in [".csv", ".html", ".parquet"]: if ext == ".html": to_remove = output_folder / f"{benchmark.name}_{filename}" else: to_remove = output_folder / filename file = to_remove.with_suffix(ext) if file.exists(): was_removed = True print(f"rm {file}") file.unlink() json_path = output_folder / "cache_run_list.json" if was_removed and json_path.exists(): print(f"Removing {filename}.{ext} entry from {json_path}") with open(json_path, "r") as cache_run: json_file = json.load(cache_run) json_file.pop(f"{filename}.{ext}", None) with open(json_path, "w") as cache_run: json.dump(json_file, cache_run) # Delete cache files print("Clear joblib cache") benchmark.mem.clear(warn=False)
def plot(benchmark, filename=None, kinds=('suboptimality_curve', ), display=True, html=True, plotly=False, all_files=False): if all_files: assert filename is None, ( "Cannot use `--all` and `--filename` simultaneously.") assert html, '`--all` can only be used for HTML plot generation.' filename = 'all' if html and len(kinds) > 0: warnings.warn("Cannot specify '--kind' for HTML plot, this options " "will be ignored.") # Get the result file benchmark = Benchmark(benchmark) result_filename = benchmark.get_result_file(filename) # Plot the results. from benchopt.plotting import plot_benchmark plot_benchmark(result_filename, benchmark, kinds=kinds, display=display, plotly=plotly, html=html)
def plot(benchmark, filename=None, kinds=('suboptimality_curve', ), display=True, plotly=False): # Get the result file benchmark = Benchmark(benchmark) result_filename = benchmark.get_result_file(filename) # Plot the results. df = pd.read_csv(result_filename) plot_benchmark(df, benchmark, kinds=kinds, display=display, plotly=plotly)
def clean(benchmark, token=None, filename=None): benchmark = Benchmark(benchmark) # Delete result files output_folder = benchmark.get_output_folder() print(f"rm -rf {output_folder}") rm_folder(output_folder) # Delete cache files cache_folder = benchmark.get_cache_location() print(f"rm -rf {cache_folder}") rm_folder(cache_folder)
def config(ctx, benchmark, token=None, filename=None): ctx.ensure_object(dict) if benchmark is None: config = get_global_config_file() else: benchmark = Benchmark(benchmark) config = benchmark.get_config_file() if ctx.invoked_subcommand is None: print(f"Config file is: {config.resolve()}") ctx.obj['config'] = config ctx.obj['benchmark_name'] = benchmark.name if benchmark else None
def info(benchmark, solver_names, dataset_names, env_name='False', verbose=False): # benchmark benchmark = Benchmark(benchmark) print(f"Info regarding the benchmark '{benchmark.name}'") # validate solvers and datasets benchmark.validate_dataset_patterns(dataset_names) benchmark.validate_solver_patterns(solver_names) # get solvers and datasets in the benchmark all_solvers = benchmark.get_solvers() all_datasets = benchmark.get_datasets() # enable verbosity if any environment was provided if env_name is not None and env_name != 'False': verbose = True # conda env check only in verbose case if verbose: # Check conda env name env_name = check_conda_env(env_name, benchmark.name) # check conda environment validity check_benchopt = _run_shell_in_conda_env("benchopt --version", env_name=env_name, capture_stdout=True) if check_benchopt != 0: warnings.warn( f"Environment '{env_name}' does not exist " "or is not configured for benchopt, " "benchmark requirement availability will not be checked, " "see the command `benchopt install`.", UserWarning) env_name = None else: print("Checking benchmark requirement availability " f"in env '{env_name}'.") print("Note: you can install all dependencies from a benchmark " "with the command `benchopt install`.") # enable verbosity if any solver/dataset are specified in input if dataset_names or solver_names: verbose = True # print information print("-" * 10) if not dataset_names and not solver_names: dataset_names = ['all'] solver_names = ['all'] if dataset_names: print("# DATASETS", flush=True) print_info(dataset_names, all_datasets, env_name, verbose) if solver_names: print("# SOLVERS", flush=True) print_info(solver_names, all_solvers, env_name, verbose)
def pytest_generate_tests(metafunc): """Generate the test on the fly to take --benchmark into account. """ PARAMETRIZATION = { ('benchmark', 'dataset_simu'): lambda benchmarks: [(benchmark, dataset_class) for benchmark in benchmarks for dataset_class in benchmark.list_benchmark_datasets() if dataset_class.name.lower() == 'simulated'], ('benchmark', 'dataset_class'): lambda benchmarks: [(benchmark, dataset_class) for benchmark in benchmarks for dataset_class in benchmark.list_benchmark_datasets()], ('benchmark', 'solver_class'): lambda benchmarks: [(benchmark, solver) for benchmark in benchmarks for solver in benchmark.list_benchmark_solvers()] } # Get all benchmarks benchmarks = metafunc.config.getoption("benchmark") if benchmarks is None or len(benchmarks) == 0: benchmarks = TEST_BENCHMARK_DIR.glob('*/') benchmarks = [Benchmark(b) for b in benchmarks] benchmarks.sort() # Parametrize the tests for params, func in PARAMETRIZATION.items(): if set(params).issubset(metafunc.fixturenames): metafunc.parametrize(params, func(benchmarks), ids=class_ids)
def find_benchmark_in_args(args): "Find the benchmark in preceeding args for benchmark dependent completion." for b in args: if (Path(b) / "objective.py").exists(): return Benchmark(b) return None
def plot(benchmark, filename=None, kinds=('suboptimality_curve', ), display=True, plotly=False): # Get the result file benchmark = Benchmark(benchmark) result_filename = benchmark.get_result_file(filename) # Load the results. import pandas as pd df = pd.read_csv(result_filename) # Plot the results. from benchopt.plotting import plot_benchmark plot_benchmark(df, benchmark, kinds=kinds, display=display, plotly=plotly)
def install(benchmark, solver_names, dataset_names, force=False, recreate=False, env_name='False', confirm=False, quiet=False): # Check that the dataset/solver patterns match actual dataset benchmark = Benchmark(benchmark) print(f"Installing '{benchmark.name}' requirements") benchmark.validate_dataset_patterns(dataset_names) benchmark.validate_solver_patterns(solver_names) # Get a list of all conda envs default_conda_env, conda_envs = list_conda_envs() # If env_name is False (default), installation in the current environement. if env_name == 'False': env_name = None # incompatible with the 'recreate' flag to avoid messing with the # user environement if recreate: msg = "Cannot recreate conda env without using options " + \ "'-e/--env' or '--env-name'." raise RuntimeError(msg) # check if any current conda environment if default_conda_env is not None: # ask for user confirmation to install in current conda env if not confirm: click.confirm( f"Install in the current env '{default_conda_env}'?", abort=True ) else: raise RuntimeError("No conda environment is activated.") else: # If env_name is True, the flag `--env` has been used. Create a conda # env specific to the benchmark. Else, use the <env_name> value. if env_name == 'True': env_name = f"benchopt_{benchmark.name}" else: # check provided <env_name> # (to avoid empty name like `--env-name ""`) if len(env_name) == 0: raise RuntimeError("Empty environment name.") # avoid recreating 'base' conda env` if env_name == 'base' and recreate: raise RuntimeError( "Impossible to recreate 'base' conda environment." ) # create environment if necessary create_conda_env(env_name, recreate=recreate, quiet=quiet) # install requirements print("# Install", flush=True) benchmark.install_all_requirements( include_solvers=solver_names, include_datasets=dataset_names, env_name=env_name, force=force, quiet=quiet, )
def find_benchmark_in_args(args): "Find the benchmark in preceeding args for benchmark dependent completion." args.extend([Path.cwd()]) # default path is current working directory for b in args: if (Path(b) / "objective.py").exists(): return Benchmark(b) return None
def publish(benchmark, token=None, filename=None): if token is None: token = get_global_setting('github_token') if token is None: raise RuntimeError( "Could not find the token value to connect to GitHub.\n\n" "Please go to https://github.com/settings/tokens to generate a " "personal token $TOKEN.\nThen, either provide it with option `-t` " "or put it in a config file ./benchopt.ini\nunder section " "[benchopt] as `github_token = $TOKEN`.") # Get the result file benchmark = Benchmark(benchmark) result_filename = benchmark.get_result_file(filename) # Publish the result. publish_result_file(benchmark.name, result_filename, token)
def check_install(benchmark, module_filename, base_class_name): # benchmark benchmark = Benchmark(benchmark) # Get class to check klass = _load_class_from_module(module_filename, base_class_name, benchmark.benchmark_dir) klass.is_installed(raise_on_not_installed=True)
def test(benchmark, env_name, pytest_args): benchmark = Benchmark(benchmark) from benchopt.tests import __file__ as _bench_test_module _bench_test_module = Path(_bench_test_module).parent pytest_args = ' '.join(("-p benchopt.tests.fixtures", f"--rootdir {_bench_test_module}", *pytest_args)) if len(pytest_args) == 0: pytest_args = '-vl' env_option = '' if env_name is not None: create_conda_env(env_name, with_pytest=True) if _run_shell_in_conda_env("pytest --version", env_name=env_name) != 0: raise ModuleNotFoundError( f"pytest is not installed in conda env {env_name}.\n" f"Please run `conda install -n {env_name} pytest` to test the " "benchmark in this environment.") objective = benchmark.get_benchmark_objective() if not objective.is_installed(): objective.install(env_name=env_name) env_option = f'--test-env {env_name}' _bench_test_file = _bench_test_module / "test_benchmarks.py" cmd = ( f'pytest {pytest_args} {_bench_test_file} ' f'--benchmark {benchmark.benchmark_dir} {env_option} ' # Make sure to not modify sys.path to add test file from current env # in sub conda env as there might be different python versions. '--import-mode importlib') raise SystemExit( _run_shell_in_conda_env(cmd, env_name=env_name, capture_stdout=False) != 0)
def archive(benchmark, with_outputs): benchmark = Benchmark(benchmark) bench_dir = benchmark.benchmark_dir to_archive = ARCHIVE_ELEMENTS if with_outputs: to_archive = to_archive + ['outputs/*'] archive_name = f"{benchmark.name}.tar.gz" print(f"Creating {archive_name}...", end='', flush=True) with tarfile.open(archive_name, "w:gz") as tar: for elem_pattern in to_archive: for sub_elem in bench_dir.glob(elem_pattern): tar.add(sub_elem, sub_elem.relative_to(bench_dir.parent), filter=clean_archive) print(f"done\nResults are in {archive_name}")
import re from pathlib import Path from benchopt.benchmark import Benchmark from benchopt.utils.stream_redirection import SuppressStd # Default benchmark TEST_BENCHMARK_DIR = Path(__file__).parent / 'test_benchmarks' DUMMY_BENCHMARK_PATH = TEST_BENCHMARK_DIR / 'dummy_benchmark' # Pattern to select specific datasets or solvers. SELECT_ONE_SIMULATED = r'simulated*500*rho=0\]' SELECT_ONE_PGD = r'python-pgd*step_size=1\]' try: DUMMY_BENCHMARK = Benchmark(DUMMY_BENCHMARK_PATH) TEST_OBJECTIVE = DUMMY_BENCHMARK.get_benchmark_objective() TEST_SOLVER = [s for s in DUMMY_BENCHMARK.list_benchmark_solvers() if s.name == "Test-Solver"][0] TEST_DATASET = [d for d in DUMMY_BENCHMARK.list_benchmark_datasets() if d.name == "Test-Dataset"][0] except Exception: DUMMY_BENCHMARK = None TEST_OBJECTIVE = None TEST_SOLVER = None TEST_DATASET = None class CaptureRunOutput(object): """Context to capture run cmd output and files. """
def run(benchmark, solver_names, forced_solvers, dataset_names, objective_filters, max_runs, n_repetitions, timeout, plot=True, html=True, pdb=False, do_profile=False, env_name='False', old_objective_filters=None): if len(old_objective_filters): warnings.warn( 'Using the -p option is deprecated, use -o instead', FutureWarning, ) objective_filters = old_objective_filters from benchopt.runner import run_benchmark if do_profile: from benchopt.utils.profiling import use_profile use_profile() # needs to be called before validate_solver_patterns # Check that the dataset/solver patterns match actual dataset benchmark = Benchmark(benchmark) benchmark.validate_dataset_patterns(dataset_names) benchmark.validate_solver_patterns(solver_names+forced_solvers) benchmark.validate_objective_filters(objective_filters) # If env_name is False, the flag `--local` has been used (default) so # run in the current environement. if env_name == 'False': run_benchmark( benchmark, solver_names, forced_solvers, dataset_names=dataset_names, objective_filters=objective_filters, max_runs=max_runs, n_repetitions=n_repetitions, timeout=timeout, plot_result=plot, html=html, pdb=pdb ) print_stats() # print profiling stats (does nothing if not profiling) return _, all_conda_envs = list_conda_envs() # If env_name is True, the flag `--env` has been used. Create a conda env # specific to the benchmark (if not existing). # Else, use the <env_name> value. if env_name == 'True': env_name = f"benchopt_{benchmark.name}" install_cmd = f"`benchopt install -e {benchmark.benchmark_dir}`" else: # check provided <env_name> # (to avoid empty name like `--env-name ""`) if len(env_name) == 0: raise RuntimeError("Empty environment name.") install_cmd = ( f"`benchopt install --env-name {env_name} " f"{benchmark.benchmark_dir}`" ) # check if the environment exists if env_name not in all_conda_envs: raise RuntimeError( f"The default env '{env_name}' for benchmark {benchmark.name} " f"does not exist. Make sure to run {install_cmd} to create the " "benchmark and install the dependencies." ) # check if environment was set up with benchopt if get_benchopt_version_in_env(env_name) is None: raise RuntimeError( f"benchopt is not installed in env '{env_name}', " "see the command `benchopt install` to setup the environment." ) # run the command in the conda env solvers_option = ' '.join(['-s ' + s for s in solver_names]) forced_solvers_option = ' '.join([f"-f '{s}'" for s in forced_solvers]) datasets_option = ' '.join([f"-d '{d}'" for d in dataset_names]) objective_option = ' '.join([f"-p '{p}'" for p in objective_filters]) cmd = ( rf"benchopt run --local {benchmark.benchmark_dir} " rf"--n-repetitions {n_repetitions} " rf"--max-runs {max_runs} --timeout {timeout} " rf"{solvers_option} {forced_solvers_option} " rf"{datasets_option} {objective_option} " rf"{'--plot' if plot else '--no-plot'} " rf"{'--html' if html else '--no-html'} " rf"{'--pdb' if pdb else ''} " .replace('\\', '\\\\') ) raise SystemExit(_run_shell_in_conda_env( cmd, env_name=env_name, capture_stdout=False ) != 0)
=========================== """ import os from pathlib import Path import matplotlib.pyplot as plt from benchopt import run_benchmark from benchopt.benchmark import Benchmark from benchopt.plotting import plot_benchmark, PLOT_KINDS BENCHMARK_PATH = Path(os.getcwd()).parent / 'benchmarks' / 'logreg_l2' try: save_file = run_benchmark( Benchmark(BENCHMARK_PATH), ['sklearn', 'lightning'], dataset_names=['Simulated*n_samples=200,n_features=500*'], max_runs=100, timeout=20, n_repetitions=3, plot_result=False, show_progress=False) except RuntimeError: raise RuntimeError( "This example can only work when Logreg-l2 benchmark is cloned in a " "`benchmarks` folder. Please run:\n" "$ git clone https://github.com/benchopt/benchmark_logreg_l2 " f"{BENCHMARK_PATH.resolve()}") kinds = list(PLOT_KINDS.keys()) figs = plot_benchmark(save_file,
import matplotlib.pyplot as plt from benchopt import run_benchmark from benchopt.benchmark import Benchmark from benchopt.plotting import plot_benchmark, PLOT_KINDS BENCHMARK_PATH = Path().resolve().parent / 'benchmarks' / 'benchmark_lasso' if not BENCHMARK_PATH.exists(): raise RuntimeError( "This example can only work when Lasso benchmark is cloned in the " "example folder. Please run:\n" "$ git clone https://github.com/benchopt/benchmark_lasso " f"{BENCHMARK_PATH.resolve()}") save_file = run_benchmark( Benchmark(BENCHMARK_PATH), ['Python-PGD[use_acceleration=False]', 'R-PGD'], dataset_names=["Simulated[n_features=5000,n_samples=100,rho=0]"], objective_filters=['*[fit_intercept=False,reg=0.5]'], max_runs=100, timeout=100, n_repetitions=5, plot_result=False, show_progress=False) kinds = list(PLOT_KINDS.keys()) figs = plot_benchmark(save_file, benchmark=Benchmark(BENCHMARK_PATH), kinds=kinds, html=False) plt.show()
def get_datasets(ctx, args, incomplete): skip_import() benchmark = Benchmark(args[1]) datasets = benchmark.list_benchmark_dataset_names() datasets = [d.lower() for d in datasets] return [d for d in datasets if incomplete.lower() in d]
def get_solvers(ctx, args, incomplete): skip_import() benchmark = Benchmark(args[1]) solvers = benchmark.list_benchmark_solver_names() solvers = [s.lower() for s in solvers] return [s for s in solvers if incomplete.lower() in s]
=========================== """ import os from pathlib import Path import matplotlib.pyplot as plt from benchopt import run_benchmark from benchopt.benchmark import Benchmark from benchopt.plotting import plot_benchmark, PLOT_KINDS BENCHMARK_PATH = Path(os.getcwd()).parent / 'benchmarks' / 'logreg_l2' try: df = run_benchmark( Benchmark(BENCHMARK_PATH), ['sklearn', 'lightning'], dataset_names=['Simulated*n_samples=200,n_features=500*'], max_runs=100, timeout=20, n_repetitions=3, plot_result=False, show_progress=False) except RuntimeError: raise RuntimeError( "This example can only work when Lasso benchmark is cloned in the " "example folder. Please run:\n" "$ git clone https://github.com/benchopt/benchmark_logreg_l2 " f"{BENCHMARK_PATH.resolve()}") kinds = list(PLOT_KINDS.keys()) figs = plot_benchmark(df, benchmark=Benchmark(BENCHMARK_PATH), kinds=kinds)
from pathlib import Path import matplotlib.pyplot as plt from benchopt import run_benchmark from benchopt.benchmark import Benchmark from benchopt.plotting import plot_benchmark, PLOT_KINDS BENCHMARK_PATH = ( Path().resolve().parent / 'benchmarks' / 'benchmark_logreg_l2' ) try: save_file = run_benchmark( Benchmark(BENCHMARK_PATH), ['sklearn[liblinear]', 'sklearn[newton-cg]', 'lightning'], dataset_names=['Simulated*n_samples=200,n_features=500*'], objective_filters=[ 'L2 Logistic Regression[fit_intercept=False,lmbd=1.0]'], max_runs=100, timeout=20, n_repetitions=15, plot_result=False, show_progress=True ) except RuntimeError: raise RuntimeError( "This example can only work when Logreg-l2 benchmark is cloned in a " "`benchmarks` folder. Please run:\n" "$ git clone https://github.com/benchopt/benchmark_logreg_l2 " f"{BENCHMARK_PATH.resolve()}" )
=========================== """ from pathlib import Path import matplotlib.pyplot as plt from benchopt import run_benchmark from benchopt.benchmark import Benchmark from benchopt.plotting import plot_benchmark, PLOT_KINDS BENCHMARK_PATH = (Path().resolve().parent / 'benchmarks' / 'benchmark_logreg_l2') try: save_file = run_benchmark( Benchmark(BENCHMARK_PATH), ['sklearn[liblinear]', 'sklearn[newton-cg]', 'lightning'], dataset_names=['Simulated*[n_features=500,n_samples=200]'], objective_filters=['L2 Logistic Regression[lmbd=1.0]'], max_runs=100, timeout=20, n_repetitions=15, plot_result=False, show_progress=True) except RuntimeError: raise RuntimeError( "This example can only work when Logreg-l2 benchmark is cloned in a " "`benchmarks` folder. Please run:\n" "$ git clone https://github.com/benchopt/benchmark_logreg_l2 " f"{BENCHMARK_PATH.resolve()}")
def run(config_file=None, **kwargs): if config_file is not None: with open(config_file, "r") as f: config = yaml.safe_load(f) else: config = {} # XXX - Remove old and deprecated objective filters in version 1.3 (benchmark, solver_names, forced_solvers, dataset_names, objective_filters, max_runs, n_repetitions, timeout, n_jobs, slurm, plot, html, pdb, do_profile, env_name, output, deprecated_objective_filters, old_objective_filters) = _get_run_args(kwargs, config) if len(old_objective_filters): warnings.warn( 'Using the -p option is deprecated, use -o instead', FutureWarning, ) objective_filters = old_objective_filters if len(deprecated_objective_filters): warnings.warn( 'Using the --objective-filters option is deprecated, ' 'use --objective instead', FutureWarning) objective_filters = deprecated_objective_filters # Create the Benchmark object benchmark = Benchmark(benchmark) # If env_name is False, the flag `--local` has been used (default) so # run in the current environment. if env_name == 'False': print("Benchopt is running") if slurm is not None: print("Running on SLURM") set_slurm_launch() from benchopt.runner import run_benchmark if do_profile: from benchopt.utils.profiling import use_profile use_profile() # needs to be called before validate_solver_patterns # Check that the dataset/solver patterns match actual dataset benchmark.validate_dataset_patterns(dataset_names) benchmark.validate_objective_filters(objective_filters) # pyyaml returns tuples: make sure everything is a list benchmark.validate_solver_patterns( list(solver_names) + list(forced_solvers)) run_benchmark(benchmark, solver_names, forced_solvers, dataset_names=dataset_names, objective_filters=objective_filters, max_runs=max_runs, n_repetitions=n_repetitions, timeout=timeout, n_jobs=n_jobs, slurm=slurm, plot_result=plot, html=html, pdb=pdb, output=output) print_stats() # print profiling stats (does nothing if not profiling) return default_conda_env, all_conda_envs = list_conda_envs() # If env_name is True, the flag `--env` has been used. Create a conda env # specific to the benchmark (if not existing). # Else, use the <env_name> value. # check if any current conda environment if default_conda_env is None: raise RuntimeError( "No conda environment is activated. " "You should be in a conda environment to use " "'benchopt run' with options '-e/--env' or '--env-name'.") if env_name == 'True': env_name = f"benchopt_{benchmark.name}" install_cmd = f"`benchopt install -e {benchmark.benchmark_dir}`" else: # check provided <env_name> # (to avoid empty name like `--env-name ""`) if len(env_name) == 0: raise RuntimeError("Empty environment name.") install_cmd = (f"`benchopt install --env-name {env_name} " f"{benchmark.benchmark_dir}`") # check if the environment exists if env_name not in all_conda_envs: raise RuntimeError( f"The default env '{env_name}' for benchmark {benchmark.name} " f"does not exist. Make sure to run {install_cmd} to create the " "benchmark and install the dependencies.") print(f"Launching benchopt in env {env_name}") # check if environment was set up with benchopt benchopt_version, is_editable = get_benchopt_version_in_env(env_name) if benchopt_version is None: raise RuntimeError( f"benchopt is not installed in env '{env_name}', " "see the command `benchopt install` to setup the environment.") # check against running version from benchopt import __version__ as benchopt_version_running _, is_editable_running = get_benchopt_requirement() if (benchopt_version_running != benchopt_version and not (is_editable_running and is_editable)): warnings.warn( f"Benchopt running version ({benchopt_version_running}) " f"and version in env {env_name} ({benchopt_version}) differ") # run the command in the conda env solvers_option = ' '.join([f"-s '{s}'" for s in solver_names]) forced_solvers_option = ' '.join([f"-f '{s}'" for s in forced_solvers]) datasets_option = ' '.join([f"-d '{d}'" for d in dataset_names]) objective_option = ' '.join([f"-o '{o}'" for o in objective_filters]) cmd = (rf"benchopt run --local {benchmark.benchmark_dir} " rf"--n-repetitions {n_repetitions} " rf"--max-runs {max_runs} --timeout {timeout} " rf"--n-jobs {n_jobs} {'--slurm' if slurm else ''} " rf"{solvers_option} {forced_solvers_option} " rf"{datasets_option} {objective_option} " rf"{'--plot' if plot else '--no-plot'} " rf"{'--html' if html else '--no-html'} " rf"{'--pdb' if pdb else ''} " rf"--output {output}".replace('\\', '\\\\')) raise SystemExit( _run_shell_in_conda_env(cmd, env_name=env_name, capture_stdout=False) != 0)
""" import os from pathlib import Path import matplotlib.pyplot as plt from benchopt import run_benchmark from benchopt.benchmark import Benchmark from benchopt.tests import SELECT_ONE_SIMULATED from benchopt.plotting import plot_benchmark, PLOT_KINDS BENCHMARK_PATH = Path(os.getcwd()).parent / 'benchmarks' / 'lasso' try: df = run_benchmark( Benchmark(BENCHMARK_PATH), ['Python-PGD[^-]*use_acceleration=False', 'R-PGD', 'Julia-PGD'], dataset_names=[SELECT_ONE_SIMULATED], objective_filters=['reg=0.5'], max_runs=100, timeout=100, n_repetitions=5, plot_result=False, show_progress=False) except RuntimeError: raise RuntimeError( "This example can only work when Lasso benchmark is cloned in the " "example folder. Please run:\n" "$ git clone https://github.com/benchopt/benchmark_lasso " f"{BENCHMARK_PATH.resolve()}")
def run(benchmark, solver_names, forced_solvers, dataset_names, objective_filters, max_runs, n_repetitions, timeout, recreate=False, plot=True, pdb=False, env_name='False'): # Check that the dataset/solver patterns match actual dataset benchmark = Benchmark(benchmark) benchmark.validate_dataset_patterns(dataset_names) benchmark.validate_solver_patterns(solver_names + forced_solvers) # If env_name is False, the flag `--local` has been used (default) so # run in the current environement. if env_name == 'False': run_benchmark(benchmark, solver_names, forced_solvers, dataset_names=dataset_names, objective_filters=objective_filters, max_runs=max_runs, n_repetitions=n_repetitions, timeout=timeout, plot_result=plot, pdb=pdb) return # If env_name is True, the flag `--env` has been used. Create a conda env # specific to the benchmark. Else, use the <env_name> value. if env_name == 'True': env_name = f"benchopt_{benchmark.name}" create_conda_env(env_name, recreate=recreate) # installed required datasets benchmark.install_required_datasets(dataset_names, env_name=env_name) # Get the solvers and install them benchmark.install_required_solvers(solver_names, forced_solvers=forced_solvers, env_name=env_name) # run the command in the conda env solvers_option = ' '.join(['-s ' + s for s in solver_names]) forced_solvers_option = ' '.join(['-f ' + s for s in forced_solvers]) datasets_option = ' '.join(['-d ' + d for d in dataset_names]) objective_option = ' '.join(['-p ' + p for p in objective_filters]) cmd = (rf"benchopt run --local {benchmark.benchmark_dir} " rf"--n-repetitions {n_repetitions} " rf"--max-runs {max_runs} --timeout {timeout} " rf"{solvers_option} {forced_solvers_option} " rf"{datasets_option} {objective_option} " rf"{'--plot' if plot else '--no-plot'} " rf"{'--pdb' if pdb else ''} ".replace('\\', '\\\\')) raise SystemExit( _run_shell_in_conda_env(cmd, env_name=env_name, capture_stdout=False) != 0)
from pathlib import Path from benchopt.benchmark import Benchmark from benchopt.utils.stream_redirection import SuppressStd # Default benchmark TEST_BENCHMARK_DIR = Path(__file__).parent / 'test_benchmarks' DUMMY_BENCHMARK_PATH = TEST_BENCHMARK_DIR / 'dummy_benchmark' # Pattern to select specific datasets or solvers. SELECT_ONE_SIMULATED = r'simulated*500*rho=0]' SELECT_ONE_PGD = r'python-pgd*step_size=1]' SELECT_ONE_OBJECTIVE = r'dummy*reg=0.1]' try: DUMMY_BENCHMARK = Benchmark(DUMMY_BENCHMARK_PATH) TEST_OBJECTIVE = DUMMY_BENCHMARK.get_benchmark_objective() TEST_SOLVER = [ s for s in DUMMY_BENCHMARK.get_solvers() if s.name == "Test-Solver" ][0] TEST_DATASET = [ d for d in DUMMY_BENCHMARK.get_datasets() if d.name == "Test-Dataset" ][0] except Exception: DUMMY_BENCHMARK = None TEST_OBJECTIVE = None TEST_SOLVER = None TEST_DATASET = None class CaptureRunOutput(object):
from pathlib import Path from benchopt.benchmark import Benchmark # Default benchmark TEST_BENCHMARK_DIR = Path(__file__).parent / 'test_benchmarks' DUMMY_BENCHMARK_PATH = TEST_BENCHMARK_DIR / 'dummy_benchmark' # Pattern to select specific datasets or solvers. SELECT_ONE_SIMULATED = r'simulated*500*rho=0\]' SELECT_ONE_PGD = r'python-pgd*step_size=1\]' try: DUMMY_BENCHMARK = Benchmark(DUMMY_BENCHMARK_PATH) TEST_OBJECTIVE = DUMMY_BENCHMARK.get_benchmark_objective() TEST_SOLVER = [ s for s in DUMMY_BENCHMARK.list_benchmark_solvers() if s.name == "Test-Solver" ][0] TEST_DATASET = [ d for d in DUMMY_BENCHMARK.list_benchmark_datasets() if d.name == "Test-Dataset" ][0] except Exception: DUMMY_BENCHMARK = None TEST_OBJECTIVE = None TEST_SOLVER = None TEST_DATASET = None
import matplotlib.pyplot as plt from benchopt import run_benchmark from benchopt.benchmark import Benchmark from benchopt.tests import SELECT_ONE_SIMULATED from benchopt.plotting import plot_benchmark, PLOT_KINDS BENCHMARK_PATH = Path().resolve().parent / 'benchmarks' / 'benchmark_lasso' if not BENCHMARK_PATH.exists(): raise RuntimeError( "This example can only work when Lasso benchmark is cloned in the " "example folder. Please run:\n" "$ git clone https://github.com/benchopt/benchmark_lasso " f"{BENCHMARK_PATH.resolve()}") save_file = run_benchmark(Benchmark(BENCHMARK_PATH), ['Python-PGD[use_acceleration=False]', 'R-PGD'], dataset_names=[SELECT_ONE_SIMULATED], objective_filters=['*fit_intercept=False,reg=0.5'], max_runs=100, timeout=100, n_repetitions=5, plot_result=False, show_progress=False) kinds = list(PLOT_KINDS.keys()) figs = plot_benchmark(save_file, benchmark=Benchmark(BENCHMARK_PATH), kinds=kinds, html=False) plt.show()