def clean(benchmark, token=None, filename='all'):
benchmark = Benchmark(benchmark)
# Delete result files
output_folder = benchmark.get_output_folder()
if output_folder.exists() and filename == 'all':
print(f"rm -rf {output_folder}")
rm_folder(output_folder)
else:
was_removed = False
for ext in [".csv", ".html", ".parquet"]:
if ext == ".html":
to_remove = output_folder / f"{benchmark.name}_{filename}"
else:
to_remove = output_folder / filename
file = to_remove.with_suffix(ext)
if file.exists():
was_removed = True
print(f"rm {file}")
file.unlink()
json_path = output_folder / "cache_run_list.json"
if was_removed and json_path.exists():
print(f"Removing {filename}.{ext} entry from {json_path}")
with open(json_path, "r") as cache_run:
json_file = json.load(cache_run)
json_file.pop(f"{filename}.{ext}", None)
with open(json_path, "w") as cache_run:
json.dump(json_file, cache_run)
# Delete cache files
print("Clear joblib cache")
benchmark.mem.clear(warn=False)
def plot(benchmark,
filename=None,
kinds=('suboptimality_curve', ),
display=True,
html=True,
plotly=False,
all_files=False):
if all_files:
assert filename is None, (
"Cannot use `--all` and `--filename` simultaneously.")
assert html, '`--all` can only be used for HTML plot generation.'
filename = 'all'
if html and len(kinds) > 0:
warnings.warn("Cannot specify '--kind' for HTML plot, this options "
"will be ignored.")
# Get the result file
benchmark = Benchmark(benchmark)
result_filename = benchmark.get_result_file(filename)
# Plot the results.
from benchopt.plotting import plot_benchmark
plot_benchmark(result_filename,
benchmark,
kinds=kinds,
display=display,
plotly=plotly,
html=html)
def plot(benchmark,
filename=None,
kinds=('suboptimality_curve', ),
display=True,
plotly=False):
# Get the result file
benchmark = Benchmark(benchmark)
result_filename = benchmark.get_result_file(filename)
# Plot the results.
df = pd.read_csv(result_filename)
plot_benchmark(df, benchmark, kinds=kinds, display=display, plotly=plotly)
def clean(benchmark, token=None, filename=None):
benchmark = Benchmark(benchmark)
# Delete result files
output_folder = benchmark.get_output_folder()
print(f"rm -rf {output_folder}")
rm_folder(output_folder)
# Delete cache files
cache_folder = benchmark.get_cache_location()
print(f"rm -rf {cache_folder}")
rm_folder(cache_folder)
def config(ctx, benchmark, token=None, filename=None):
ctx.ensure_object(dict)
if benchmark is None:
config = get_global_config_file()
else:
benchmark = Benchmark(benchmark)
config = benchmark.get_config_file()
if ctx.invoked_subcommand is None:
print(f"Config file is: {config.resolve()}")
ctx.obj['config'] = config
ctx.obj['benchmark_name'] = benchmark.name if benchmark else None
def info(benchmark,
solver_names,
dataset_names,
env_name='False',
verbose=False):
# benchmark
benchmark = Benchmark(benchmark)
print(f"Info regarding the benchmark '{benchmark.name}'")
# validate solvers and datasets
benchmark.validate_dataset_patterns(dataset_names)
benchmark.validate_solver_patterns(solver_names)
# get solvers and datasets in the benchmark
all_solvers = benchmark.get_solvers()
all_datasets = benchmark.get_datasets()
# enable verbosity if any environment was provided
if env_name is not None and env_name != 'False':
verbose = True
# conda env check only in verbose case
if verbose:
# Check conda env name
env_name = check_conda_env(env_name, benchmark.name)
# check conda environment validity
check_benchopt = _run_shell_in_conda_env("benchopt --version",
env_name=env_name,
capture_stdout=True)
if check_benchopt != 0:
warnings.warn(
f"Environment '{env_name}' does not exist "
"or is not configured for benchopt, "
"benchmark requirement availability will not be checked, "
"see the command `benchopt install`.", UserWarning)
env_name = None
else:
print("Checking benchmark requirement availability "
f"in env '{env_name}'.")
print("Note: you can install all dependencies from a benchmark "
"with the command `benchopt install`.")
# enable verbosity if any solver/dataset are specified in input
if dataset_names or solver_names:
verbose = True
# print information
print("-" * 10)
if not dataset_names and not solver_names:
dataset_names = ['all']
solver_names = ['all']
if dataset_names:
print("# DATASETS", flush=True)
print_info(dataset_names, all_datasets, env_name, verbose)
if solver_names:
print("# SOLVERS", flush=True)
print_info(solver_names, all_solvers, env_name, verbose)
def pytest_generate_tests(metafunc):
"""Generate the test on the fly to take --benchmark into account.
"""
PARAMETRIZATION = {
('benchmark', 'dataset_simu'):
lambda benchmarks: [(benchmark, dataset_class)
for benchmark in benchmarks for dataset_class in
benchmark.list_benchmark_datasets()
if dataset_class.name.lower() == 'simulated'],
('benchmark', 'dataset_class'):
lambda benchmarks: [(benchmark, dataset_class)
for benchmark in benchmarks for dataset_class in
benchmark.list_benchmark_datasets()],
('benchmark', 'solver_class'):
lambda benchmarks: [(benchmark, solver) for benchmark in benchmarks
for solver in benchmark.list_benchmark_solvers()]
}
# Get all benchmarks
benchmarks = metafunc.config.getoption("benchmark")
if benchmarks is None or len(benchmarks) == 0:
benchmarks = TEST_BENCHMARK_DIR.glob('*/')
benchmarks = [Benchmark(b) for b in benchmarks]
benchmarks.sort()
# Parametrize the tests
for params, func in PARAMETRIZATION.items():
if set(params).issubset(metafunc.fixturenames):
metafunc.parametrize(params, func(benchmarks), ids=class_ids)
def find_benchmark_in_args(args):
"Find the benchmark in preceeding args for benchmark dependent completion."
for b in args:
if (Path(b) / "objective.py").exists():
return Benchmark(b)
return None
def plot(benchmark,
filename=None,
kinds=('suboptimality_curve', ),
display=True,
plotly=False):
# Get the result file
benchmark = Benchmark(benchmark)
result_filename = benchmark.get_result_file(filename)
# Load the results.
import pandas as pd
df = pd.read_csv(result_filename)
# Plot the results.
from benchopt.plotting import plot_benchmark
plot_benchmark(df, benchmark, kinds=kinds, display=display, plotly=plotly)
def install(benchmark, solver_names, dataset_names, force=False,
recreate=False, env_name='False', confirm=False, quiet=False):
# Check that the dataset/solver patterns match actual dataset
benchmark = Benchmark(benchmark)
print(f"Installing '{benchmark.name}' requirements")
benchmark.validate_dataset_patterns(dataset_names)
benchmark.validate_solver_patterns(solver_names)
# Get a list of all conda envs
default_conda_env, conda_envs = list_conda_envs()
# If env_name is False (default), installation in the current environement.
if env_name == 'False':
env_name = None
# incompatible with the 'recreate' flag to avoid messing with the
# user environement
if recreate:
msg = "Cannot recreate conda env without using options " + \
"'-e/--env' or '--env-name'."
raise RuntimeError(msg)
# check if any current conda environment
if default_conda_env is not None:
# ask for user confirmation to install in current conda env
if not confirm:
click.confirm(
f"Install in the current env '{default_conda_env}'?",
abort=True
)
else:
raise RuntimeError("No conda environment is activated.")
else:
# If env_name is True, the flag `--env` has been used. Create a conda
# env specific to the benchmark. Else, use the <env_name> value.
if env_name == 'True':
env_name = f"benchopt_{benchmark.name}"
else:
# check provided <env_name>
# (to avoid empty name like `--env-name ""`)
if len(env_name) == 0:
raise RuntimeError("Empty environment name.")
# avoid recreating 'base' conda env`
if env_name == 'base' and recreate:
raise RuntimeError(
"Impossible to recreate 'base' conda environment."
)
# create environment if necessary
create_conda_env(env_name, recreate=recreate, quiet=quiet)
# install requirements
print("# Install", flush=True)
benchmark.install_all_requirements(
include_solvers=solver_names, include_datasets=dataset_names,
env_name=env_name, force=force, quiet=quiet,
)
def find_benchmark_in_args(args):
"Find the benchmark in preceeding args for benchmark dependent completion."
args.extend([Path.cwd()]) # default path is current working directory
for b in args:
if (Path(b) / "objective.py").exists():
return Benchmark(b)
return None
def publish(benchmark, token=None, filename=None):
if token is None:
token = get_global_setting('github_token')
if token is None:
raise RuntimeError(
"Could not find the token value to connect to GitHub.\n\n"
"Please go to https://github.com/settings/tokens to generate a "
"personal token $TOKEN.\nThen, either provide it with option `-t` "
"or put it in a config file ./benchopt.ini\nunder section "
"[benchopt] as `github_token = $TOKEN`.")
# Get the result file
benchmark = Benchmark(benchmark)
result_filename = benchmark.get_result_file(filename)
# Publish the result.
publish_result_file(benchmark.name, result_filename, token)
def check_install(benchmark, module_filename, base_class_name):
# benchmark
benchmark = Benchmark(benchmark)
# Get class to check
klass = _load_class_from_module(module_filename, base_class_name,
benchmark.benchmark_dir)
klass.is_installed(raise_on_not_installed=True)
def test(benchmark, env_name, pytest_args):
benchmark = Benchmark(benchmark)
from benchopt.tests import __file__ as _bench_test_module
_bench_test_module = Path(_bench_test_module).parent
pytest_args = ' '.join(("-p benchopt.tests.fixtures",
f"--rootdir {_bench_test_module}", *pytest_args))
if len(pytest_args) == 0:
pytest_args = '-vl'
env_option = ''
if env_name is not None:
create_conda_env(env_name, with_pytest=True)
if _run_shell_in_conda_env("pytest --version", env_name=env_name) != 0:
raise ModuleNotFoundError(
f"pytest is not installed in conda env {env_name}.\n"
f"Please run `conda install -n {env_name} pytest` to test the "
"benchmark in this environment.")
objective = benchmark.get_benchmark_objective()
if not objective.is_installed():
objective.install(env_name=env_name)
env_option = f'--test-env {env_name}'
_bench_test_file = _bench_test_module / "test_benchmarks.py"
cmd = (
f'pytest {pytest_args} {_bench_test_file} '
f'--benchmark {benchmark.benchmark_dir} {env_option} '
# Make sure to not modify sys.path to add test file from current env
# in sub conda env as there might be different python versions.
'--import-mode importlib')
raise SystemExit(
_run_shell_in_conda_env(cmd, env_name=env_name, capture_stdout=False)
!= 0)
def archive(benchmark, with_outputs):
benchmark = Benchmark(benchmark)
bench_dir = benchmark.benchmark_dir
to_archive = ARCHIVE_ELEMENTS
if with_outputs:
to_archive = to_archive + ['outputs/*']
archive_name = f"{benchmark.name}.tar.gz"
print(f"Creating {archive_name}...", end='', flush=True)
with tarfile.open(archive_name, "w:gz") as tar:
for elem_pattern in to_archive:
for sub_elem in bench_dir.glob(elem_pattern):
tar.add(sub_elem,
sub_elem.relative_to(bench_dir.parent),
filter=clean_archive)
print(f"done\nResults are in {archive_name}")
import re
from pathlib import Path
from benchopt.benchmark import Benchmark
from benchopt.utils.stream_redirection import SuppressStd
# Default benchmark
TEST_BENCHMARK_DIR = Path(__file__).parent / 'test_benchmarks'
DUMMY_BENCHMARK_PATH = TEST_BENCHMARK_DIR / 'dummy_benchmark'
# Pattern to select specific datasets or solvers.
SELECT_ONE_SIMULATED = r'simulated*500*rho=0\]'
SELECT_ONE_PGD = r'python-pgd*step_size=1\]'
try:
DUMMY_BENCHMARK = Benchmark(DUMMY_BENCHMARK_PATH)
TEST_OBJECTIVE = DUMMY_BENCHMARK.get_benchmark_objective()
TEST_SOLVER = [s for s in DUMMY_BENCHMARK.list_benchmark_solvers()
if s.name == "Test-Solver"][0]
TEST_DATASET = [d for d in DUMMY_BENCHMARK.list_benchmark_datasets()
if d.name == "Test-Dataset"][0]
except Exception:
DUMMY_BENCHMARK = None
TEST_OBJECTIVE = None
TEST_SOLVER = None
TEST_DATASET = None
class CaptureRunOutput(object):
"""Context to capture run cmd output and files.
"""
def run(benchmark, solver_names, forced_solvers, dataset_names,
objective_filters, max_runs, n_repetitions, timeout,
plot=True, html=True, pdb=False, do_profile=False,
env_name='False', old_objective_filters=None):
if len(old_objective_filters):
warnings.warn(
'Using the -p option is deprecated, use -o instead',
FutureWarning,
)
objective_filters = old_objective_filters
from benchopt.runner import run_benchmark
if do_profile:
from benchopt.utils.profiling import use_profile
use_profile() # needs to be called before validate_solver_patterns
# Check that the dataset/solver patterns match actual dataset
benchmark = Benchmark(benchmark)
benchmark.validate_dataset_patterns(dataset_names)
benchmark.validate_solver_patterns(solver_names+forced_solvers)
benchmark.validate_objective_filters(objective_filters)
# If env_name is False, the flag `--local` has been used (default) so
# run in the current environement.
if env_name == 'False':
run_benchmark(
benchmark, solver_names, forced_solvers,
dataset_names=dataset_names,
objective_filters=objective_filters,
max_runs=max_runs, n_repetitions=n_repetitions,
timeout=timeout, plot_result=plot, html=html, pdb=pdb
)
print_stats() # print profiling stats (does nothing if not profiling)
return
_, all_conda_envs = list_conda_envs()
# If env_name is True, the flag `--env` has been used. Create a conda env
# specific to the benchmark (if not existing).
# Else, use the <env_name> value.
if env_name == 'True':
env_name = f"benchopt_{benchmark.name}"
install_cmd = f"`benchopt install -e {benchmark.benchmark_dir}`"
else:
# check provided <env_name>
# (to avoid empty name like `--env-name ""`)
if len(env_name) == 0:
raise RuntimeError("Empty environment name.")
install_cmd = (
f"`benchopt install --env-name {env_name} "
f"{benchmark.benchmark_dir}`"
)
# check if the environment exists
if env_name not in all_conda_envs:
raise RuntimeError(
f"The default env '{env_name}' for benchmark {benchmark.name} "
f"does not exist. Make sure to run {install_cmd} to create the "
"benchmark and install the dependencies."
)
# check if environment was set up with benchopt
if get_benchopt_version_in_env(env_name) is None:
raise RuntimeError(
f"benchopt is not installed in env '{env_name}', "
"see the command `benchopt install` to setup the environment."
)
# run the command in the conda env
solvers_option = ' '.join(['-s ' + s for s in solver_names])
forced_solvers_option = ' '.join([f"-f '{s}'" for s in forced_solvers])
datasets_option = ' '.join([f"-d '{d}'" for d in dataset_names])
objective_option = ' '.join([f"-p '{p}'" for p in objective_filters])
cmd = (
rf"benchopt run --local {benchmark.benchmark_dir} "
rf"--n-repetitions {n_repetitions} "
rf"--max-runs {max_runs} --timeout {timeout} "
rf"{solvers_option} {forced_solvers_option} "
rf"{datasets_option} {objective_option} "
rf"{'--plot' if plot else '--no-plot'} "
rf"{'--html' if html else '--no-html'} "
rf"{'--pdb' if pdb else ''} "
.replace('\\', '\\\\')
)
raise SystemExit(_run_shell_in_conda_env(
cmd, env_name=env_name, capture_stdout=False
) != 0)
===========================
"""
import os
from pathlib import Path
import matplotlib.pyplot as plt
from benchopt import run_benchmark
from benchopt.benchmark import Benchmark
from benchopt.plotting import plot_benchmark, PLOT_KINDS
BENCHMARK_PATH = Path(os.getcwd()).parent / 'benchmarks' / 'logreg_l2'
try:
save_file = run_benchmark(
Benchmark(BENCHMARK_PATH), ['sklearn', 'lightning'],
dataset_names=['Simulated*n_samples=200,n_features=500*'],
max_runs=100,
timeout=20,
n_repetitions=3,
plot_result=False,
show_progress=False)
except RuntimeError:
raise RuntimeError(
"This example can only work when Logreg-l2 benchmark is cloned in a "
"`benchmarks` folder. Please run:\n"
"$ git clone https://github.com/benchopt/benchmark_logreg_l2 "
f"{BENCHMARK_PATH.resolve()}")
kinds = list(PLOT_KINDS.keys())
figs = plot_benchmark(save_file,
import matplotlib.pyplot as plt
from benchopt import run_benchmark
from benchopt.benchmark import Benchmark
from benchopt.plotting import plot_benchmark, PLOT_KINDS
BENCHMARK_PATH = Path().resolve().parent / 'benchmarks' / 'benchmark_lasso'
if not BENCHMARK_PATH.exists():
raise RuntimeError(
"This example can only work when Lasso benchmark is cloned in the "
"example folder. Please run:\n"
"$ git clone https://github.com/benchopt/benchmark_lasso "
f"{BENCHMARK_PATH.resolve()}")
save_file = run_benchmark(
Benchmark(BENCHMARK_PATH), ['Python-PGD[use_acceleration=False]', 'R-PGD'],
dataset_names=["Simulated[n_features=5000,n_samples=100,rho=0]"],
objective_filters=['*[fit_intercept=False,reg=0.5]'],
max_runs=100,
timeout=100,
n_repetitions=5,
plot_result=False,
show_progress=False)
kinds = list(PLOT_KINDS.keys())
figs = plot_benchmark(save_file,
benchmark=Benchmark(BENCHMARK_PATH),
kinds=kinds,
html=False)
plt.show()
def get_datasets(ctx, args, incomplete):
skip_import()
benchmark = Benchmark(args[1])
datasets = benchmark.list_benchmark_dataset_names()
datasets = [d.lower() for d in datasets]
return [d for d in datasets if incomplete.lower() in d]
def get_solvers(ctx, args, incomplete):
skip_import()
benchmark = Benchmark(args[1])
solvers = benchmark.list_benchmark_solver_names()
solvers = [s.lower() for s in solvers]
return [s for s in solvers if incomplete.lower() in s]
===========================
"""
import os
from pathlib import Path
import matplotlib.pyplot as plt
from benchopt import run_benchmark
from benchopt.benchmark import Benchmark
from benchopt.plotting import plot_benchmark, PLOT_KINDS
BENCHMARK_PATH = Path(os.getcwd()).parent / 'benchmarks' / 'logreg_l2'
try:
df = run_benchmark(
Benchmark(BENCHMARK_PATH), ['sklearn', 'lightning'],
dataset_names=['Simulated*n_samples=200,n_features=500*'],
max_runs=100,
timeout=20,
n_repetitions=3,
plot_result=False,
show_progress=False)
except RuntimeError:
raise RuntimeError(
"This example can only work when Lasso benchmark is cloned in the "
"example folder. Please run:\n"
"$ git clone https://github.com/benchopt/benchmark_logreg_l2 "
f"{BENCHMARK_PATH.resolve()}")
kinds = list(PLOT_KINDS.keys())
figs = plot_benchmark(df, benchmark=Benchmark(BENCHMARK_PATH), kinds=kinds)
from pathlib import Path
import matplotlib.pyplot as plt
from benchopt import run_benchmark
from benchopt.benchmark import Benchmark
from benchopt.plotting import plot_benchmark, PLOT_KINDS
BENCHMARK_PATH = (
Path().resolve().parent / 'benchmarks' / 'benchmark_logreg_l2'
)
try:
save_file = run_benchmark(
Benchmark(BENCHMARK_PATH), ['sklearn[liblinear]', 'sklearn[newton-cg]',
'lightning'],
dataset_names=['Simulated*n_samples=200,n_features=500*'],
objective_filters=[
'L2 Logistic Regression[fit_intercept=False,lmbd=1.0]'],
max_runs=100, timeout=20, n_repetitions=15,
plot_result=False, show_progress=True
)
except RuntimeError:
raise RuntimeError(
"This example can only work when Logreg-l2 benchmark is cloned in a "
"`benchmarks` folder. Please run:\n"
"$ git clone https://github.com/benchopt/benchmark_logreg_l2 "
f"{BENCHMARK_PATH.resolve()}"
)
===========================
"""
from pathlib import Path
import matplotlib.pyplot as plt
from benchopt import run_benchmark
from benchopt.benchmark import Benchmark
from benchopt.plotting import plot_benchmark, PLOT_KINDS
BENCHMARK_PATH = (Path().resolve().parent / 'benchmarks' /
'benchmark_logreg_l2')
try:
save_file = run_benchmark(
Benchmark(BENCHMARK_PATH),
['sklearn[liblinear]', 'sklearn[newton-cg]', 'lightning'],
dataset_names=['Simulated*[n_features=500,n_samples=200]'],
objective_filters=['L2 Logistic Regression[lmbd=1.0]'],
max_runs=100,
timeout=20,
n_repetitions=15,
plot_result=False,
show_progress=True)
except RuntimeError:
raise RuntimeError(
"This example can only work when Logreg-l2 benchmark is cloned in a "
"`benchmarks` folder. Please run:\n"
"$ git clone https://github.com/benchopt/benchmark_logreg_l2 "
f"{BENCHMARK_PATH.resolve()}")
def run(config_file=None, **kwargs):
if config_file is not None:
with open(config_file, "r") as f:
config = yaml.safe_load(f)
else:
config = {}
# XXX - Remove old and deprecated objective filters in version 1.3
(benchmark, solver_names, forced_solvers, dataset_names, objective_filters,
max_runs, n_repetitions, timeout, n_jobs, slurm, plot, html, pdb,
do_profile, env_name, output, deprecated_objective_filters,
old_objective_filters) = _get_run_args(kwargs, config)
if len(old_objective_filters):
warnings.warn(
'Using the -p option is deprecated, use -o instead',
FutureWarning,
)
objective_filters = old_objective_filters
if len(deprecated_objective_filters):
warnings.warn(
'Using the --objective-filters option is deprecated, '
'use --objective instead', FutureWarning)
objective_filters = deprecated_objective_filters
# Create the Benchmark object
benchmark = Benchmark(benchmark)
# If env_name is False, the flag `--local` has been used (default) so
# run in the current environment.
if env_name == 'False':
print("Benchopt is running")
if slurm is not None:
print("Running on SLURM")
set_slurm_launch()
from benchopt.runner import run_benchmark
if do_profile:
from benchopt.utils.profiling import use_profile
use_profile() # needs to be called before validate_solver_patterns
# Check that the dataset/solver patterns match actual dataset
benchmark.validate_dataset_patterns(dataset_names)
benchmark.validate_objective_filters(objective_filters)
# pyyaml returns tuples: make sure everything is a list
benchmark.validate_solver_patterns(
list(solver_names) + list(forced_solvers))
run_benchmark(benchmark,
solver_names,
forced_solvers,
dataset_names=dataset_names,
objective_filters=objective_filters,
max_runs=max_runs,
n_repetitions=n_repetitions,
timeout=timeout,
n_jobs=n_jobs,
slurm=slurm,
plot_result=plot,
html=html,
pdb=pdb,
output=output)
print_stats() # print profiling stats (does nothing if not profiling)
return
default_conda_env, all_conda_envs = list_conda_envs()
# If env_name is True, the flag `--env` has been used. Create a conda env
# specific to the benchmark (if not existing).
# Else, use the <env_name> value.
# check if any current conda environment
if default_conda_env is None:
raise RuntimeError(
"No conda environment is activated. "
"You should be in a conda environment to use "
"'benchopt run' with options '-e/--env' or '--env-name'.")
if env_name == 'True':
env_name = f"benchopt_{benchmark.name}"
install_cmd = f"`benchopt install -e {benchmark.benchmark_dir}`"
else:
# check provided <env_name>
# (to avoid empty name like `--env-name ""`)
if len(env_name) == 0:
raise RuntimeError("Empty environment name.")
install_cmd = (f"`benchopt install --env-name {env_name} "
f"{benchmark.benchmark_dir}`")
# check if the environment exists
if env_name not in all_conda_envs:
raise RuntimeError(
f"The default env '{env_name}' for benchmark {benchmark.name} "
f"does not exist. Make sure to run {install_cmd} to create the "
"benchmark and install the dependencies.")
print(f"Launching benchopt in env {env_name}")
# check if environment was set up with benchopt
benchopt_version, is_editable = get_benchopt_version_in_env(env_name)
if benchopt_version is None:
raise RuntimeError(
f"benchopt is not installed in env '{env_name}', "
"see the command `benchopt install` to setup the environment.")
# check against running version
from benchopt import __version__ as benchopt_version_running
_, is_editable_running = get_benchopt_requirement()
if (benchopt_version_running != benchopt_version
and not (is_editable_running and is_editable)):
warnings.warn(
f"Benchopt running version ({benchopt_version_running}) "
f"and version in env {env_name} ({benchopt_version}) differ")
# run the command in the conda env
solvers_option = ' '.join([f"-s '{s}'" for s in solver_names])
forced_solvers_option = ' '.join([f"-f '{s}'" for s in forced_solvers])
datasets_option = ' '.join([f"-d '{d}'" for d in dataset_names])
objective_option = ' '.join([f"-o '{o}'" for o in objective_filters])
cmd = (rf"benchopt run --local {benchmark.benchmark_dir} "
rf"--n-repetitions {n_repetitions} "
rf"--max-runs {max_runs} --timeout {timeout} "
rf"--n-jobs {n_jobs} {'--slurm' if slurm else ''} "
rf"{solvers_option} {forced_solvers_option} "
rf"{datasets_option} {objective_option} "
rf"{'--plot' if plot else '--no-plot'} "
rf"{'--html' if html else '--no-html'} "
rf"{'--pdb' if pdb else ''} "
rf"--output {output}".replace('\\', '\\\\'))
raise SystemExit(
_run_shell_in_conda_env(cmd, env_name=env_name, capture_stdout=False)
!= 0)
"""
import os
from pathlib import Path
import matplotlib.pyplot as plt
from benchopt import run_benchmark
from benchopt.benchmark import Benchmark
from benchopt.tests import SELECT_ONE_SIMULATED
from benchopt.plotting import plot_benchmark, PLOT_KINDS
BENCHMARK_PATH = Path(os.getcwd()).parent / 'benchmarks' / 'lasso'
try:
df = run_benchmark(
Benchmark(BENCHMARK_PATH),
['Python-PGD[^-]*use_acceleration=False', 'R-PGD', 'Julia-PGD'],
dataset_names=[SELECT_ONE_SIMULATED],
objective_filters=['reg=0.5'],
max_runs=100,
timeout=100,
n_repetitions=5,
plot_result=False,
show_progress=False)
except RuntimeError:
raise RuntimeError(
"This example can only work when Lasso benchmark is cloned in the "
"example folder. Please run:\n"
"$ git clone https://github.com/benchopt/benchmark_lasso "
f"{BENCHMARK_PATH.resolve()}")
def run(benchmark,
solver_names,
forced_solvers,
dataset_names,
objective_filters,
max_runs,
n_repetitions,
timeout,
recreate=False,
plot=True,
pdb=False,
env_name='False'):
# Check that the dataset/solver patterns match actual dataset
benchmark = Benchmark(benchmark)
benchmark.validate_dataset_patterns(dataset_names)
benchmark.validate_solver_patterns(solver_names + forced_solvers)
# If env_name is False, the flag `--local` has been used (default) so
# run in the current environement.
if env_name == 'False':
run_benchmark(benchmark,
solver_names,
forced_solvers,
dataset_names=dataset_names,
objective_filters=objective_filters,
max_runs=max_runs,
n_repetitions=n_repetitions,
timeout=timeout,
plot_result=plot,
pdb=pdb)
return
# If env_name is True, the flag `--env` has been used. Create a conda env
# specific to the benchmark. Else, use the <env_name> value.
if env_name == 'True':
env_name = f"benchopt_{benchmark.name}"
create_conda_env(env_name, recreate=recreate)
# installed required datasets
benchmark.install_required_datasets(dataset_names, env_name=env_name)
# Get the solvers and install them
benchmark.install_required_solvers(solver_names,
forced_solvers=forced_solvers,
env_name=env_name)
# run the command in the conda env
solvers_option = ' '.join(['-s ' + s for s in solver_names])
forced_solvers_option = ' '.join(['-f ' + s for s in forced_solvers])
datasets_option = ' '.join(['-d ' + d for d in dataset_names])
objective_option = ' '.join(['-p ' + p for p in objective_filters])
cmd = (rf"benchopt run --local {benchmark.benchmark_dir} "
rf"--n-repetitions {n_repetitions} "
rf"--max-runs {max_runs} --timeout {timeout} "
rf"{solvers_option} {forced_solvers_option} "
rf"{datasets_option} {objective_option} "
rf"{'--plot' if plot else '--no-plot'} "
rf"{'--pdb' if pdb else ''} ".replace('\\', '\\\\'))
raise SystemExit(
_run_shell_in_conda_env(cmd, env_name=env_name, capture_stdout=False)
!= 0)
from pathlib import Path
from benchopt.benchmark import Benchmark
from benchopt.utils.stream_redirection import SuppressStd
# Default benchmark
TEST_BENCHMARK_DIR = Path(__file__).parent / 'test_benchmarks'
DUMMY_BENCHMARK_PATH = TEST_BENCHMARK_DIR / 'dummy_benchmark'
# Pattern to select specific datasets or solvers.
SELECT_ONE_SIMULATED = r'simulated*500*rho=0]'
SELECT_ONE_PGD = r'python-pgd*step_size=1]'
SELECT_ONE_OBJECTIVE = r'dummy*reg=0.1]'
try:
DUMMY_BENCHMARK = Benchmark(DUMMY_BENCHMARK_PATH)
TEST_OBJECTIVE = DUMMY_BENCHMARK.get_benchmark_objective()
TEST_SOLVER = [
s for s in DUMMY_BENCHMARK.get_solvers() if s.name == "Test-Solver"
][0]
TEST_DATASET = [
d for d in DUMMY_BENCHMARK.get_datasets() if d.name == "Test-Dataset"
][0]
except Exception:
DUMMY_BENCHMARK = None
TEST_OBJECTIVE = None
TEST_SOLVER = None
TEST_DATASET = None
class CaptureRunOutput(object):
from pathlib import Path
from benchopt.benchmark import Benchmark
# Default benchmark
TEST_BENCHMARK_DIR = Path(__file__).parent / 'test_benchmarks'
DUMMY_BENCHMARK_PATH = TEST_BENCHMARK_DIR / 'dummy_benchmark'
# Pattern to select specific datasets or solvers.
SELECT_ONE_SIMULATED = r'simulated*500*rho=0\]'
SELECT_ONE_PGD = r'python-pgd*step_size=1\]'
try:
DUMMY_BENCHMARK = Benchmark(DUMMY_BENCHMARK_PATH)
TEST_OBJECTIVE = DUMMY_BENCHMARK.get_benchmark_objective()
TEST_SOLVER = [
s for s in DUMMY_BENCHMARK.list_benchmark_solvers()
if s.name == "Test-Solver"
][0]
TEST_DATASET = [
d for d in DUMMY_BENCHMARK.list_benchmark_datasets()
if d.name == "Test-Dataset"
][0]
except Exception:
DUMMY_BENCHMARK = None
TEST_OBJECTIVE = None
TEST_SOLVER = None
TEST_DATASET = None
import matplotlib.pyplot as plt
from benchopt import run_benchmark
from benchopt.benchmark import Benchmark
from benchopt.tests import SELECT_ONE_SIMULATED
from benchopt.plotting import plot_benchmark, PLOT_KINDS
BENCHMARK_PATH = Path().resolve().parent / 'benchmarks' / 'benchmark_lasso'
if not BENCHMARK_PATH.exists():
raise RuntimeError(
"This example can only work when Lasso benchmark is cloned in the "
"example folder. Please run:\n"
"$ git clone https://github.com/benchopt/benchmark_lasso "
f"{BENCHMARK_PATH.resolve()}")
save_file = run_benchmark(Benchmark(BENCHMARK_PATH),
['Python-PGD[use_acceleration=False]', 'R-PGD'],
dataset_names=[SELECT_ONE_SIMULATED],
objective_filters=['*fit_intercept=False,reg=0.5'],
max_runs=100,
timeout=100,
n_repetitions=5,
plot_result=False,
show_progress=False)
kinds = list(PLOT_KINDS.keys())
figs = plot_benchmark(save_file,
benchmark=Benchmark(BENCHMARK_PATH),
kinds=kinds,
html=False)
plt.show()