def run(benchmark,
solver_names,
forced_solvers,
dataset_names,
objective_filters,
max_runs,
n_repetitions,
timeout,
recreate=False,
local=True,
no_plot=False):
"""Run a benchmark in a separate conda env where the deps will be installed
"""
# Check that the dataset/solver patterns match actual dataset
validate_dataset_patterns(benchmark, dataset_names)
validate_solver_patterns(benchmark, solver_names + forced_solvers)
if local:
run_benchmark(benchmark,
solver_names,
forced_solvers,
dataset_names=dataset_names,
objective_filters=objective_filters,
max_runs=max_runs,
n_repetitions=n_repetitions,
timeout=timeout,
plot_result=not no_plot)
return
benchmark_name = Path(benchmark).resolve().name
env_name = f"benchopt_{benchmark_name}"
create_conda_env(env_name, recreate=recreate)
# installed required datasets
install_required_datasets(benchmark, dataset_names, env_name=env_name)
# Get the solvers and install them
install_required_solvers(benchmark,
solver_names,
forced_solvers=forced_solvers,
env_name=env_name)
# run the command in the conda env
solvers_option = ' '.join(['-s ' + s for s in solver_names])
forced_solvers_option = ' '.join(['-f ' + s for s in forced_solvers])
datasets_option = ' '.join(['-d ' + d for d in dataset_names])
objective_option = ' '.join(['-p ' + p for p in objective_filters])
cmd = (f"benchopt run {benchmark} --local --n-repetitions {n_repetitions} "
f"--max-runs {max_runs} --timeout {timeout} "
f"{solvers_option} {forced_solvers_option} "
f"{datasets_option} {objective_option} "
f"{'--no-plot' if no_plot else ''} ")
raise SystemExit(
_run_shell_in_conda_env(cmd, env_name=env_name, capture_stdout=False)
!= 0)
def run(benchmarks, solver_names, forced_solvers, max_samples, repetition):
"""Run benchmark."""
all_benchmarks = get_all_benchmarks()
if benchmarks == 'all':
benchmarks = all_benchmarks
check_benchmarks(benchmarks, all_benchmarks)
for benchmark in benchmarks:
run_benchmark(benchmark,
solver_names,
forced_solvers,
max_samples=max_samples,
n_rep=repetition)
def run(benchmark, solver_names, forced_solvers, dataset_names, max_samples,
recreate, local, repetition):
"""Run a benchmark in a separate venv where the solvers will be installed
"""
if local:
run_benchmark(benchmark,
solver_names,
forced_solvers,
dataset_names,
max_samples=max_samples,
n_rep=repetition)
return
# Create the virtual env
create_venv(benchmark, recreate=recreate)
# installed required datasets
install_required_datasets(benchmark, dataset_names, env_name=benchmark)
# Get the solvers and install them
solvers = list_benchmark_solvers(benchmark)
exclude = get_benchmark_setting(benchmark, 'exclude_solvers')
solvers = filter_solvers(solvers,
solver_names=solver_names,
forced_solvers=forced_solvers,
exclude=exclude)
install_solvers(solvers=solvers,
forced_solvers=forced_solvers,
env_name=benchmark)
# run the command in the virtual env
solvers_option = ' '.join(['-s ' + s for s in solver_names])
forced_solvers_option = ' '.join(['-f ' + s for s in forced_solvers])
datasets_option = ' '.join(['-d ' + d for d in dataset_names])
cmd = (f"benchopt run -l --max-samples {max_samples} -n {repetition} "
f"{solvers_option} {forced_solvers_option} {datasets_option} "
f"{benchmark}")
raise SystemExit(
_run_bash_in_env(cmd, env_name=benchmark, capture_stdout=False))
def run(benchmark,
solver_names,
forced_solvers,
dataset_names,
objective_filters,
max_runs,
n_repetitions,
timeout,
recreate=False,
no_plot=False,
pdb=False,
env_name=False):
"""Run a benchmark in a separate conda env where the deps will be installed
"""
# Check that the dataset/solver patterns match actual dataset
validate_benchmark(benchmark)
validate_dataset_patterns(benchmark, dataset_names)
validate_solver_patterns(benchmark, solver_names + forced_solvers)
# If env_name is False, the flag `--local` has been used (default) so
# run in the current environement.
if env_name is False:
run_benchmark(benchmark,
solver_names,
forced_solvers,
dataset_names=dataset_names,
objective_filters=objective_filters,
max_runs=max_runs,
n_repetitions=n_repetitions,
timeout=timeout,
plot_result=not no_plot,
pdb=pdb)
return
# If env_name is True, the flag `--env` has been used. Create a conda env
# specific to the benchmark. Else, use the <env_name> value.
if env_name is True:
benchmark_name = Path(benchmark).resolve().name
env_name = f"benchopt_{benchmark_name}"
create_conda_env(env_name, recreate=recreate)
# installed required datasets
install_required_datasets(benchmark, dataset_names, env_name=env_name)
# Get the solvers and install them
install_required_solvers(benchmark,
solver_names,
forced_solvers=forced_solvers,
env_name=env_name)
# run the command in the conda env
solvers_option = ' '.join(['-s ' + s for s in solver_names])
forced_solvers_option = ' '.join(['-f ' + s for s in forced_solvers])
datasets_option = ' '.join(['-d ' + d for d in dataset_names])
objective_option = ' '.join(['-p ' + p for p in objective_filters])
cmd = (
rf"benchopt run {benchmark} --local --n-repetitions {n_repetitions} "
rf"--max-runs {max_runs} --timeout {timeout} "
rf"{solvers_option} {forced_solvers_option} "
rf"{datasets_option} {objective_option} "
rf"{'--no-plot' if no_plot else ''} "
rf"{'--pdb' if pdb else ''} ".replace('\\', '\\\\'))
raise SystemExit(
_run_shell_in_conda_env(cmd, env_name=env_name, capture_stdout=False)
!= 0)
"""
import os
from pathlib import Path
import matplotlib.pyplot as plt
from benchopt import run_benchmark
from benchopt.viz import plot_benchmark, PLOT_KINDS
BENCHMARK_PATH = Path(os.getcwd()).parent / 'benchmarks' / 'lasso'
try:
df = run_benchmark(
str(BENCHMARK_PATH),
['Python-PGD*use_acceleration=False', 'R-PGD', 'Julia-PGD'],
dataset_names=['Simulated*n_samples=100,n_features=500*'],
objective_filters=['reg=0.5'],
max_runs=100,
timeout=100,
n_repetitions=5,
plot_result=False,
show_progress=False)
except RuntimeError:
raise RuntimeError(
"This example can only work when Lasso benchmark is cloned in the "
"example folder. Please run:\n"
"$ git clone https://github.com/benchopt/benchmark_lasso "
f"{BENCHMARK_PATH.resolve()}")
kinds = list(PLOT_KINDS.keys())
figs = plot_benchmark(df, benchmark=str(BENCHMARK_PATH), kinds=kinds)
plt.show()
"""
import os
from pathlib import Path
import matplotlib.pyplot as plt
from benchopt import run_benchmark
from benchopt.viz import plot_benchmark, PLOT_KINDS
BENCHMARK_PATH = Path(os.getcwd()).parent / 'benchmarks' / 'logreg_l2'
try:
df = run_benchmark(
str(BENCHMARK_PATH), ['sklearn', 'lightning'],
dataset_names=['Simulated*n_samples=200,n_features=500*'],
max_runs=100,
timeout=20,
n_repetitions=3,
plot_result=False,
show_progress=False)
except RuntimeError:
raise RuntimeError(
"This example can only work when Lasso benchmark is cloned in the "
"example folder. Please run:\n"
"$ git clone https://github.com/benchopt/benchmark_logreg_l2 "
f"{BENCHMARK_PATH.resolve()}")
kinds = list(PLOT_KINDS.keys())
figs = plot_benchmark(df, benchmark=str(BENCHMARK_PATH), kinds=kinds)
plt.show()
import os
from pathlib import Path
import matplotlib.pyplot as plt
from benchopt import run_benchmark
from benchopt.benchmark import Benchmark
from benchopt.tests import SELECT_ONE_SIMULATED
from benchopt.plotting import plot_benchmark, PLOT_KINDS
BENCHMARK_PATH = Path(os.getcwd()).parent / 'benchmarks' / 'lasso'
try:
df = run_benchmark(
Benchmark(BENCHMARK_PATH),
['Python-PGD[^-]*use_acceleration=False', 'R-PGD', 'Julia-PGD'],
dataset_names=[SELECT_ONE_SIMULATED],
objective_filters=['reg=0.5'],
max_runs=100,
timeout=100,
n_repetitions=5,
plot_result=False,
show_progress=False)
except RuntimeError:
raise RuntimeError(
"This example can only work when Lasso benchmark is cloned in the "
"example folder. Please run:\n"
"$ git clone https://github.com/benchopt/benchmark_lasso "
f"{BENCHMARK_PATH.resolve()}")
kinds = list(PLOT_KINDS.keys())
figs = plot_benchmark(df, benchmark=Benchmark(BENCHMARK_PATH), kinds=kinds)
plt.show()
from pathlib import Path
import matplotlib.pyplot as plt
from benchopt import run_benchmark
from benchopt.benchmark import Benchmark
from benchopt.plotting import plot_benchmark, PLOT_KINDS
BENCHMARK_PATH = (Path().resolve().parent / 'benchmarks' /
'benchmark_logreg_l2')
try:
save_file = run_benchmark(
Benchmark(BENCHMARK_PATH),
['sklearn[liblinear]', 'sklearn[newton-cg]', 'lightning'],
dataset_names=['Simulated*[n_features=500,n_samples=200]'],
objective_filters=['L2 Logistic Regression[lmbd=1.0]'],
max_runs=100,
timeout=20,
n_repetitions=15,
plot_result=False,
show_progress=True)
except RuntimeError:
raise RuntimeError(
"This example can only work when Logreg-l2 benchmark is cloned in a "
"`benchmarks` folder. Please run:\n"
"$ git clone https://github.com/benchopt/benchmark_logreg_l2 "
f"{BENCHMARK_PATH.resolve()}")
kinds = list(PLOT_KINDS.keys())
figs = plot_benchmark(save_file,
benchmark=Benchmark(BENCHMARK_PATH),
import matplotlib.pyplot as plt
from benchopt import run_benchmark
from benchopt.benchmark import Benchmark
from benchopt.plotting import plot_benchmark, PLOT_KINDS
BENCHMARK_PATH = Path().resolve().parent / 'benchmarks' / 'benchmark_lasso'
if not BENCHMARK_PATH.exists():
raise RuntimeError(
"This example can only work when Lasso benchmark is cloned in the "
"example folder. Please run:\n"
"$ git clone https://github.com/benchopt/benchmark_lasso "
f"{BENCHMARK_PATH.resolve()}")
save_file = run_benchmark(
Benchmark(BENCHMARK_PATH), ['Python-PGD[use_acceleration=False]', 'R-PGD'],
dataset_names=["Simulated[n_features=5000,n_samples=100,rho=0]"],
objective_filters=['*[fit_intercept=False,reg=0.5]'],
max_runs=100,
timeout=100,
n_repetitions=5,
plot_result=False,
show_progress=False)
kinds = list(PLOT_KINDS.keys())
figs = plot_benchmark(save_file,
benchmark=Benchmark(BENCHMARK_PATH),
kinds=kinds,
html=False)
plt.show()
from glob import glob
from benchopt import run_benchmark
benchs = glob("benchmarks/*/bench_*.py")
for b in benchs:
b = b.replace('/', '.').replace('.py', '')
run_benchmark(b)
