def plot(benchmark,
filename=None,
kinds=('suboptimality_curve', ),
display=True,
html=True,
plotly=False,
all_files=False):
if all_files:
assert filename is None, (
"Cannot use `--all` and `--filename` simultaneously.")
assert html, '`--all` can only be used for HTML plot generation.'
filename = 'all'
if html and len(kinds) > 0:
warnings.warn("Cannot specify '--kind' for HTML plot, this options "
"will be ignored.")
# Get the result file
benchmark = Benchmark(benchmark)
result_filename = benchmark.get_result_file(filename)
# Plot the results.
from benchopt.plotting import plot_benchmark
plot_benchmark(result_filename,
benchmark,
kinds=kinds,
display=display,
plotly=plotly,
html=html)
def plot(benchmark,
filename=None,
kinds=('suboptimality_curve', ),
display=True,
plotly=False):
# Get the result file
benchmark = Benchmark(benchmark)
result_filename = benchmark.get_result_file(filename)
# Plot the results.
df = pd.read_csv(result_filename)
plot_benchmark(df, benchmark, kinds=kinds, display=display, plotly=plotly)
def plot(benchmark,
filename=None,
kinds=('suboptimality_curve', ),
display=True,
plotly=False):
# Get the result file
benchmark = Benchmark(benchmark)
result_filename = benchmark.get_result_file(filename)
# Load the results.
import pandas as pd
df = pd.read_csv(result_filename)
# Plot the results.
from benchopt.plotting import plot_benchmark
plot_benchmark(df, benchmark, kinds=kinds, display=display, plotly=plotly)
def run_benchmark(benchmark,
solver_names=None,
forced_solvers=None,
dataset_names=None,
objective_filters=None,
max_runs=10,
n_repetitions=1,
timeout=100,
plot_result=True,
show_progress=True,
pdb=False):
"""Run full benchmark.
Parameters
----------
benchmark : benchopt.Benchmark object
Object to represent the benchmark.
solver_names : list | None
List of solvers to include in the benchmark. If None
all solvers available are run.
forced_solvers : list | None
List of solvers to include in the benchmark and for
which one forces recomputation.
dataset_names : list | None
List of datasets to include. If None all available
datasets are used.
objective_filters : list | None
Filters to select specific objective parameters. If None,
all objective parameters are tested
max_runs : int
The maximum number of solver runs to perform to estimate
the convergence curve.
n_repetitions : int
The number of repetitions to run. Defaults to 1.
timeout : float
The maximum duration in seconds of the solver run.
plot_result : bool
If set to True (default), display the result plot and save them in
the benchmark directory.
show_progress : bool
If show_progress is set to True, display the progress of the benchmark.
pdb : bool
It pdb is set to True, open a debugger on error.
Returns
-------
df : instance of pandas.DataFrame
The benchmark results. If multiple metrics were computed, each
one is stored in a separate column. If the number of metrics computed
by the objective is not the same for all parameters, the missing data
is set to `NaN`.
"""
# Load the objective class for this benchmark and the datasets
objective_class = benchmark.get_benchmark_objective()
datasets = benchmark.list_benchmark_datasets()
# Load the solvers and filter them to get the one to run
solver_classes = benchmark.list_benchmark_solvers()
included_solvers = _check_name_lists(solver_names, forced_solvers)
run_statistics = []
for dataset_class in datasets:
for dataset_parameters in product_param(dataset_class.parameters):
dataset = dataset_class.get_instance(**dataset_parameters)
if not is_matched(str(dataset), dataset_names):
continue
print(f"{dataset}".ljust(LINE_LENGTH))
if not dataset.is_installed(
raise_on_not_installed=RAISE_INSTALL_ERROR):
print(
colorify(f"Dataset {dataset} is not installed.",
RED).ljust(LINE_LENGTH))
continue
dimension, data = dataset._get_data()
for obj_parameters in product_param(objective_class.parameters):
objective = objective_class.get_instance(**obj_parameters)
if not is_matched(str(objective), objective_filters):
continue
print(f"|--{objective}".ljust(LINE_LENGTH))
objective.set_dataset(dataset)
for solver_class in solver_classes:
for solver_parameters in product_param(
solver_class.parameters):
# Instantiate solver
solver = solver_class.get_instance(**solver_parameters)
if not is_matched(solver, included_solvers):
continue
# Get the solver's name
tag = colorify(f"|----{solver}:")
# check if the module caught a failed import
if not solver.is_installed(
raise_on_not_installed=RAISE_INSTALL_ERROR):
status = colorify("not installed", RED)
print(f"{tag} {status}".ljust(LINE_LENGTH))
continue
# Set objective an skip if necessary.
skip, reason = solver._set_objective(objective)
if skip:
print(f"{tag} {colorify('skip', YELLOW)}".ljust(
LINE_LENGTH))
if reason is not None:
print(f'Reason: {reason}')
continue
# Get meta
meta = dict(objective_name=str(objective),
data_name=str(dataset),
dimension=dimension)
force = (forced_solvers is not None
and len(forced_solvers) > 0
and is_matched(str(solver), forced_solvers))
run_statistics.extend(
run_one_solver(benchmark=benchmark,
objective=objective,
solver=solver,
meta=meta,
tag=tag,
max_runs=max_runs,
n_repetitions=n_repetitions,
timeout=timeout,
show_progress=show_progress,
force=force,
pdb=pdb))
import pandas as pd
df = pd.DataFrame(run_statistics)
if df.empty:
print(colorify('No output produced.', RED).ljust(LINE_LENGTH))
return
# raise SystemExit(1)
# Save output in CSV file in the benchmark folder
timestamp = datetime.now().strftime('%Y-%m-%d_%Hh%Mm%S')
output_dir = benchmark.get_output_folder()
save_file = output_dir / f'benchopt_run_{timestamp}.csv'
df.to_csv(save_file)
print(colorify(f'Saving result in: {save_file}', GREEN))
if plot_result:
from benchopt.plotting import plot_benchmark
plot_benchmark(df, benchmark)
return df
from pathlib import Path
import matplotlib.pyplot as plt
from benchopt import run_benchmark
from benchopt.benchmark import Benchmark
from benchopt.tests import SELECT_ONE_SIMULATED
from benchopt.plotting import plot_benchmark, PLOT_KINDS
BENCHMARK_PATH = Path(os.getcwd()).parent / 'benchmarks' / 'lasso'
try:
df = run_benchmark(
Benchmark(BENCHMARK_PATH),
['Python-PGD[^-]*use_acceleration=False', 'R-PGD', 'Julia-PGD'],
dataset_names=[SELECT_ONE_SIMULATED],
objective_filters=['reg=0.5'],
max_runs=100,
timeout=100,
n_repetitions=5,
plot_result=False,
show_progress=False)
except RuntimeError:
raise RuntimeError(
"This example can only work when Lasso benchmark is cloned in the "
"example folder. Please run:\n"
"$ git clone https://github.com/benchopt/benchmark_lasso "
f"{BENCHMARK_PATH.resolve()}")
kinds = list(PLOT_KINDS.keys())
figs = plot_benchmark(df, benchmark=Benchmark(BENCHMARK_PATH), kinds=kinds)
plt.show()
from benchopt.plotting import plot_benchmark, PLOT_KINDS
BENCHMARK_PATH = (Path().resolve().parent / 'benchmarks' /
'benchmark_logreg_l2')
try:
save_file = run_benchmark(
Benchmark(BENCHMARK_PATH),
['sklearn[liblinear]', 'sklearn[newton-cg]', 'lightning'],
dataset_names=['Simulated*[n_features=500,n_samples=200]'],
objective_filters=['L2 Logistic Regression[lmbd=1.0]'],
max_runs=100,
timeout=20,
n_repetitions=15,
plot_result=False,
show_progress=True)
except RuntimeError:
raise RuntimeError(
"This example can only work when Logreg-l2 benchmark is cloned in a "
"`benchmarks` folder. Please run:\n"
"$ git clone https://github.com/benchopt/benchmark_logreg_l2 "
f"{BENCHMARK_PATH.resolve()}")
kinds = list(PLOT_KINDS.keys())
figs = plot_benchmark(save_file,
benchmark=Benchmark(BENCHMARK_PATH),
kinds=kinds,
html=False)
plt.show()
def run_benchmark(benchmark, solver_names=None, forced_solvers=None,
dataset_names=None, objective_filters=None, max_runs=10,
n_repetitions=1, timeout=100, n_jobs=1, slurm=None,
plot_result=True, html=True, show_progress=True, pdb=False,
output="None"):
"""Run full benchmark.
Parameters
----------
benchmark : benchopt.Benchmark object
Object to represent the benchmark.
solver_names : list | None
List of solvers to include in the benchmark. If None
all solvers available are run.
forced_solvers : list | None
List of solvers to include in the benchmark and for
which one forces recomputation.
dataset_names : list | None
List of datasets to include. If None all available
datasets are used.
objective_filters : list | None
Filters to select specific objective parameters. If None,
all objective parameters are tested
max_runs : int
The maximum number of solver runs to perform to estimate
the convergence curve.
n_repetitions : int
The number of repetitions to run. Defaults to 1.
timeout : float
The maximum duration in seconds of the solver run.
n_jobs : int
Maximal number of workers to use to run the benchmark in parallel.
slurm : Path | None
If not None, launch the job on a slurm cluster using the file to get
the cluster config parameters.
plot_result : bool
If set to True (default), display the result plot and save them in
the benchmark directory.
html : bool
If set to True (default), display the result plot in HTML, otherwise
in matplotlib figures, default is True.
show_progress : bool
If show_progress is set to True, display the progress of the benchmark.
pdb : bool
It pdb is set to True, open a debugger on error.
output_name : str
Filename for the parquet output. If given, the results will
be stored at <BENCHMARK>/outputs/<filename>.parquet.
Returns
-------
df : instance of pandas.DataFrame
The benchmark results. If multiple metrics were computed, each
one is stored in a separate column. If the number of metrics computed
by the objective is not the same for all parameters, the missing data
is set to `NaN`.
"""
output_name = output
# List all datasets, objective and solvers to run based on the filters
# provided. Merge the solver_names and forced to run all necessary solvers.
solver_names = _check_name_lists(solver_names, forced_solvers)
output = TerminalOutput(n_repetitions, show_progress)
output.set(verbose=True)
all_runs = benchmark.get_all_runs(
solver_names, forced_solvers, dataset_names, objective_filters,
output=output
)
common_kwargs = dict(
benchmark=benchmark, n_repetitions=n_repetitions, max_runs=max_runs,
timeout=timeout, pdb=pdb
)
if slurm is not None:
from .utils.slurm_executor import run_on_slurm
results = run_on_slurm(slurm, run_one_solver, common_kwargs, all_runs)
else:
results = Parallel(n_jobs=n_jobs)(
delayed(run_one_solver)(**common_kwargs, **kwargs)
for kwargs in all_runs
)
run_statistics = []
for curve in results:
run_statistics.extend(curve)
import pandas as pd
df = pd.DataFrame(run_statistics)
if df.empty:
output.savefile_status()
raise SystemExit(1)
# Save output in parquet file in the benchmark folder
timestamp = datetime.now().strftime('%Y-%m-%d_%Hh%Mm%S')
output_dir = benchmark.get_output_folder()
if output_name == "None":
save_file = output_dir / f'benchopt_run_{timestamp}.parquet'
else:
save_file = output_dir / f"{output_name}.parquet"
save_file = uniquify_results(save_file)
df.to_parquet(save_file)
output.savefile_status(save_file=save_file)
if plot_result:
from benchopt.plotting import plot_benchmark
plot_benchmark(save_file, benchmark, html=html)
return save_file