def plot(benchmark, filename=None, kinds=('suboptimality_curve', ), display=True, html=True, plotly=False, all_files=False): if all_files: assert filename is None, ( "Cannot use `--all` and `--filename` simultaneously.") assert html, '`--all` can only be used for HTML plot generation.' filename = 'all' if html and len(kinds) > 0: warnings.warn("Cannot specify '--kind' for HTML plot, this options " "will be ignored.") # Get the result file benchmark = Benchmark(benchmark) result_filename = benchmark.get_result_file(filename) # Plot the results. from benchopt.plotting import plot_benchmark plot_benchmark(result_filename, benchmark, kinds=kinds, display=display, plotly=plotly, html=html)
def plot(benchmark, filename=None, kinds=('suboptimality_curve', ), display=True, plotly=False): # Get the result file benchmark = Benchmark(benchmark) result_filename = benchmark.get_result_file(filename) # Plot the results. df = pd.read_csv(result_filename) plot_benchmark(df, benchmark, kinds=kinds, display=display, plotly=plotly)
def plot(benchmark, filename=None, kinds=('suboptimality_curve', ), display=True, plotly=False): # Get the result file benchmark = Benchmark(benchmark) result_filename = benchmark.get_result_file(filename) # Load the results. import pandas as pd df = pd.read_csv(result_filename) # Plot the results. from benchopt.plotting import plot_benchmark plot_benchmark(df, benchmark, kinds=kinds, display=display, plotly=plotly)
def run_benchmark(benchmark, solver_names=None, forced_solvers=None, dataset_names=None, objective_filters=None, max_runs=10, n_repetitions=1, timeout=100, plot_result=True, show_progress=True, pdb=False): """Run full benchmark. Parameters ---------- benchmark : benchopt.Benchmark object Object to represent the benchmark. solver_names : list | None List of solvers to include in the benchmark. If None all solvers available are run. forced_solvers : list | None List of solvers to include in the benchmark and for which one forces recomputation. dataset_names : list | None List of datasets to include. If None all available datasets are used. objective_filters : list | None Filters to select specific objective parameters. If None, all objective parameters are tested max_runs : int The maximum number of solver runs to perform to estimate the convergence curve. n_repetitions : int The number of repetitions to run. Defaults to 1. timeout : float The maximum duration in seconds of the solver run. plot_result : bool If set to True (default), display the result plot and save them in the benchmark directory. show_progress : bool If show_progress is set to True, display the progress of the benchmark. pdb : bool It pdb is set to True, open a debugger on error. Returns ------- df : instance of pandas.DataFrame The benchmark results. If multiple metrics were computed, each one is stored in a separate column. If the number of metrics computed by the objective is not the same for all parameters, the missing data is set to `NaN`. """ # Load the objective class for this benchmark and the datasets objective_class = benchmark.get_benchmark_objective() datasets = benchmark.list_benchmark_datasets() # Load the solvers and filter them to get the one to run solver_classes = benchmark.list_benchmark_solvers() included_solvers = _check_name_lists(solver_names, forced_solvers) run_statistics = [] for dataset_class in datasets: for dataset_parameters in product_param(dataset_class.parameters): dataset = dataset_class.get_instance(**dataset_parameters) if not is_matched(str(dataset), dataset_names): continue print(f"{dataset}".ljust(LINE_LENGTH)) if not dataset.is_installed( raise_on_not_installed=RAISE_INSTALL_ERROR): print( colorify(f"Dataset {dataset} is not installed.", RED).ljust(LINE_LENGTH)) continue dimension, data = dataset._get_data() for obj_parameters in product_param(objective_class.parameters): objective = objective_class.get_instance(**obj_parameters) if not is_matched(str(objective), objective_filters): continue print(f"|--{objective}".ljust(LINE_LENGTH)) objective.set_dataset(dataset) for solver_class in solver_classes: for solver_parameters in product_param( solver_class.parameters): # Instantiate solver solver = solver_class.get_instance(**solver_parameters) if not is_matched(solver, included_solvers): continue # Get the solver's name tag = colorify(f"|----{solver}:") # check if the module caught a failed import if not solver.is_installed( raise_on_not_installed=RAISE_INSTALL_ERROR): status = colorify("not installed", RED) print(f"{tag} {status}".ljust(LINE_LENGTH)) continue # Set objective an skip if necessary. skip, reason = solver._set_objective(objective) if skip: print(f"{tag} {colorify('skip', YELLOW)}".ljust( LINE_LENGTH)) if reason is not None: print(f'Reason: {reason}') continue # Get meta meta = dict(objective_name=str(objective), data_name=str(dataset), dimension=dimension) force = (forced_solvers is not None and len(forced_solvers) > 0 and is_matched(str(solver), forced_solvers)) run_statistics.extend( run_one_solver(benchmark=benchmark, objective=objective, solver=solver, meta=meta, tag=tag, max_runs=max_runs, n_repetitions=n_repetitions, timeout=timeout, show_progress=show_progress, force=force, pdb=pdb)) import pandas as pd df = pd.DataFrame(run_statistics) if df.empty: print(colorify('No output produced.', RED).ljust(LINE_LENGTH)) return # raise SystemExit(1) # Save output in CSV file in the benchmark folder timestamp = datetime.now().strftime('%Y-%m-%d_%Hh%Mm%S') output_dir = benchmark.get_output_folder() save_file = output_dir / f'benchopt_run_{timestamp}.csv' df.to_csv(save_file) print(colorify(f'Saving result in: {save_file}', GREEN)) if plot_result: from benchopt.plotting import plot_benchmark plot_benchmark(df, benchmark) return df
from pathlib import Path import matplotlib.pyplot as plt from benchopt import run_benchmark from benchopt.benchmark import Benchmark from benchopt.tests import SELECT_ONE_SIMULATED from benchopt.plotting import plot_benchmark, PLOT_KINDS BENCHMARK_PATH = Path(os.getcwd()).parent / 'benchmarks' / 'lasso' try: df = run_benchmark( Benchmark(BENCHMARK_PATH), ['Python-PGD[^-]*use_acceleration=False', 'R-PGD', 'Julia-PGD'], dataset_names=[SELECT_ONE_SIMULATED], objective_filters=['reg=0.5'], max_runs=100, timeout=100, n_repetitions=5, plot_result=False, show_progress=False) except RuntimeError: raise RuntimeError( "This example can only work when Lasso benchmark is cloned in the " "example folder. Please run:\n" "$ git clone https://github.com/benchopt/benchmark_lasso " f"{BENCHMARK_PATH.resolve()}") kinds = list(PLOT_KINDS.keys()) figs = plot_benchmark(df, benchmark=Benchmark(BENCHMARK_PATH), kinds=kinds) plt.show()
from benchopt.plotting import plot_benchmark, PLOT_KINDS BENCHMARK_PATH = (Path().resolve().parent / 'benchmarks' / 'benchmark_logreg_l2') try: save_file = run_benchmark( Benchmark(BENCHMARK_PATH), ['sklearn[liblinear]', 'sklearn[newton-cg]', 'lightning'], dataset_names=['Simulated*[n_features=500,n_samples=200]'], objective_filters=['L2 Logistic Regression[lmbd=1.0]'], max_runs=100, timeout=20, n_repetitions=15, plot_result=False, show_progress=True) except RuntimeError: raise RuntimeError( "This example can only work when Logreg-l2 benchmark is cloned in a " "`benchmarks` folder. Please run:\n" "$ git clone https://github.com/benchopt/benchmark_logreg_l2 " f"{BENCHMARK_PATH.resolve()}") kinds = list(PLOT_KINDS.keys()) figs = plot_benchmark(save_file, benchmark=Benchmark(BENCHMARK_PATH), kinds=kinds, html=False) plt.show()
def run_benchmark(benchmark, solver_names=None, forced_solvers=None, dataset_names=None, objective_filters=None, max_runs=10, n_repetitions=1, timeout=100, n_jobs=1, slurm=None, plot_result=True, html=True, show_progress=True, pdb=False, output="None"): """Run full benchmark. Parameters ---------- benchmark : benchopt.Benchmark object Object to represent the benchmark. solver_names : list | None List of solvers to include in the benchmark. If None all solvers available are run. forced_solvers : list | None List of solvers to include in the benchmark and for which one forces recomputation. dataset_names : list | None List of datasets to include. If None all available datasets are used. objective_filters : list | None Filters to select specific objective parameters. If None, all objective parameters are tested max_runs : int The maximum number of solver runs to perform to estimate the convergence curve. n_repetitions : int The number of repetitions to run. Defaults to 1. timeout : float The maximum duration in seconds of the solver run. n_jobs : int Maximal number of workers to use to run the benchmark in parallel. slurm : Path | None If not None, launch the job on a slurm cluster using the file to get the cluster config parameters. plot_result : bool If set to True (default), display the result plot and save them in the benchmark directory. html : bool If set to True (default), display the result plot in HTML, otherwise in matplotlib figures, default is True. show_progress : bool If show_progress is set to True, display the progress of the benchmark. pdb : bool It pdb is set to True, open a debugger on error. output_name : str Filename for the parquet output. If given, the results will be stored at <BENCHMARK>/outputs/<filename>.parquet. Returns ------- df : instance of pandas.DataFrame The benchmark results. If multiple metrics were computed, each one is stored in a separate column. If the number of metrics computed by the objective is not the same for all parameters, the missing data is set to `NaN`. """ output_name = output # List all datasets, objective and solvers to run based on the filters # provided. Merge the solver_names and forced to run all necessary solvers. solver_names = _check_name_lists(solver_names, forced_solvers) output = TerminalOutput(n_repetitions, show_progress) output.set(verbose=True) all_runs = benchmark.get_all_runs( solver_names, forced_solvers, dataset_names, objective_filters, output=output ) common_kwargs = dict( benchmark=benchmark, n_repetitions=n_repetitions, max_runs=max_runs, timeout=timeout, pdb=pdb ) if slurm is not None: from .utils.slurm_executor import run_on_slurm results = run_on_slurm(slurm, run_one_solver, common_kwargs, all_runs) else: results = Parallel(n_jobs=n_jobs)( delayed(run_one_solver)(**common_kwargs, **kwargs) for kwargs in all_runs ) run_statistics = [] for curve in results: run_statistics.extend(curve) import pandas as pd df = pd.DataFrame(run_statistics) if df.empty: output.savefile_status() raise SystemExit(1) # Save output in parquet file in the benchmark folder timestamp = datetime.now().strftime('%Y-%m-%d_%Hh%Mm%S') output_dir = benchmark.get_output_folder() if output_name == "None": save_file = output_dir / f'benchopt_run_{timestamp}.parquet' else: save_file = output_dir / f"{output_name}.parquet" save_file = uniquify_results(save_file) df.to_parquet(save_file) output.savefile_status(save_file=save_file) if plot_result: from benchopt.plotting import plot_benchmark plot_benchmark(save_file, benchmark, html=html) return save_file