def info(benchmark,
solver_names,
dataset_names,
env_name='False',
verbose=False):
# benchmark
benchmark = Benchmark(benchmark)
print(f"Info regarding the benchmark '{benchmark.name}'")
# validate solvers and datasets
benchmark.validate_dataset_patterns(dataset_names)
benchmark.validate_solver_patterns(solver_names)
# get solvers and datasets in the benchmark
all_solvers = benchmark.get_solvers()
all_datasets = benchmark.get_datasets()
# enable verbosity if any environment was provided
if env_name is not None and env_name != 'False':
verbose = True
# conda env check only in verbose case
if verbose:
# Check conda env name
env_name = check_conda_env(env_name, benchmark.name)
# check conda environment validity
check_benchopt = _run_shell_in_conda_env("benchopt --version",
env_name=env_name,
capture_stdout=True)
if check_benchopt != 0:
warnings.warn(
f"Environment '{env_name}' does not exist "
"or is not configured for benchopt, "
"benchmark requirement availability will not be checked, "
"see the command `benchopt install`.", UserWarning)
env_name = None
else:
print("Checking benchmark requirement availability "
f"in env '{env_name}'.")
print("Note: you can install all dependencies from a benchmark "
"with the command `benchopt install`.")
# enable verbosity if any solver/dataset are specified in input
if dataset_names or solver_names:
verbose = True
# print information
print("-" * 10)
if not dataset_names and not solver_names:
dataset_names = ['all']
solver_names = ['all']
if dataset_names:
print("# DATASETS", flush=True)
print_info(dataset_names, all_datasets, env_name, verbose)
if solver_names:
print("# SOLVERS", flush=True)
print_info(solver_names, all_solvers, env_name, verbose)
def install(benchmark, solver_names, dataset_names, force=False,
recreate=False, env_name='False', confirm=False, quiet=False):
# Check that the dataset/solver patterns match actual dataset
benchmark = Benchmark(benchmark)
print(f"Installing '{benchmark.name}' requirements")
benchmark.validate_dataset_patterns(dataset_names)
benchmark.validate_solver_patterns(solver_names)
# Get a list of all conda envs
default_conda_env, conda_envs = list_conda_envs()
# If env_name is False (default), installation in the current environement.
if env_name == 'False':
env_name = None
# incompatible with the 'recreate' flag to avoid messing with the
# user environement
if recreate:
msg = "Cannot recreate conda env without using options " + \
"'-e/--env' or '--env-name'."
raise RuntimeError(msg)
# check if any current conda environment
if default_conda_env is not None:
# ask for user confirmation to install in current conda env
if not confirm:
click.confirm(
f"Install in the current env '{default_conda_env}'?",
abort=True
)
else:
raise RuntimeError("No conda environment is activated.")
else:
# If env_name is True, the flag `--env` has been used. Create a conda
# env specific to the benchmark. Else, use the <env_name> value.
if env_name == 'True':
env_name = f"benchopt_{benchmark.name}"
else:
# check provided <env_name>
# (to avoid empty name like `--env-name ""`)
if len(env_name) == 0:
raise RuntimeError("Empty environment name.")
# avoid recreating 'base' conda env`
if env_name == 'base' and recreate:
raise RuntimeError(
"Impossible to recreate 'base' conda environment."
)
# create environment if necessary
create_conda_env(env_name, recreate=recreate, quiet=quiet)
# install requirements
print("# Install", flush=True)
benchmark.install_all_requirements(
include_solvers=solver_names, include_datasets=dataset_names,
env_name=env_name, force=force, quiet=quiet,
)
def run(config_file=None, **kwargs):
if config_file is not None:
with open(config_file, "r") as f:
config = yaml.safe_load(f)
else:
config = {}
# XXX - Remove old and deprecated objective filters in version 1.3
(benchmark, solver_names, forced_solvers, dataset_names, objective_filters,
max_runs, n_repetitions, timeout, n_jobs, slurm, plot, html, pdb,
do_profile, env_name, output, deprecated_objective_filters,
old_objective_filters) = _get_run_args(kwargs, config)
if len(old_objective_filters):
warnings.warn(
'Using the -p option is deprecated, use -o instead',
FutureWarning,
)
objective_filters = old_objective_filters
if len(deprecated_objective_filters):
warnings.warn(
'Using the --objective-filters option is deprecated, '
'use --objective instead', FutureWarning)
objective_filters = deprecated_objective_filters
# Create the Benchmark object
benchmark = Benchmark(benchmark)
# If env_name is False, the flag `--local` has been used (default) so
# run in the current environment.
if env_name == 'False':
print("Benchopt is running")
if slurm is not None:
print("Running on SLURM")
set_slurm_launch()
from benchopt.runner import run_benchmark
if do_profile:
from benchopt.utils.profiling import use_profile
use_profile() # needs to be called before validate_solver_patterns
# Check that the dataset/solver patterns match actual dataset
benchmark.validate_dataset_patterns(dataset_names)
benchmark.validate_objective_filters(objective_filters)
# pyyaml returns tuples: make sure everything is a list
benchmark.validate_solver_patterns(
list(solver_names) + list(forced_solvers))
run_benchmark(benchmark,
solver_names,
forced_solvers,
dataset_names=dataset_names,
objective_filters=objective_filters,
max_runs=max_runs,
n_repetitions=n_repetitions,
timeout=timeout,
n_jobs=n_jobs,
slurm=slurm,
plot_result=plot,
html=html,
pdb=pdb,
output=output)
print_stats() # print profiling stats (does nothing if not profiling)
return
default_conda_env, all_conda_envs = list_conda_envs()
# If env_name is True, the flag `--env` has been used. Create a conda env
# specific to the benchmark (if not existing).
# Else, use the <env_name> value.
# check if any current conda environment
if default_conda_env is None:
raise RuntimeError(
"No conda environment is activated. "
"You should be in a conda environment to use "
"'benchopt run' with options '-e/--env' or '--env-name'.")
if env_name == 'True':
env_name = f"benchopt_{benchmark.name}"
install_cmd = f"`benchopt install -e {benchmark.benchmark_dir}`"
else:
# check provided <env_name>
# (to avoid empty name like `--env-name ""`)
if len(env_name) == 0:
raise RuntimeError("Empty environment name.")
install_cmd = (f"`benchopt install --env-name {env_name} "
f"{benchmark.benchmark_dir}`")
# check if the environment exists
if env_name not in all_conda_envs:
raise RuntimeError(
f"The default env '{env_name}' for benchmark {benchmark.name} "
f"does not exist. Make sure to run {install_cmd} to create the "
"benchmark and install the dependencies.")
print(f"Launching benchopt in env {env_name}")
# check if environment was set up with benchopt
benchopt_version, is_editable = get_benchopt_version_in_env(env_name)
if benchopt_version is None:
raise RuntimeError(
f"benchopt is not installed in env '{env_name}', "
"see the command `benchopt install` to setup the environment.")
# check against running version
from benchopt import __version__ as benchopt_version_running
_, is_editable_running = get_benchopt_requirement()
if (benchopt_version_running != benchopt_version
and not (is_editable_running and is_editable)):
warnings.warn(
f"Benchopt running version ({benchopt_version_running}) "
f"and version in env {env_name} ({benchopt_version}) differ")
# run the command in the conda env
solvers_option = ' '.join([f"-s '{s}'" for s in solver_names])
forced_solvers_option = ' '.join([f"-f '{s}'" for s in forced_solvers])
datasets_option = ' '.join([f"-d '{d}'" for d in dataset_names])
objective_option = ' '.join([f"-o '{o}'" for o in objective_filters])
cmd = (rf"benchopt run --local {benchmark.benchmark_dir} "
rf"--n-repetitions {n_repetitions} "
rf"--max-runs {max_runs} --timeout {timeout} "
rf"--n-jobs {n_jobs} {'--slurm' if slurm else ''} "
rf"{solvers_option} {forced_solvers_option} "
rf"{datasets_option} {objective_option} "
rf"{'--plot' if plot else '--no-plot'} "
rf"{'--html' if html else '--no-html'} "
rf"{'--pdb' if pdb else ''} "
rf"--output {output}".replace('\\', '\\\\'))
raise SystemExit(
_run_shell_in_conda_env(cmd, env_name=env_name, capture_stdout=False)
!= 0)
def run(benchmark,
solver_names,
forced_solvers,
dataset_names,
objective_filters,
max_runs,
n_repetitions,
timeout,
recreate=False,
plot=True,
pdb=False,
env_name='False'):
# Check that the dataset/solver patterns match actual dataset
benchmark = Benchmark(benchmark)
benchmark.validate_dataset_patterns(dataset_names)
benchmark.validate_solver_patterns(solver_names + forced_solvers)
# If env_name is False, the flag `--local` has been used (default) so
# run in the current environement.
if env_name == 'False':
run_benchmark(benchmark,
solver_names,
forced_solvers,
dataset_names=dataset_names,
objective_filters=objective_filters,
max_runs=max_runs,
n_repetitions=n_repetitions,
timeout=timeout,
plot_result=plot,
pdb=pdb)
return
# If env_name is True, the flag `--env` has been used. Create a conda env
# specific to the benchmark. Else, use the <env_name> value.
if env_name == 'True':
env_name = f"benchopt_{benchmark.name}"
create_conda_env(env_name, recreate=recreate)
# installed required datasets
benchmark.install_required_datasets(dataset_names, env_name=env_name)
# Get the solvers and install them
benchmark.install_required_solvers(solver_names,
forced_solvers=forced_solvers,
env_name=env_name)
# run the command in the conda env
solvers_option = ' '.join(['-s ' + s for s in solver_names])
forced_solvers_option = ' '.join(['-f ' + s for s in forced_solvers])
datasets_option = ' '.join(['-d ' + d for d in dataset_names])
objective_option = ' '.join(['-p ' + p for p in objective_filters])
cmd = (rf"benchopt run --local {benchmark.benchmark_dir} "
rf"--n-repetitions {n_repetitions} "
rf"--max-runs {max_runs} --timeout {timeout} "
rf"{solvers_option} {forced_solvers_option} "
rf"{datasets_option} {objective_option} "
rf"{'--plot' if plot else '--no-plot'} "
rf"{'--pdb' if pdb else ''} ".replace('\\', '\\\\'))
raise SystemExit(
_run_shell_in_conda_env(cmd, env_name=env_name, capture_stdout=False)
!= 0)
def run(benchmark, solver_names, forced_solvers, dataset_names,
objective_filters, max_runs, n_repetitions, timeout,
plot=True, html=True, pdb=False, do_profile=False,
env_name='False', old_objective_filters=None):
if len(old_objective_filters):
warnings.warn(
'Using the -p option is deprecated, use -o instead',
FutureWarning,
)
objective_filters = old_objective_filters
from benchopt.runner import run_benchmark
if do_profile:
from benchopt.utils.profiling import use_profile
use_profile() # needs to be called before validate_solver_patterns
# Check that the dataset/solver patterns match actual dataset
benchmark = Benchmark(benchmark)
benchmark.validate_dataset_patterns(dataset_names)
benchmark.validate_solver_patterns(solver_names+forced_solvers)
benchmark.validate_objective_filters(objective_filters)
# If env_name is False, the flag `--local` has been used (default) so
# run in the current environement.
if env_name == 'False':
run_benchmark(
benchmark, solver_names, forced_solvers,
dataset_names=dataset_names,
objective_filters=objective_filters,
max_runs=max_runs, n_repetitions=n_repetitions,
timeout=timeout, plot_result=plot, html=html, pdb=pdb
)
print_stats() # print profiling stats (does nothing if not profiling)
return
_, all_conda_envs = list_conda_envs()
# If env_name is True, the flag `--env` has been used. Create a conda env
# specific to the benchmark (if not existing).
# Else, use the <env_name> value.
if env_name == 'True':
env_name = f"benchopt_{benchmark.name}"
install_cmd = f"`benchopt install -e {benchmark.benchmark_dir}`"
else:
# check provided <env_name>
# (to avoid empty name like `--env-name ""`)
if len(env_name) == 0:
raise RuntimeError("Empty environment name.")
install_cmd = (
f"`benchopt install --env-name {env_name} "
f"{benchmark.benchmark_dir}`"
)
# check if the environment exists
if env_name not in all_conda_envs:
raise RuntimeError(
f"The default env '{env_name}' for benchmark {benchmark.name} "
f"does not exist. Make sure to run {install_cmd} to create the "
"benchmark and install the dependencies."
)
# check if environment was set up with benchopt
if get_benchopt_version_in_env(env_name) is None:
raise RuntimeError(
f"benchopt is not installed in env '{env_name}', "
"see the command `benchopt install` to setup the environment."
)
# run the command in the conda env
solvers_option = ' '.join(['-s ' + s for s in solver_names])
forced_solvers_option = ' '.join([f"-f '{s}'" for s in forced_solvers])
datasets_option = ' '.join([f"-d '{d}'" for d in dataset_names])
objective_option = ' '.join([f"-p '{p}'" for p in objective_filters])
cmd = (
rf"benchopt run --local {benchmark.benchmark_dir} "
rf"--n-repetitions {n_repetitions} "
rf"--max-runs {max_runs} --timeout {timeout} "
rf"{solvers_option} {forced_solvers_option} "
rf"{datasets_option} {objective_option} "
rf"{'--plot' if plot else '--no-plot'} "
rf"{'--html' if html else '--no-html'} "
rf"{'--pdb' if pdb else ''} "
.replace('\\', '\\\\')
)
raise SystemExit(_run_shell_in_conda_env(
cmd, env_name=env_name, capture_stdout=False
) != 0)