def run(benchmark,
solver_names,
forced_solvers,
dataset_names,
objective_filters,
max_runs,
n_repetitions,
timeout,
recreate=False,
plot=True,
pdb=False,
env_name='False'):
# Check that the dataset/solver patterns match actual dataset
benchmark = Benchmark(benchmark)
benchmark.validate_dataset_patterns(dataset_names)
benchmark.validate_solver_patterns(solver_names + forced_solvers)
# If env_name is False, the flag `--local` has been used (default) so
# run in the current environement.
if env_name == 'False':
run_benchmark(benchmark,
solver_names,
forced_solvers,
dataset_names=dataset_names,
objective_filters=objective_filters,
max_runs=max_runs,
n_repetitions=n_repetitions,
timeout=timeout,
plot_result=plot,
pdb=pdb)
return
# If env_name is True, the flag `--env` has been used. Create a conda env
# specific to the benchmark. Else, use the <env_name> value.
if env_name == 'True':
env_name = f"benchopt_{benchmark.name}"
create_conda_env(env_name, recreate=recreate)
# installed required datasets
benchmark.install_required_datasets(dataset_names, env_name=env_name)
# Get the solvers and install them
benchmark.install_required_solvers(solver_names,
forced_solvers=forced_solvers,
env_name=env_name)
# run the command in the conda env
solvers_option = ' '.join(['-s ' + s for s in solver_names])
forced_solvers_option = ' '.join(['-f ' + s for s in forced_solvers])
datasets_option = ' '.join(['-d ' + d for d in dataset_names])
objective_option = ' '.join(['-p ' + p for p in objective_filters])
cmd = (rf"benchopt run --local {benchmark.benchmark_dir} "
rf"--n-repetitions {n_repetitions} "
rf"--max-runs {max_runs} --timeout {timeout} "
rf"{solvers_option} {forced_solvers_option} "
rf"{datasets_option} {objective_option} "
rf"{'--plot' if plot else '--no-plot'} "
rf"{'--pdb' if pdb else ''} ".replace('\\', '\\\\'))
raise SystemExit(
_run_shell_in_conda_env(cmd, env_name=env_name, capture_stdout=False)
!= 0)
def install(benchmark, solver_names, dataset_names, force=False,
recreate=False, env_name='False', confirm=False):
# Check that the dataset/solver patterns match actual dataset
benchmark = Benchmark(benchmark)
print(f"Installing '{benchmark.name}' requirements")
benchmark.validate_dataset_patterns(dataset_names)
benchmark.validate_solver_patterns(solver_names)
# Get a list of all conda envs
default_conda_env, conda_envs = list_conda_envs()
# If env_name is False (default), installation in the current environement.
if env_name == 'False':
env_name = None
# incompatible with the 'recreate' flag to avoid messing with the
# user environement
if recreate:
msg = "Cannot recreate conda env without using options " + \
"'-e/--env' or '--env-name'."
raise RuntimeError(msg)
# check if any current conda environment
if default_conda_env is not None:
# ask for user confirmation to install in current conda env
if not confirm:
click.confirm(
f"Install in the current env '{default_conda_env}'?",
abort=True
)
else:
raise RuntimeError("No conda environment is activated.")
else:
# If env_name is True, the flag `--env` has been used. Create a conda
# env specific to the benchmark. Else, use the <env_name> value.
if env_name == 'True':
env_name = f"benchopt_{benchmark.name}"
else:
# check provided <env_name>
# (to avoid empty name like `--env-name ""`)
if len(env_name) == 0:
raise RuntimeError("Empty environment name.")
# avoid recreating 'base' conda env`
if env_name == 'base' and recreate:
raise RuntimeError(
"Impossible to recreate 'base' conda environment."
)
# create environment if necessary
create_conda_env(env_name, recreate=recreate)
# If force is True (default is False), it forces the reinstallation of
# selected solvers (all solvers from the benchmark by default)
forced_solvers = ()
if force:
if len(solver_names) > 0:
forced_solvers = solver_names
else:
forced_solvers = benchmark.list_benchmark_solver_names()
# same for datasets
forced_datasets = ()
if force:
if len(dataset_names) > 0:
forced_datasets = dataset_names
else:
forced_datasets = benchmark.list_benchmark_dataset_names()
# install required datasets
print("# Datasets", flush=True)
benchmark.install_required_datasets(
dataset_names, forced_datasets=forced_datasets, env_name=env_name
)
# install required solvers
print("# Solvers", flush=True)
benchmark.install_required_solvers(
solver_names, forced_solvers=forced_solvers, env_name=env_name
)