def test(benchmark, env_name, pytest_args):
pytest_args = ' '.join(pytest_args)
if len(pytest_args) == 0:
pytest_args = '-vl'
env_option = ''
if env_name is not None:
create_conda_env(env_name, with_pytest=True)
if _run_shell_in_conda_env("pytest --version", env_name=env_name) != 0:
raise ModuleNotFoundError(
f"pytest is not installed in conda env {env_name}.\n"
f"Please run `conda install -n {env_name} pytest` to test the "
"benchmark in this environment."
)
env_option = f'--test-env {env_name}'
from benchopt.tests import __file__ as _bench_test_module
BENCHMARK_TEST_FILE = (
Path(_bench_test_module).parent / "test_benchmarks.py"
)
cmd = (
f'pytest {pytest_args} {BENCHMARK_TEST_FILE} '
f'--benchmark {benchmark} {env_option} '
# Make sure to not modify sys.path to add test file from current env
# in sub conda env as there might be different python versions.
'--import-mode importlib'
)
raise SystemExit(_run_shell_in_conda_env(
cmd, env_name=env_name, capture_stdout=False
) != 0)
def install(benchmark, solver_names, dataset_names, force=False,
recreate=False, env_name='False', confirm=False, quiet=False):
# Check that the dataset/solver patterns match actual dataset
benchmark = Benchmark(benchmark)
print(f"Installing '{benchmark.name}' requirements")
benchmark.validate_dataset_patterns(dataset_names)
benchmark.validate_solver_patterns(solver_names)
# Get a list of all conda envs
default_conda_env, conda_envs = list_conda_envs()
# If env_name is False (default), installation in the current environement.
if env_name == 'False':
env_name = None
# incompatible with the 'recreate' flag to avoid messing with the
# user environement
if recreate:
msg = "Cannot recreate conda env without using options " + \
"'-e/--env' or '--env-name'."
raise RuntimeError(msg)
# check if any current conda environment
if default_conda_env is not None:
# ask for user confirmation to install in current conda env
if not confirm:
click.confirm(
f"Install in the current env '{default_conda_env}'?",
abort=True
)
else:
raise RuntimeError("No conda environment is activated.")
else:
# If env_name is True, the flag `--env` has been used. Create a conda
# env specific to the benchmark. Else, use the <env_name> value.
if env_name == 'True':
env_name = f"benchopt_{benchmark.name}"
else:
# check provided <env_name>
# (to avoid empty name like `--env-name ""`)
if len(env_name) == 0:
raise RuntimeError("Empty environment name.")
# avoid recreating 'base' conda env`
if env_name == 'base' and recreate:
raise RuntimeError(
"Impossible to recreate 'base' conda environment."
)
# create environment if necessary
create_conda_env(env_name, recreate=recreate, quiet=quiet)
# install requirements
print("# Install", flush=True)
benchmark.install_all_requirements(
include_solvers=solver_names, include_datasets=dataset_names,
env_name=env_name, force=force, quiet=quiet,
)
def empty_env_name(request):
global _EMPTY_ENV_NAME
if _EMPTY_ENV_NAME is None:
env_name = f"_benchopt_test_env_{uuid.uuid4()}"
request.addfinalizer(delete_empty_env)
_EMPTY_ENV_NAME = env_name
create_conda_env(_EMPTY_ENV_NAME, empty=True)
return _EMPTY_ENV_NAME
def test_env_name(request):
global _TEST_ENV_NAME
if _TEST_ENV_NAME is None:
env_name = request.config.getoption("--test-env")
recreate = request.config.getoption("--recreate")
if env_name is None:
env_name = f"_benchopt_test_env_{uuid.uuid4()}"
request.addfinalizer(delete_test_env)
_TEST_ENV_NAME = env_name
create_conda_env(_TEST_ENV_NAME, recreate=recreate)
return _TEST_ENV_NAME
def test(benchmark, env_name, pytest_args):
benchmark = Benchmark(benchmark)
from benchopt.tests import __file__ as _bench_test_module
_bench_test_module = Path(_bench_test_module).parent
pytest_args = ' '.join(("-p benchopt.tests.fixtures",
f"--rootdir {_bench_test_module}", *pytest_args))
if len(pytest_args) == 0:
pytest_args = '-vl'
env_option = ''
if env_name is not None:
create_conda_env(env_name, with_pytest=True)
if _run_shell_in_conda_env("pytest --version", env_name=env_name) != 0:
raise ModuleNotFoundError(
f"pytest is not installed in conda env {env_name}.\n"
f"Please run `conda install -n {env_name} pytest` to test the "
"benchmark in this environment.")
objective = benchmark.get_benchmark_objective()
if not objective.is_installed():
objective.install(env_name=env_name)
env_option = f'--test-env {env_name}'
_bench_test_file = _bench_test_module / "test_benchmarks.py"
cmd = (
f'pytest {pytest_args} {_bench_test_file} '
f'--benchmark {benchmark.benchmark_dir} {env_option} '
# Make sure to not modify sys.path to add test file from current env
# in sub conda env as there might be different python versions.
'--import-mode importlib')
raise SystemExit(
_run_shell_in_conda_env(cmd, env_name=env_name, capture_stdout=False)
!= 0)