def test_invalid_dataset(self):
with pytest.raises(click.BadParameter, match=r"invalid_dataset"):
run([
str(DUMMY_BENCHMARK), '-l', '-d', 'invalid_dataset', '-s',
'pgd'
],
'benchopt',
standalone_mode=False)
def setup_class(cls):
"Make sure at least one result file is available"
out = SuppressStd()
with out:
run([
str(DUMMY_BENCHMARK), '-l', '-d', SELECT_ONE_SIMULATED, '-s',
'pgd*False', '-n', '1', '-r', '1', '-p', '0.1', '--no-plot'
],
'benchopt',
standalone_mode=False)
result_files = re.findall(r'Saving result in: (.*\.csv)', out.output)
assert len(result_files) == 1, out.output
result_file = result_files[0]
cls.result_file = result_file
def test_benchopt_run(self):
with CaptureRunOutput() as out:
run([
str(DUMMY_BENCHMARK), '-l', '-d', SELECT_ONE_SIMULATED, '-f',
'pgd*False', '-n', '1', '-r', '1', '-p', '0.1', '--no-plot'
],
'benchopt',
standalone_mode=False)
out.check_output('Simulated', repetition=1)
out.check_output('Dummy Sparse Regression', repetition=1)
out.check_output(r'Python-PGD\[use_acceleration=False\]', repetition=2)
out.check_output(r'Python-PGD\[use_acceleration=True\]', repetition=0)
# Make sure the results were saved in a result file
assert len(out.result_files) == 1, out.output
def test_benchopt_run_in_env(self, test_env_name):
with CaptureRunOutput() as out:
with pytest.raises(SystemExit, match='False'):
run([
str(DUMMY_BENCHMARK_PATH), '--env-name', test_env_name,
'-d', SELECT_ONE_SIMULATED, '-f', SELECT_ONE_PGD, '-n',
'1', '-r', '1', '-p', '0.1', '--no-plot'
],
'benchopt',
standalone_mode=False)
out.check_output(f'conda activate {test_env_name}')
out.check_output('Simulated', repetition=1)
out.check_output('Dummy Sparse Regression', repetition=1)
out.check_output(r'Python-PGD\[step_size=1\]', repetition=2)
out.check_output(r'Python-PGD\[step_size=1.5\]', repetition=0)
# Make sure the results were saved in a result file
assert len(out.result_files) == 1, out.output
def test_benchopt_caching(self):
n_rep = 2
run_cmd = [
str(DUMMY_BENCHMARK), '-l', '-d', SELECT_ONE_SIMULATED, '-s',
'pgd*False', '-n', '1', '-r',
str(n_rep), '-p', '0.1', '--no-plot'
]
# Make a first run that should be put in cache
with CaptureRunOutput() as out:
run(run_cmd, 'benchopt', standalone_mode=False)
# Now check that the cache is hit when running the benchmark a
# second time without force
with CaptureRunOutput() as out:
run(run_cmd, 'benchopt', standalone_mode=False)
out.check_output(r'Python-PGD\[use_acceleration=False\]', repetition=1)
# Make sure that -f option forces the re-run for the solver
run_cmd[4] = '-f'
with CaptureRunOutput() as out:
run(run_cmd, 'benchopt', standalone_mode=False)
out.check_output(r'Python-PGD\[use_acceleration=False\]',
repetition=n_rep + 1)
def test_invalid_benchmark(self, invalid_benchmark, match):
with pytest.raises(click.BadParameter, match=match):
run([invalid_benchmark], 'benchopt', standalone_mode=False)
def test_invalid_benchmark(self):
with pytest.raises(click.BadParameter, match=r"invalid_benchmark"):
run(['invalid_benchmark'], 'benchopt', standalone_mode=False)
def test_invalid_benchmark():
with pytest.raises(SystemExit, match=r"2"):
run(['invalid_benchmark'], 'benchopt')
def test_invalid_dataset():
with pytest.raises(SystemExit, match=r"2"):
run(['lasso', '-d', 'invalid_dataset', '-s', 'baseline'], 'benchopt')
def test_invalid_benchmark():
with pytest.raises(AssertionError, match=r"{'fake_test'} is not"):
run(['fake_test'], [], '')
def test_invalid_solver(self):
with pytest.raises(click.BadParameter, match=r"invalid_solver"):
run([str(DUMMY_BENCHMARK_PATH), '-l', '-s', 'invalid_solver'],
'benchopt',
standalone_mode=False)