def test_benchopt_caching(self): # Check that the computation caching is working properly. n_rep = 2 run_cmd = [ str(DUMMY_BENCHMARK_PATH), '-l', '-d', SELECT_ONE_SIMULATED, '-s', SELECT_ONE_PGD, '-n', '1', '-r', str(n_rep), '-p', '0.1', '--no-plot' ] # Make a first run that should be put in cache with CaptureRunOutput() as out: run(run_cmd, 'benchopt', standalone_mode=False) # Now check that the cache is hit when running the benchmark a # second time without force with CaptureRunOutput() as out: run(run_cmd, 'benchopt', standalone_mode=False) out.check_output(r'Python-PGD\[step_size=1\]:', repetition=1) # Make sure that -f option forces the re-run for the solver run_cmd[4] = '-f' with CaptureRunOutput() as out: run(run_cmd, 'benchopt', standalone_mode=False) out.check_output(r'Python-PGD\[step_size=1\]:', repetition=2 * n_rep + 1)
def test_benchopt_run_custom_parameters(self): SELECT_DATASETS = r'simulated[n_features=[100, 200]]' SELECT_SOLVERS = r'python-pgd-with-cb[use_acceleration=[True, False]]' SELECT_OBJECTIVES = r'dummy*[0.1, 0.2]' with CaptureRunOutput() as out: run([ str(DUMMY_BENCHMARK_PATH), '-l', '-d', SELECT_DATASETS, '-f', SELECT_SOLVERS, '-n', '1', '-r', '1', '-o', SELECT_OBJECTIVES, '--no-plot' ], 'benchopt', standalone_mode=False) out.check_output(r'Simulated\[n_features=100,', repetition=1) out.check_output(r'Simulated\[n_features=200,', repetition=1) out.check_output(r'Simulated\[n_features=5000,', repetition=0) out.check_output(r'Dummy Sparse Regression\[reg=0.1\]', repetition=2) out.check_output(r'Dummy Sparse Regression\[reg=0.2\]', repetition=2) out.check_output(r'Dummy Sparse Regression\[reg=0.05\]', repetition=0) out.check_output(r'--Python-PGD\[', repetition=0) out.check_output(r'--Python-PGD-with-cb\[use_acceleration=False\]:', repetition=28) out.check_output(r'--Python-PGD-with-cb\[use_acceleration=True\]:', repetition=28)
def test_benchopt_install(self): with CaptureRunOutput() as out: install( [str(DUMMY_BENCHMARK_PATH), '-d', SELECT_ONE_SIMULATED, '-s', SELECT_ONE_PGD, '-y'], 'benchopt', standalone_mode=False ) out.check_output(f"Installing '{DUMMY_BENCHMARK.name}' requirements") out.check_output("already available\n", repetition=2)
def test_benchopt_run_config_file(self): tmp = tempfile.NamedTemporaryFile(mode="w+") tmp.write("some_unknown_option: 0") tmp.flush() with pytest.raises(ValueError, match="Invalid config file option"): run(f'{str(DUMMY_BENCHMARK_PATH)} --config {tmp.name}'.split(), 'benchopt', standalone_mode=False) config = f""" objective-filter: - {SELECT_ONE_OBJECTIVE} dataset: - {SELECT_ONE_SIMULATED} n-repetitions: 2 max-runs: 1 force-solver: - python-pgd[step_size=[2, 3]] - Test-Solver """ tmp = tempfile.NamedTemporaryFile(mode="w+") tmp.write(config) tmp.flush() run_cmd = [ str(DUMMY_BENCHMARK_PATH), '--config', tmp.name, '--no-plot' ] with CaptureRunOutput() as out: run(run_cmd, 'benchopt', standalone_mode=False) out.check_output(r'Test-Solver:', repetition=11) out.check_output(r'Python-PGD\[step_size=2\]:', repetition=11) out.check_output(r'Python-PGD\[step_size=3\]:', repetition=11) # test that CLI options take precedence with CaptureRunOutput() as out: run(run_cmd + ['-f', 'Test-Solver'], 'benchopt', standalone_mode=False) out.check_output(r'Test-Solver:', repetition=11) out.check_output(r'Python-PGD\[step_size=1.5\]:', repetition=0)
def test_benchopt_caching(self, n_rep): clean([str(DUMMY_BENCHMARK_PATH)], 'benchopt', standalone_mode=False) # XXX - remove once this is fixed upstream with joblib/joblib#1289 _FUNCTION_HASHES.clear() # Check that the computation caching is working properly. run_cmd = [ str(DUMMY_BENCHMARK_PATH), '-l', '-d', SELECT_ONE_SIMULATED, '-s', SELECT_ONE_PGD, '-n', '1', '-r', str(n_rep), '-o', SELECT_ONE_OBJECTIVE, '--no-plot' ] # Make a first run that should be put in cache with CaptureRunOutput() as out: run(run_cmd, 'benchopt', standalone_mode=False) # Check that this run was properly done. If only one is detected, this # could indicate that the clean command does not work properly. out.check_output(r'Python-PGD\[step_size=1\]:', repetition=5 * n_rep + 1) # Now check that the cache is hit when running the benchmark a # second time without force with CaptureRunOutput() as out: run(run_cmd, 'benchopt', standalone_mode=False) out.check_output(r'Python-PGD\[step_size=1\]:', repetition=1) # Check that the cache is also hit when running in parallel with CaptureRunOutput() as out: run(run_cmd + ['-j', 2], 'benchopt', standalone_mode=False) out.check_output(r'Python-PGD\[step_size=1\]:', repetition=1) # Make sure that -f option forces the re-run for the solver run_cmd[4] = '-f' with CaptureRunOutput() as out: run(run_cmd, 'benchopt', standalone_mode=False) out.check_output(r'Python-PGD\[step_size=1\]:', repetition=5 * n_rep + 1)
def test_benchopt_run(self): with CaptureRunOutput() as out: run([str(DUMMY_BENCHMARK_PATH), '-l', '-d', SELECT_ONE_SIMULATED, '-f', SELECT_ONE_PGD, '-n', '1', '-r', '1', '-o', SELECT_ONE_OBJECTIVE], 'benchopt', standalone_mode=False) out.check_output('Simulated', repetition=1) out.check_output('Dummy Sparse Regression', repetition=1) out.check_output(r'Python-PGD\[step_size=1\]:', repetition=3) out.check_output(r'Python-PGD\[step_size=1.5\]:', repetition=0) # Make sure the results were saved in a result file assert len(out.result_files) == 1, out.output
def test_benchopt_install_in_env_with_requirements(self, test_env_name): objective = REQUIREMENT_BENCHMARK.get_benchmark_objective() out = 'already installed but failed to import.' if not objective.is_installed(env_name=test_env_name): with CaptureRunOutput() as out: install([ str(REQUIREMENT_BENCHMARK_PATH), '--env-name', test_env_name ], 'benchopt', standalone_mode=False) assert objective.is_installed(env_name=test_env_name), out # XXX: run the bench with CaptureRunOutput() as out: with pytest.raises(SystemExit, match='False'): run_cmd = [ str(REQUIREMENT_BENCHMARK_PATH), '--env-name', test_env_name, '-n', '10', '-r', '1', '--no-plot' ] run(run_cmd, 'benchopt', standalone_mode=False) out.check_output(r"done \(not enough run\)", repetition=1)
def test_changing_output_name(self): command = [ str(DUMMY_BENCHMARK_PATH), '-l', '-s', SELECT_ONE_PGD, '-d', SELECT_ONE_SIMULATED, '-n', '1', '--output', 'unique_name', '--no-plot' ] with CaptureRunOutput() as out: run(command, 'benchopt', standalone_mode=False) run(command, 'benchopt', standalone_mode=False) result_files = re.findall(r'Saving result in: (.*\.parquet)', out.output) names = [Path(result_file).stem for result_file in result_files] assert names[0] == 'unique_name' and names[1] == 'unique_name_1'
def test_benchopt_run_profile(self): with CaptureRunOutput() as out: run_cmd = [str(DUMMY_BENCHMARK_PATH), '-d', SELECT_ONE_SIMULATED, '-f', SELECT_ONE_PGD, '-n', '1', '-r', '1', '-o', SELECT_ONE_OBJECTIVE, '--profile', '--no-plot'] run(run_cmd, 'benchopt', standalone_mode=False) out.check_output('Using profiling', repetition=1) out.check_output("File: .*benchopt/tests/test_benchmarks/" "dummy_benchmark/solvers/python_pgd.py", repetition=1) out.check_output(r'\s+'.join([ "Line #", "Hits", "Time", "Per Hit", "% Time", "Line Contents" ]), repetition=1) out.check_output(r"def run\(self, n_iter\):", repetition=1)
def test_benchopt_run_in_env(self, test_env_name): with CaptureRunOutput() as out: with pytest.raises(SystemExit, match='False'): run([str(DUMMY_BENCHMARK_PATH), '--env-name', test_env_name, '-d', SELECT_ONE_SIMULATED, '-f', SELECT_ONE_PGD, '-n', '1', '-r', '1', '-o', SELECT_ONE_OBJECTIVE, '--no-plot'], 'benchopt', standalone_mode=False) out.check_output(f'conda activate {test_env_name}') out.check_output('Simulated', repetition=1) out.check_output('Dummy Sparse Regression', repetition=1) out.check_output(r'Python-PGD\[step_size=1\]:', repetition=3) out.check_output(r'Python-PGD\[step_size=1.5\]:', repetition=0) # Make sure the results were saved in a result file assert len(out.result_files) == 1, out.output