def test_raise_all_exceptions(self): """Checks that an exception nested in the benchmarking stack is raised when `raise_all_exceptions` is True. """ def broken_benchmark_replication(*args, **kwargs) -> Experiment: raise ValueError("Oh, exception!") with self.assertRaisesRegex(ValueError, "Oh, exception!"): full_benchmark_run( problem_groups={ self.CATEGORY_NAME: [SimpleBenchmarkProblem(branin, noise_sd=0.4)] }, method_groups={ self.CATEGORY_NAME: [ GenerationStrategy(steps=[ GenerationStep(model=Models.SOBOL, num_trials=-1) ]) ] }, num_replications=3, num_trials=5, raise_all_exceptions=True, benchmark_replication=broken_benchmark_replication, )
def test_basic(self): """Run through the benchmarking loop.""" results = full_benchmark_run( problem_groups={ self.CATEGORY_NAME: [ SimpleBenchmarkProblem(branin, noise_sd=0.4), BenchmarkProblem( name="Branin", search_space=get_branin_search_space(), optimization_config=get_branin_optimization_config(), ), BenchmarkProblem( search_space=get_branin_search_space(), optimization_config=get_optimization_config(), ), ] }, method_groups={ self.CATEGORY_NAME: [ GenerationStrategy(steps=[ GenerationStep(model=Models.SOBOL, num_trials=-1) ]) ] }, num_replications=3, num_trials=5, # Just to have it be more telling if something is broken raise_all_exceptions=True, batch_size=[[1], [3], [1]], ) self.assertEqual(len(results["Branin"]["Sobol"]), 3)
def run_branin_and_gramacy_100_benchmarks(rep): strategy0 = GenerationStrategy( name="Sobol", steps=[ GenerationStep(model=Models.SOBOL, num_arms=-1, model_kwargs={'seed': rep + 1}) ], ) strategy1 = ALEBOStrategy(D=100, d=4, init_size=10) strategy2 = REMBOStrategy(D=100, d=2, init_per_proj=2) strategy3 = HeSBOStrategy(D=100, d=4, init_per_proj=10, name=f"HeSBO, d=2d") all_benchmarks = full_benchmark_run( num_replications=1, num_trials=50, batch_size=1, methods=[strategy0, strategy1, strategy2, strategy3], problems=[branin_100, gramacy_100], ) with open( f'results/branin_gramacy_100_alebo_rembo_hesbo_sobol_rep_{rep}.json', "w") as fout: json.dump(object_to_json(all_benchmarks), fout)
def run_hartmann6_benchmarks(D, rep, random_subspace=False): if D == 100: problem = hartmann6_100 elif D == 1000 and not random_subspace: problem = hartmann6_1000 elif D == 1000 and random_subspace: problem = hartmann6_random_subspace_1000 strategy0 = GenerationStrategy( name="Sobol", steps=[ GenerationStep(model=Models.SOBOL, num_arms=-1, model_kwargs={'seed': rep + 1}) ], ) strategy1 = ALEBOStrategy(D=D, d=12, init_size=10) strategy2 = REMBOStrategy(D=D, d=6, init_per_proj=2) strategy3 = HeSBOStrategy(D=D, d=6, init_per_proj=10, name=f"HeSBO, d=d") strategy4 = HeSBOStrategy(D=D, d=12, init_per_proj=10, name=f"HeSBO, d=2d") all_benchmarks = full_benchmark_run( num_replications=1, # Running them 1 at a time for distributed num_trials=200, batch_size=1, methods=[strategy0, strategy1, strategy2, strategy3, strategy4], problems=[problem], ) rs_str = 'random_subspace_' if random_subspace else '' with open( f'results/hartmann6_{rs_str}{D}_alebo_rembo_hesbo_sobol_rep_{rep}.json', "w") as fout: json.dump(object_to_json(all_benchmarks), fout)
def run_sensitivity_d_e_benchmarks(rep): strategies = [ ALEBOStrategy(D=100, d=d_e, init_size=10, name=f'ALEBO, d={d_e}') for d_e in [2, 3, 5, 6, 7, 8] ] all_benchmarks = full_benchmark_run( num_replications=1, num_trials=50, batch_size=1, methods=strategies, problems=[branin_100], ) with open(f'results/sensitivity_d_e_rep_{rep}.json', "w") as fout: json.dump(object_to_json(all_benchmarks), fout)
def run_sensitivity_D_benchmarks(rep): results_dict = {} for D, problem in branin_by_D.items(): strategy1 = ALEBOStrategy(D=D, d=4, init_size=10) all_benchmarks = full_benchmark_run( num_replications=1, num_trials=50, batch_size=1, methods=[strategy1], problems=[problem], ) results_dict[D] = object_to_json(all_benchmarks) with open(f'results/sensitivity_D_rep_{rep}.json', "w") as fout: json.dump(results_dict, fout)
def run_ablation_benchmarks(rep): strategies = [ ALEBOStrategy_projection_ablation(D=100, d=4, init_size=10, name='ALEBO, projection ablation'), ALEBOStrategy_kernel_ablation(D=100, d=4, init_size=10, name='ALEBO, kernel ablation'), ALEBOStrategy(D=100, d=4, init_size=10, name='ALEBO, base'), ] all_benchmarks = full_benchmark_run( num_replications=1, num_trials=50, batch_size=1, methods=strategies, problems=[branin_100], ) with open(f'results/ablation_rep_{rep}.json', "w") as fout: json.dump(object_to_json(all_benchmarks), fout)