def compile_sensitivity_benchmarks():
all_results = {}
for rep in range(50):
## Sensitivity to D
with open(f'results/sensitivity_D_rep_{rep}.json', 'r') as fin:
results_dict = json.load(fin)
for D, obj in results_dict.items():
res_i = object_from_json(obj)
all_results = merge_benchmark_results(all_results, res_i)
## Sensitivity to d_e
with open(f'results/sensitivity_d_e_rep_{rep}.json', 'r') as fin:
res_i = object_from_json(json.load(fin))
all_results = merge_benchmark_results(all_results, res_i)
all_problems = (
[hartmann6_100, hartmann6_1000, branin_100, gramacy_100]
+ list(branin_by_D.values())
)
problems = [branin_100] + list(branin_by_D.values())
res = {
p.name+'_sensitivity': aggregate_problem_results(runs=all_results[p.name], problem=p)
for p in problems
}
# Save
with open(f'results/sensitivity_aggregated_results.json', "w") as fout:
json.dump(object_to_json(res), fout)
def compile_branin_gramacy_100():
all_results = {}
for rep in range(50):
with open(f'results/branin_gramacy_100_alebo_rembo_hesbo_sobol_rep_{rep}.json', 'r') as fin:
res_i = object_from_json(json.load(fin))
all_results = merge_benchmark_results(all_results, res_i)
for method_name in ['addgpucb', 'cmaes', 'ebo', 'smac', 'turbo']:
with open(f'results/branin_100_{method_name}_rep_{rep}.json', 'r') as fin:
exp_i = object_from_json(json.load(fin))
all_results = add_exp(
res=all_results,
exp=exp_i,
problem_name='Branin, D=100',
method_name=method_name,
)
res = {
p.name: aggregate_problem_results(runs=all_results[p.name], problem=p)
for p in [branin_100, gramacy_100]
}
# Add in RRembo results
for proj in ['standard', 'reverse']:
method = f'rrembos_{proj}_kPsi'
with open(f'results/branin_100_{method}.json', 'r') as fin:
A = json.load(fin)
res['Branin, D=100'].objective_at_true_best[method] = np.minimum.accumulate(np.array(A), axis=1)
# Save
with open(f'results/branin_gramacy_100_aggregated_results.json', "w") as fout:
json.dump(object_to_json(res), fout)
def testEncodeDecodeSimpleBenchmarkProblem(self):
branin_problem = get_branin_simple_benchmark_problem()
sum_problem = get_sum_simple_benchmark_problem()
new_branin_problem = object_from_json(object_to_json(branin_problem))
new_sum_problem = object_from_json(object_to_json(sum_problem))
self.assertEqual(branin_problem.f(1, 2), new_branin_problem.f(1, 2),
branin(1, 2))
self.assertEqual(sum_problem.f([1, 2]), new_sum_problem.f([1, 2]), 3)
def from_json_snapshot(serialized: Dict[str, Any]) -> "AxClient":
"""Recreate an `AxClient` from a JSON snapshot."""
experiment = object_from_json(serialized.pop("experiment"))
ax_client = AxClient(
generation_strategy=generation_strategy_from_json(
generation_strategy_json=serialized.pop(
"generation_strategy")),
enforce_sequential_optimization=serialized.pop(
"_enforce_sequential_optimization"),
)
ax_client._experiment = experiment
ax_client._updated_trials = object_from_json(
serialized.pop("_updated_trials"))
return ax_client
def testEncodeDecodeSimpleBenchmarkProblem(self):
branin_problem = get_branin_simple_benchmark_problem()
sum_problem = get_sum_simple_benchmark_problem()
new_branin_problem = object_from_json(object_to_json(branin_problem))
new_sum_problem = object_from_json(object_to_json(sum_problem))
self.assertEqual(branin_problem.f(1, 2), new_branin_problem.f(1, 2),
branin(1, 2))
self.assertEqual(sum_problem.f([1, 2]), new_sum_problem.f([1, 2]), 3)
# Test using `from_botorch`.
ackley_problem = SimpleBenchmarkProblem(f=from_botorch(Ackley()),
noise_sd=0.0,
minimize=True)
new_ackley_problem = object_from_json(object_to_json(ackley_problem))
self.assertEqual(ackley_problem.f(1, 2), new_ackley_problem.f(1, 2),
ackley(1, 2))
def testEncodeDecode(self):
for class_, fake_func in TEST_CASES:
# Can't load trials from JSON, because a batch needs an experiment
# in order to be initialized
if class_ == "BatchTrial" or class_ == "Trial":
continue
# Can't load parameter constraints from JSON, because they require
# a SearchSpace in order to be initialized
if class_ == "OrderConstraint" or class_ == "SumConstraint":
continue
original_object = fake_func()
json_object = object_to_json(original_object)
converted_object = object_from_json(json_object)
if class_ == "SimpleExperiment":
# Evaluation functions will be different, so need to do
# this so equality test passes
with self.assertRaises(Exception):
converted_object.evaluation_function()
original_object.evaluation_function = None
converted_object.evaluation_function = None
self.assertEqual(
original_object,
converted_object,
msg=f"Error encoding/decoding {class_}.",
)
def compile_nasbench():
all_res = {}
# TuRBO and CMAES
for method in ['turbo', 'cmaes']:
all_res[method] = []
for rep in range(100):
with open(f'results/nasbench_{method}_rep_{rep}.json', 'r') as fin:
fs, feas = json.load(fin)
# Set infeasible points to nan
fs = np.array(fs)
fs[~np.array(feas)] = np.nan
all_res[method].append(fs)
# Ax methods
for method in ['Sobol', 'ALEBO', 'HeSBO', 'REMBO']:
all_res[method] = []
for rep in range(100):
with open(f'results/nasbench_{method}_rep_{rep}.json', 'r') as fin:
exp = object_from_json(json.load(fin))
# Pull out results and set infeasible points to nan
df = exp.fetch_data().df.sort_values(by='arm_name')
df_obj = df[df['metric_name'] == 'final_test_accuracy'].copy().reset_index(drop=True)
df_con = df[df['metric_name'] == 'final_training_time'].copy().reset_index(drop=True)
infeas = df_con['mean'] > 1800
df_obj.loc[infeas, 'mean'] = np.nan
all_res[method].append(df_obj['mean'].values)
for method, arr in all_res.items():
all_res[method] = np.fmax.accumulate(np.vstack(all_res[method]), axis=1)
with open(f'results/nasbench_aggregated_results.json', "w") as fout:
json.dump(object_to_json(all_res), fout)
def generation_strategy_from_sqa(
self, gs_sqa: SQAGenerationStrategy) -> GenerationStrategy:
"""Convert SQALchemy generation strategy to Ax `GenerationStrategy`."""
steps = object_from_json(gs_sqa.steps)
gs = GenerationStrategy(name=gs_sqa.name, steps=steps)
gs._curr = gs._steps[gs_sqa.curr_index]
gs._generator_runs = [
self.generator_run_from_sqa(gr) for gr in gs_sqa.generator_runs
]
if len(gs._generator_runs) > 0:
# Generation strategy had an initialized model.
# pyre-ignore[16]: SQAGenerationStrategy does not have `experiment` attr.
gs._experiment = self.experiment_from_sqa(gs_sqa.experiment)
# If model in the current step was not directly from the `Models` enum,
# pass its type to `restore_model_from_generator_run`, which will then
# attempt to use this type to recreate the model.
if type(gs._curr.model) != Models:
models_enum = type(gs._curr.model)
assert issubclass(models_enum, Models)
# pyre-ignore[6]: `models_enum` typing hackiness
gs._restore_model_from_generator_run(models_enum=models_enum)
else:
gs._restore_model_from_generator_run()
gs._db_id = gs_sqa.id
return gs
def load_object(self, object_ax_type):
object_dict = self.connector.extract_object(object_ax_type,
self.test_name)
if object_ax_type == 'GenerationStrategy':
object_dict['had_initialized_model'] = False
object_ax = object_from_json(object_dict)
return object_ax
def extract_sensitivity_results():
res = {}
for fname in [
'branin_gramacy_100',
'sensitivity',
]:
with open(f'../benchmarks/results/{fname}_aggregated_results.json',
'r') as fin:
res.update(object_from_json(json.load(fin)))
# Results for D=100
ys1 = {}
for d in [2, 3, 4, 5, 6, 7, 8]:
if d == 4:
ys1[d] = res['Branin, D=100'].objective_at_true_best['ALEBO']
else:
ys1[d] = res['Branin, D=100_sensitivity'].objective_at_true_best[
f'ALEBO, d={d}']
# Results for d_e=4
ys2 = {}
for D in [50, 100, 200, 500, 1000]:
if D == 100:
ys2[D] = res['Branin, D=100'].objective_at_true_best['ALEBO']
else:
ys2[D] = res[f'Branin, D={D}_sensitivity'].objective_at_true_best[
'ALEBO']
return ys1, ys2
def generation_strategy_from_sqa(
self,
gs_sqa: SQAGenerationStrategy,
experiment: Optional[Experiment] = None,
reduced_state: bool = False,
) -> GenerationStrategy:
"""Convert SQALchemy generation strategy to Ax `GenerationStrategy`."""
steps = object_from_json(
gs_sqa.steps,
decoder_registry=self.config.json_decoder_registry,
class_decoder_registry=self.config.json_class_decoder_registry,
)
gs = GenerationStrategy(name=gs_sqa.name, steps=steps)
gs._curr = gs._steps[gs_sqa.curr_index]
immutable_ss_and_oc = (experiment.immutable_search_space_and_opt_config
if experiment is not None else False)
if reduced_state and gs_sqa.generator_runs:
# Only fully load the last of the generator runs, load the rest with
# reduced state.
gs._generator_runs = [
self.generator_run_from_sqa(
generator_run_sqa=gr,
reduced_state=True,
immutable_search_space_and_opt_config=immutable_ss_and_oc,
) for gr in gs_sqa.generator_runs[:-1]
]
gs._generator_runs.append(
self.generator_run_from_sqa(
generator_run_sqa=gs_sqa.generator_runs[-1],
reduced_state=False,
immutable_search_space_and_opt_config=immutable_ss_and_oc,
))
else:
gs._generator_runs = [
self.generator_run_from_sqa(
generator_run_sqa=gr,
reduced_state=False,
immutable_search_space_and_opt_config=immutable_ss_and_oc,
) for gr in gs_sqa.generator_runs
]
if len(gs._generator_runs) > 0:
# Generation strategy had an initialized model.
if experiment is None:
raise SQADecodeError(
"Cannot decode a generation strategy with a non-zero number of "
"generator runs without an experiment.")
gs._experiment = experiment
# If model in the current step was not directly from the `Models` enum,
# pass its type to `restore_model_from_generator_run`, which will then
# attempt to use this type to recreate the model.
if type(gs._curr.model) != Models:
models_enum = type(gs._curr.model)
assert issubclass(models_enum, ModelRegistryBase)
# pyre-ignore[6]: `models_enum` typing hackiness
gs._restore_model_from_generator_run(models_enum=models_enum)
else:
gs._restore_model_from_generator_run()
gs.db_id = gs_sqa.id
return gs
def load_object(self, object_ax_type):
filepath = self.fp_main + self.fp_suffix_dict[object_ax_type]
with open(filepath, "r") as file:
object_dict = json.loads(file.read())
if object_ax_type == 'GenerationStrategy':
object_dict['had_initialized_model'] = False
object_ax = object_from_json(object_dict)
return object_ax
def from_json_snapshot(serialized):
"""Recreate a `CoreAxClient` from a JSON snapshot."""
ax_client = CoreAxClient(
experiment=object_from_json(serialized.pop("experiment")),
generation_strategy=generation_strategy_from_json(
generation_strategy_json=serialized.pop(
"generation_strategy")))
return ax_client
def compile_hartmann6(D, random_subspace=False):
if D == 100:
problem = hartmann6_100
other_methods = ['addgpucb', 'cmaes', 'ebo', 'smac', 'turbo']
rls = ['rrembos_standard_kPsi', 'rrembos_reverse_kPsi', 'coordinatelinebo', 'descentlinebo', 'randomlinebo']
rs_str = ''
elif D == 1000 and not random_subspace:
problem = hartmann6_1000
other_methods = ['cmaes', 'smac', 'turbo']
rls = ['rrembos_standard_kPsi']
rs_str = ''
elif D == 1000 and random_subspace:
problem = hartmann6_random_subspace_1000
other_methods = ['cmaes', 'smac', 'turbo']
rls = []
rs_str = 'random_subspace_'
all_results = {}
for rep in range(50):
with open(f'results/hartmann6_{rs_str}{D}_alebo_rembo_hesbo_sobol_rep_{rep}.json', 'r') as fin:
res_i = object_from_json(json.load(fin))
all_results = merge_benchmark_results(all_results, res_i)
for method_name in other_methods:
if D==1000 and method_name == 'smac' and rep > 9:
# SMAC D=1000 only run for 10 reps
continue
with open(f'results/hartmann6_{rs_str}{D}_{method_name}_rep_{rep}.json', 'r') as fin:
exp_i = object_from_json(json.load(fin))
all_results = add_exp(res=all_results, exp=exp_i, problem_name=problem.name, method_name=method_name)
res = aggregate_problem_results(runs=all_results[problem.name], problem=problem)
# Add in RRembo and LineBOresults
for method in rls:
with open(f'results/hartmann6_{D}_{method}.json', 'r') as fin:
A = json.load(fin)
res.objective_at_true_best[method] = np.minimum.accumulate(np.array(A), axis=1)
# Save
with open(f'results/hartmann6_{rs_str}{D}_aggregated_results.json', "w") as fout:
json.dump(object_to_json({problem.name: res}), fout)
def load_experiment(filepath: str) -> Experiment:
"""Load experiment from file.
1) Read file.
2) Convert dictionary to Ax experiment instance.
"""
with open(filepath, "r") as file:
json_experiment = json.loads(file.read())
return object_from_json(json_experiment)
def testEncodeDecodeSimpleBenchmarkProblem(self):
branin_problem = get_branin_simple_benchmark_problem()
sum_problem = get_sum_simple_benchmark_problem()
new_branin_problem = object_from_json(
object_to_json(
branin_problem,
encoder_registry=DEPRECATED_ENCODER_REGISTRY,
class_encoder_registry=DEPRECATED_CLASS_ENCODER_REGISTRY,
),
decoder_registry=DEPRECATED_DECODER_REGISTRY,
class_decoder_registry=DEPRECATED_CLASS_DECODER_REGISTRY,
)
new_sum_problem = object_from_json(
object_to_json(
sum_problem,
encoder_registry=DEPRECATED_ENCODER_REGISTRY,
class_encoder_registry=DEPRECATED_CLASS_ENCODER_REGISTRY,
),
decoder_registry=DEPRECATED_DECODER_REGISTRY,
class_decoder_registry=DEPRECATED_CLASS_DECODER_REGISTRY,
)
self.assertEqual(
branin_problem.f(1, 2), new_branin_problem.f(1, 2), branin(1, 2)
)
self.assertEqual(sum_problem.f([1, 2]), new_sum_problem.f([1, 2]), 3)
# Test using `from_botorch`.
ackley_problem = SimpleBenchmarkProblem(
f=from_botorch(Ackley()), noise_sd=0.0, minimize=True
)
new_ackley_problem = object_from_json(
object_to_json(
ackley_problem,
encoder_registry=DEPRECATED_ENCODER_REGISTRY,
class_encoder_registry=DEPRECATED_CLASS_ENCODER_REGISTRY,
),
decoder_registry=DEPRECATED_DECODER_REGISTRY,
class_decoder_registry=DEPRECATED_CLASS_DECODER_REGISTRY,
)
self.assertEqual(
ackley_problem.f(1, 2), new_ackley_problem.f(1, 2), ackley(1, 2)
)
def from_json_snapshot(serialized: Dict[str, Any], **kwargs) -> "AxClient":
"""Recreate an `AxClient` from a JSON snapshot."""
experiment = object_from_json(serialized.pop("experiment"))
serialized_generation_strategy = serialized.pop("generation_strategy")
ax_client = AxClient(
generation_strategy=generation_strategy_from_json(
generation_strategy_json=serialized_generation_strategy)
if serialized_generation_strategy is not None else None,
enforce_sequential_optimization=serialized.pop(
"_enforce_sequential_optimization"),
**kwargs,
)
ax_client._experiment = experiment
return ax_client
def table_S1_data():
with open('../benchmarks/results/all_aggregated_results.json', 'r') as fin:
res = object_from_json(json.load(fin))
for D in [100, 1000]:
pname = f'Hartmann6, D={D}'
print('-----', pname)
for m, ts in res[pname].gen_times.items():
# Get average total time for fit and gen
t = np.mean(ts)
t += np.mean(res[pname].fit_times[m])
# Divide by 200 to be time per iteration
t /= 200.
print(f'{m}: {t}')
def metric_from_sqa_util(self, metric_sqa: SQAMetric) -> Metric:
"""Convert SQLAlchemy Metric to Ax Metric"""
metric_class = REVERSE_METRIC_REGISTRY.get(metric_sqa.metric_type)
if metric_class is None:
raise SQADecodeError(
f"Cannot decode SQAMetric because {metric_sqa.metric_type} "
f"is an invalid type.")
args = dict(object_from_json(metric_sqa.properties) or {})
args["name"] = metric_sqa.name
args["lower_is_better"] = metric_sqa.lower_is_better
args = metric_class.deserialize_init_args(args=args)
metric = metric_class(**args)
metric.db_id = metric_sqa.id
return metric
def testEncodeDecodeTorchTensor(self):
x = torch.tensor(
[[1.0, 2.0], [3.0, 4.0]], dtype=torch.float64, device=torch.device("cpu")
)
expected_json = {
"__type": "Tensor",
"value": [[1.0, 2.0], [3.0, 4.0]],
"dtype": {"__type": "torch_dtype", "value": "torch.float64"},
"device": {"__type": "torch_device", "value": "cpu"},
}
x_json = object_to_json(x)
self.assertEqual(expected_json, x_json)
x2 = object_from_json(x_json)
self.assertTrue(torch.equal(x, x2))
def load_experiment(
filepath: str,
decoder_registry: Dict[str, Type],
class_decoder_registry: Dict[str, Callable[[Dict[str, Any]], Any]],
) -> Experiment:
"""Load experiment from file.
1) Read file.
2) Convert dictionary to Ax experiment instance.
"""
with open(filepath, "r") as file:
json_experiment = json.loads(file.read())
return object_from_json(
json_experiment, decoder_registry, class_decoder_registry
)
def generation_strategy_from_sqa(
self, gs_sqa: SQAGenerationStrategy) -> GenerationStrategy:
"""Convert SQALchemy generation strategy to Ax `GenerationStrategy`."""
steps = object_from_json(gs_sqa.steps)
gs = GenerationStrategy(name=gs_sqa.name, steps=steps)
gs._curr = gs._steps[gs_sqa.curr_index]
gs._generator_runs = [
self.generator_run_from_sqa(gr) for gr in gs_sqa.generator_runs
]
if len(gs._generator_runs) > 0:
# Generation strategy had an initialized model.
# pyre-ignore[16]: SQAGenerationStrategy does not have `experiment` attr.
gs._experiment = self.experiment_from_sqa(gs_sqa.experiment)
gs._restore_model_from_generator_run()
gs._db_id = gs_sqa.id
return gs
def testEncodeDecodeNumpy(self):
arr = np.array([[1, 2, 3], [4, 5, 6]])
self.assertTrue(
np.array_equal(
arr,
object_from_json(
object_to_json(
arr,
encoder_registry=DEPRECATED_ENCODER_REGISTRY,
class_encoder_registry=DEPRECATED_CLASS_ENCODER_REGISTRY,
),
decoder_registry=DEPRECATED_DECODER_REGISTRY,
class_decoder_registry=DEPRECATED_CLASS_DECODER_REGISTRY,
),
)
)
def compile_ablation_benchmarks():
all_results = {}
for rep in range(100):
with open(f'results/ablation_rep_{rep}.json', 'r') as fin:
res_i = object_from_json(json.load(fin))
all_results = merge_benchmark_results(all_results, res_i)
problems = [branin_100]
res = {
p.name+'_ablation': aggregate_problem_results(runs=all_results[p.name], problem=p)
for p in problems
}
# Save
with open(f'results/ablation_aggregated_results.json', "w") as fout:
json.dump(object_to_json(res), fout)
def extract_ablation_results():
with open(f'../benchmarks/results/ablation_aggregated_results.json',
'r') as fin:
res = object_from_json(json.load(fin))
# Results
ys = {
'ALEBO':
res['Branin, D=100_ablation'].objective_at_true_best['ALEBO, base'],
'Ablation: Matern kernel':
res['Branin, D=100_ablation'].
objective_at_true_best['ALEBO, kernel ablation'],
'Ablation: Normal projection':
res['Branin, D=100_ablation'].
objective_at_true_best['ALEBO, projection ablation'],
}
return ys
def make_nasbench_figure():
with open('../benchmarks/results/nasbench_aggregated_results.json',
'r') as fin:
res = object_from_json(json.load(fin))
# A map from method idx in plot_method_names to the name used in res
method_idx_to_res_name = {
1: 'REMBO',
3: 'HeSBO',
9: 'cmaes',
10: 'turbo',
11: 'Sobol',
0: 'ALEBO',
}
plot_method_names[3] = 'HeSBO'
plot_colors['HeSBO'] = plt.cm.tab20(3)
fig = plt.figure(figsize=(2.96, 2.0))
ax1 = fig.add_subplot(111)
method_idx = 1
method_names_used = []
for i, m in method_idx_to_res_name.items():
f = np.nanmean(res[m], axis=0)
sem = np.nanstd(res[m], axis=0) / np.sqrt(res[m].shape[0])
x = np.arange(1, 51)
mname = plot_method_names[i]
color = plot_colors[mname]
ax1.plot(x, f, color=color, label=mname)
ax1.errorbar(x, f, yerr=2 * sem, color=color, alpha=0.5, ls='')
ax1.set_xlim([0, 51])
ax1.set_ylabel('Best feasible test accuracy', fontsize=9)
ax1.set_xlabel('Function evaluations', fontsize=9)
#ax1.legend(bbox_to_anchor=(1.0, 1.24), ncol=4, fontsize=6, columnspacing=1.65)
ax1.legend(ncol=2, loc='lower right', fontsize=7)
ax1.set_ylim([0.92, 0.936])
ax1.set_yticks([0.92, 0.925, 0.93, 0.935])
ax1.set_yticklabels(['92.0\%', '92.5\%', '93.0\%', '93.5\%'])
ax1.grid(alpha=0.2, zorder=-10)
plt.subplots_adjust(right=0.98, bottom=0.17, left=0.19, top=0.98)
plt.savefig(f'pdfs/nas.pdf', pad_inches=0)
def testEncodeDecodeTorchTensor(self):
x = torch.tensor(
[[1.0, 2.0], [3.0, 4.0]], dtype=torch.float64, device=torch.device("cpu")
)
expected_json = {
"__type": "Tensor",
"value": [[1.0, 2.0], [3.0, 4.0]],
"dtype": {"__type": "torch_dtype", "value": "torch.float64"},
"device": {"__type": "torch_device", "value": "cpu"},
}
x_json = object_to_json(
x,
encoder_registry=DEPRECATED_ENCODER_REGISTRY,
class_encoder_registry=DEPRECATED_CLASS_ENCODER_REGISTRY,
)
self.assertEqual(expected_json, x_json)
x2 = object_from_json(
x_json,
decoder_registry=DEPRECATED_DECODER_REGISTRY,
class_decoder_registry=DEPRECATED_CLASS_DECODER_REGISTRY,
)
self.assertTrue(torch.equal(x, x2))
def testEncodeDecode(self):
for class_, fake_func in TEST_CASES:
# Can't load trials from JSON, because a batch needs an experiment
# in order to be initialized
if class_ == "BatchTrial" or class_ == "Trial":
continue
# Can't load parameter constraints from JSON, because they require
# a SearchSpace in order to be initialized
if class_ == "OrderConstraint" or class_ == "SumConstraint":
continue
original_object = fake_func()
# Can't load generation strategy from JSON, because it requires an
# experiment to reload.
if class_ == "GenerationStrategy":
with self.assertRaisesRegex(ValueError,
".* custom .* cannot "):
original_object._uses_registered_models = False
object_to_json(original_object)
continue
json_object = object_to_json(original_object)
converted_object = object_from_json(json_object)
if class_ == "SimpleExperiment":
# Evaluation functions will be different, so need to do
# this so equality test passes
with self.assertRaises(Exception):
converted_object.evaluation_function()
original_object.evaluation_function = None
converted_object.evaluation_function = None
self.assertEqual(
original_object,
converted_object,
msg=f"Error encoding/decoding {class_}.",
)
def test_encode_decode_numpy(self):
arr = np.array([[1, 2, 3], [4, 5, 6]])
self.assertTrue(np.array_equal(arr, object_from_json(object_to_json(arr))))
def make_fig_5():
# Load in the benchmark results
res = {}
for fname in ['hartmann6_1000', 'branin_gramacy_100']:
with open(f'../benchmarks/results/{fname}_aggregated_results.json',
'r') as fin:
res.update(object_from_json(json.load(fin)))
# A map from method idx in plot_method_names to the name used in res
method_idx_to_res_name = {
0: 'ALEBO',
1: 'REMBO',
2: 'HeSBO, d=d',
3: 'HeSBO, d=2d',
4: 'rrembos_standard_kPsi',
5: 'rrembos_reverse_kPsi',
6: 'ebo',
7: 'addgpucb',
8: 'smac',
9: 'cmaes',
10: 'turbo',
11: 'Sobol',
12: 'coordinatelinebo',
13: 'randomlinebo',
14: 'descentlinebo',
}
# Make the figure
fig = plt.figure(figsize=(5.5, 3.7))
####### Branin, D=100
ax1 = fig.add_subplot(231)
ax2 = fig.add_subplot(234)
res_h = res['Branin, D=100']
for idx, m in enumerate(plot_method_names):
res_name = method_idx_to_res_name[idx]
if res_name not in res_h.objective_at_true_best:
continue # Not run on this problem
Y = res_h.objective_at_true_best[res_name]
f = Y.mean(axis=0)
x = np.arange(1, 51)
color = plot_colors[m]
ax1.plot(x, f, color=color, label=m)
parts = ax2.violinplot(positions=[idx],
dataset=Y[:, 49],
showmeans=True)
for pc in parts['bodies']:
pc.set_facecolor(color)
pc.set_edgecolor(color)
for field in ['cmeans', 'cmaxes', 'cmins', 'cbars']:
parts[field].set_color(color)
ax1.set_xlim([0, 51])
ax1.set_ylabel('Best value found', fontsize=7)
ax1.set_xlabel('Function evaluations', fontsize=7)
ax1.axhline(y=0.397887, c='gray', ls='--')
ax1.grid(alpha=0.2, zorder=-10)
ax1.set_ylim([0, 7])
ax2.set_xticks(range(12))
ax2.set_xticklabels([])
ax2.set_ylabel('Final value', fontsize=7)
ax2.grid(alpha=0.2, zorder=-10)
ax2.set_xticklabels([plot_method_names[i] for i in range(12)], fontsize=6)
ax2.xaxis.set_tick_params(rotation=90)
# Make the legend
custom_lines = []
names = []
for i in range(12):
names.append(plot_method_names[i])
custom_lines.append(
Line2D([0], [0], color=plot_colors[plot_method_names[i]], lw=2))
order = range(12)
names = [names[o] for o in order]
custom_lines = [custom_lines[o] for o in order]
ax1.legend(custom_lines,
names,
ncol=6,
fontsize=5.5,
bbox_to_anchor=(3.52, -2.26))
ax1.set_title('Branin, $d$=2, $D$=100', fontsize=8)
####### Hartmann6, D=1000
ax1 = fig.add_subplot(232)
ax2 = fig.add_subplot(235)
res_h = res['Hartmann6, D=1000']
for idx, m in enumerate(plot_method_names):
res_name = method_idx_to_res_name[idx]
if res_name not in res_h.objective_at_true_best:
continue # Not run on this problem
Y = res_h.objective_at_true_best[res_name]
f = Y.mean(axis=0)
x = np.arange(1, 201)
color = plot_colors[m]
ax1.plot(x, f, color=color, label=m)
parts = ax2.violinplot(positions=[idx],
dataset=Y[:, 199],
showmeans=True)
for pc in parts['bodies']:
pc.set_facecolor(color)
pc.set_edgecolor(color)
for field in ['cmeans', 'cmaxes', 'cmins', 'cbars']:
parts[field].set_color(color)
ax1.set_xlim([0, 201])
#ax1.set_ylabel('Best value found', fontsize=9)
ax1.set_xlabel('Function evaluations', fontsize=7)
ax1.axhline(y=-3.32237, c='gray', ls='--')
ax1.grid(alpha=0.2, zorder=-10)
ax1.set_ylim([-3.5, -0.5])
ax2.set_xticks(range(12))
ax2.set_xticklabels([])
#ax2.set_ylabel('Final value', fontsize=9)
ax2.grid(alpha=0.2, zorder=-10)
ax2.set_xticklabels([plot_method_names[i] for i in range(12)], fontsize=6)
ax2.xaxis.set_tick_params(rotation=90)
ax1.set_title('Hartmann6, $d$=6, $D$=1000', fontsize=8)
####### Gramacy, D=100
ax1 = fig.add_subplot(233)
ax2 = fig.add_subplot(236)
res_h = res['Gramacy, D=100']
for idx, m in enumerate(plot_method_names):
res_name = method_idx_to_res_name[idx]
if res_name not in res_h.objective_at_true_best:
continue # Not run on this problem
Y = res_h.objective_at_true_best[res_name]
f = Y.mean(axis=0)
x = np.arange(1, 51)
color = plot_colors[m]
ax1.plot(x, f, color=color, label=m)
parts = ax2.violinplot(positions=[idx],
dataset=Y[:, 49],
showmeans=True)
for pc in parts['bodies']:
pc.set_facecolor(color)
pc.set_edgecolor(color)
for field in ['cmeans', 'cmaxes', 'cmins', 'cbars']:
parts[field].set_color(color)
ax1.set_xlim([0, 51])
#ax1.set_ylabel('Best value found', fontsize=9)
ax1.set_xlabel('Function evaluations', fontsize=7)
ax1.set_ylim([0.58, 1])
ax1.axhline(y=0.5998, c='gray', ls='--')
ax1.grid(alpha=0.2, zorder=-10)
ax2.set_xticks(range(12))
ax2.set_xticklabels([plot_method_names[i] for i in range(12)], fontsize=6)
ax2.xaxis.set_tick_params(rotation=90)
#ax2.set_ylabel('Final value', fontsize=9)
ax2.grid(alpha=0.2, zorder=-10)
ax1.set_title('Gramacy, $d$=2, $D$=100', fontsize=8)
plt.subplots_adjust(right=0.995,
bottom=0.3,
left=0.07,
top=0.94,
wspace=0.25,
hspace=0.45)
plt.savefig('pdfs/benchmark_results_t.pdf', pad_inches=0)