def compile_sensitivity_benchmarks(): all_results = {} for rep in range(50): ## Sensitivity to D with open(f'results/sensitivity_D_rep_{rep}.json', 'r') as fin: results_dict = json.load(fin) for D, obj in results_dict.items(): res_i = object_from_json(obj) all_results = merge_benchmark_results(all_results, res_i) ## Sensitivity to d_e with open(f'results/sensitivity_d_e_rep_{rep}.json', 'r') as fin: res_i = object_from_json(json.load(fin)) all_results = merge_benchmark_results(all_results, res_i) all_problems = ( [hartmann6_100, hartmann6_1000, branin_100, gramacy_100] + list(branin_by_D.values()) ) problems = [branin_100] + list(branin_by_D.values()) res = { p.name+'_sensitivity': aggregate_problem_results(runs=all_results[p.name], problem=p) for p in problems } # Save with open(f'results/sensitivity_aggregated_results.json', "w") as fout: json.dump(object_to_json(res), fout)
def compile_branin_gramacy_100(): all_results = {} for rep in range(50): with open(f'results/branin_gramacy_100_alebo_rembo_hesbo_sobol_rep_{rep}.json', 'r') as fin: res_i = object_from_json(json.load(fin)) all_results = merge_benchmark_results(all_results, res_i) for method_name in ['addgpucb', 'cmaes', 'ebo', 'smac', 'turbo']: with open(f'results/branin_100_{method_name}_rep_{rep}.json', 'r') as fin: exp_i = object_from_json(json.load(fin)) all_results = add_exp( res=all_results, exp=exp_i, problem_name='Branin, D=100', method_name=method_name, ) res = { p.name: aggregate_problem_results(runs=all_results[p.name], problem=p) for p in [branin_100, gramacy_100] } # Add in RRembo results for proj in ['standard', 'reverse']: method = f'rrembos_{proj}_kPsi' with open(f'results/branin_100_{method}.json', 'r') as fin: A = json.load(fin) res['Branin, D=100'].objective_at_true_best[method] = np.minimum.accumulate(np.array(A), axis=1) # Save with open(f'results/branin_gramacy_100_aggregated_results.json', "w") as fout: json.dump(object_to_json(res), fout)
def testEncodeDecodeSimpleBenchmarkProblem(self): branin_problem = get_branin_simple_benchmark_problem() sum_problem = get_sum_simple_benchmark_problem() new_branin_problem = object_from_json(object_to_json(branin_problem)) new_sum_problem = object_from_json(object_to_json(sum_problem)) self.assertEqual(branin_problem.f(1, 2), new_branin_problem.f(1, 2), branin(1, 2)) self.assertEqual(sum_problem.f([1, 2]), new_sum_problem.f([1, 2]), 3)
def from_json_snapshot(serialized: Dict[str, Any]) -> "AxClient": """Recreate an `AxClient` from a JSON snapshot.""" experiment = object_from_json(serialized.pop("experiment")) ax_client = AxClient( generation_strategy=generation_strategy_from_json( generation_strategy_json=serialized.pop( "generation_strategy")), enforce_sequential_optimization=serialized.pop( "_enforce_sequential_optimization"), ) ax_client._experiment = experiment ax_client._updated_trials = object_from_json( serialized.pop("_updated_trials")) return ax_client
def testEncodeDecodeSimpleBenchmarkProblem(self): branin_problem = get_branin_simple_benchmark_problem() sum_problem = get_sum_simple_benchmark_problem() new_branin_problem = object_from_json(object_to_json(branin_problem)) new_sum_problem = object_from_json(object_to_json(sum_problem)) self.assertEqual(branin_problem.f(1, 2), new_branin_problem.f(1, 2), branin(1, 2)) self.assertEqual(sum_problem.f([1, 2]), new_sum_problem.f([1, 2]), 3) # Test using `from_botorch`. ackley_problem = SimpleBenchmarkProblem(f=from_botorch(Ackley()), noise_sd=0.0, minimize=True) new_ackley_problem = object_from_json(object_to_json(ackley_problem)) self.assertEqual(ackley_problem.f(1, 2), new_ackley_problem.f(1, 2), ackley(1, 2))
def testEncodeDecode(self): for class_, fake_func in TEST_CASES: # Can't load trials from JSON, because a batch needs an experiment # in order to be initialized if class_ == "BatchTrial" or class_ == "Trial": continue # Can't load parameter constraints from JSON, because they require # a SearchSpace in order to be initialized if class_ == "OrderConstraint" or class_ == "SumConstraint": continue original_object = fake_func() json_object = object_to_json(original_object) converted_object = object_from_json(json_object) if class_ == "SimpleExperiment": # Evaluation functions will be different, so need to do # this so equality test passes with self.assertRaises(Exception): converted_object.evaluation_function() original_object.evaluation_function = None converted_object.evaluation_function = None self.assertEqual( original_object, converted_object, msg=f"Error encoding/decoding {class_}.", )
def compile_nasbench(): all_res = {} # TuRBO and CMAES for method in ['turbo', 'cmaes']: all_res[method] = [] for rep in range(100): with open(f'results/nasbench_{method}_rep_{rep}.json', 'r') as fin: fs, feas = json.load(fin) # Set infeasible points to nan fs = np.array(fs) fs[~np.array(feas)] = np.nan all_res[method].append(fs) # Ax methods for method in ['Sobol', 'ALEBO', 'HeSBO', 'REMBO']: all_res[method] = [] for rep in range(100): with open(f'results/nasbench_{method}_rep_{rep}.json', 'r') as fin: exp = object_from_json(json.load(fin)) # Pull out results and set infeasible points to nan df = exp.fetch_data().df.sort_values(by='arm_name') df_obj = df[df['metric_name'] == 'final_test_accuracy'].copy().reset_index(drop=True) df_con = df[df['metric_name'] == 'final_training_time'].copy().reset_index(drop=True) infeas = df_con['mean'] > 1800 df_obj.loc[infeas, 'mean'] = np.nan all_res[method].append(df_obj['mean'].values) for method, arr in all_res.items(): all_res[method] = np.fmax.accumulate(np.vstack(all_res[method]), axis=1) with open(f'results/nasbench_aggregated_results.json', "w") as fout: json.dump(object_to_json(all_res), fout)
def generation_strategy_from_sqa( self, gs_sqa: SQAGenerationStrategy) -> GenerationStrategy: """Convert SQALchemy generation strategy to Ax `GenerationStrategy`.""" steps = object_from_json(gs_sqa.steps) gs = GenerationStrategy(name=gs_sqa.name, steps=steps) gs._curr = gs._steps[gs_sqa.curr_index] gs._generator_runs = [ self.generator_run_from_sqa(gr) for gr in gs_sqa.generator_runs ] if len(gs._generator_runs) > 0: # Generation strategy had an initialized model. # pyre-ignore[16]: SQAGenerationStrategy does not have `experiment` attr. gs._experiment = self.experiment_from_sqa(gs_sqa.experiment) # If model in the current step was not directly from the `Models` enum, # pass its type to `restore_model_from_generator_run`, which will then # attempt to use this type to recreate the model. if type(gs._curr.model) != Models: models_enum = type(gs._curr.model) assert issubclass(models_enum, Models) # pyre-ignore[6]: `models_enum` typing hackiness gs._restore_model_from_generator_run(models_enum=models_enum) else: gs._restore_model_from_generator_run() gs._db_id = gs_sqa.id return gs
def load_object(self, object_ax_type): object_dict = self.connector.extract_object(object_ax_type, self.test_name) if object_ax_type == 'GenerationStrategy': object_dict['had_initialized_model'] = False object_ax = object_from_json(object_dict) return object_ax
def extract_sensitivity_results(): res = {} for fname in [ 'branin_gramacy_100', 'sensitivity', ]: with open(f'../benchmarks/results/{fname}_aggregated_results.json', 'r') as fin: res.update(object_from_json(json.load(fin))) # Results for D=100 ys1 = {} for d in [2, 3, 4, 5, 6, 7, 8]: if d == 4: ys1[d] = res['Branin, D=100'].objective_at_true_best['ALEBO'] else: ys1[d] = res['Branin, D=100_sensitivity'].objective_at_true_best[ f'ALEBO, d={d}'] # Results for d_e=4 ys2 = {} for D in [50, 100, 200, 500, 1000]: if D == 100: ys2[D] = res['Branin, D=100'].objective_at_true_best['ALEBO'] else: ys2[D] = res[f'Branin, D={D}_sensitivity'].objective_at_true_best[ 'ALEBO'] return ys1, ys2
def generation_strategy_from_sqa( self, gs_sqa: SQAGenerationStrategy, experiment: Optional[Experiment] = None, reduced_state: bool = False, ) -> GenerationStrategy: """Convert SQALchemy generation strategy to Ax `GenerationStrategy`.""" steps = object_from_json( gs_sqa.steps, decoder_registry=self.config.json_decoder_registry, class_decoder_registry=self.config.json_class_decoder_registry, ) gs = GenerationStrategy(name=gs_sqa.name, steps=steps) gs._curr = gs._steps[gs_sqa.curr_index] immutable_ss_and_oc = (experiment.immutable_search_space_and_opt_config if experiment is not None else False) if reduced_state and gs_sqa.generator_runs: # Only fully load the last of the generator runs, load the rest with # reduced state. gs._generator_runs = [ self.generator_run_from_sqa( generator_run_sqa=gr, reduced_state=True, immutable_search_space_and_opt_config=immutable_ss_and_oc, ) for gr in gs_sqa.generator_runs[:-1] ] gs._generator_runs.append( self.generator_run_from_sqa( generator_run_sqa=gs_sqa.generator_runs[-1], reduced_state=False, immutable_search_space_and_opt_config=immutable_ss_and_oc, )) else: gs._generator_runs = [ self.generator_run_from_sqa( generator_run_sqa=gr, reduced_state=False, immutable_search_space_and_opt_config=immutable_ss_and_oc, ) for gr in gs_sqa.generator_runs ] if len(gs._generator_runs) > 0: # Generation strategy had an initialized model. if experiment is None: raise SQADecodeError( "Cannot decode a generation strategy with a non-zero number of " "generator runs without an experiment.") gs._experiment = experiment # If model in the current step was not directly from the `Models` enum, # pass its type to `restore_model_from_generator_run`, which will then # attempt to use this type to recreate the model. if type(gs._curr.model) != Models: models_enum = type(gs._curr.model) assert issubclass(models_enum, ModelRegistryBase) # pyre-ignore[6]: `models_enum` typing hackiness gs._restore_model_from_generator_run(models_enum=models_enum) else: gs._restore_model_from_generator_run() gs.db_id = gs_sqa.id return gs
def load_object(self, object_ax_type): filepath = self.fp_main + self.fp_suffix_dict[object_ax_type] with open(filepath, "r") as file: object_dict = json.loads(file.read()) if object_ax_type == 'GenerationStrategy': object_dict['had_initialized_model'] = False object_ax = object_from_json(object_dict) return object_ax
def from_json_snapshot(serialized): """Recreate a `CoreAxClient` from a JSON snapshot.""" ax_client = CoreAxClient( experiment=object_from_json(serialized.pop("experiment")), generation_strategy=generation_strategy_from_json( generation_strategy_json=serialized.pop( "generation_strategy"))) return ax_client
def compile_hartmann6(D, random_subspace=False): if D == 100: problem = hartmann6_100 other_methods = ['addgpucb', 'cmaes', 'ebo', 'smac', 'turbo'] rls = ['rrembos_standard_kPsi', 'rrembos_reverse_kPsi', 'coordinatelinebo', 'descentlinebo', 'randomlinebo'] rs_str = '' elif D == 1000 and not random_subspace: problem = hartmann6_1000 other_methods = ['cmaes', 'smac', 'turbo'] rls = ['rrembos_standard_kPsi'] rs_str = '' elif D == 1000 and random_subspace: problem = hartmann6_random_subspace_1000 other_methods = ['cmaes', 'smac', 'turbo'] rls = [] rs_str = 'random_subspace_' all_results = {} for rep in range(50): with open(f'results/hartmann6_{rs_str}{D}_alebo_rembo_hesbo_sobol_rep_{rep}.json', 'r') as fin: res_i = object_from_json(json.load(fin)) all_results = merge_benchmark_results(all_results, res_i) for method_name in other_methods: if D==1000 and method_name == 'smac' and rep > 9: # SMAC D=1000 only run for 10 reps continue with open(f'results/hartmann6_{rs_str}{D}_{method_name}_rep_{rep}.json', 'r') as fin: exp_i = object_from_json(json.load(fin)) all_results = add_exp(res=all_results, exp=exp_i, problem_name=problem.name, method_name=method_name) res = aggregate_problem_results(runs=all_results[problem.name], problem=problem) # Add in RRembo and LineBOresults for method in rls: with open(f'results/hartmann6_{D}_{method}.json', 'r') as fin: A = json.load(fin) res.objective_at_true_best[method] = np.minimum.accumulate(np.array(A), axis=1) # Save with open(f'results/hartmann6_{rs_str}{D}_aggregated_results.json', "w") as fout: json.dump(object_to_json({problem.name: res}), fout)
def load_experiment(filepath: str) -> Experiment: """Load experiment from file. 1) Read file. 2) Convert dictionary to Ax experiment instance. """ with open(filepath, "r") as file: json_experiment = json.loads(file.read()) return object_from_json(json_experiment)
def testEncodeDecodeSimpleBenchmarkProblem(self): branin_problem = get_branin_simple_benchmark_problem() sum_problem = get_sum_simple_benchmark_problem() new_branin_problem = object_from_json( object_to_json( branin_problem, encoder_registry=DEPRECATED_ENCODER_REGISTRY, class_encoder_registry=DEPRECATED_CLASS_ENCODER_REGISTRY, ), decoder_registry=DEPRECATED_DECODER_REGISTRY, class_decoder_registry=DEPRECATED_CLASS_DECODER_REGISTRY, ) new_sum_problem = object_from_json( object_to_json( sum_problem, encoder_registry=DEPRECATED_ENCODER_REGISTRY, class_encoder_registry=DEPRECATED_CLASS_ENCODER_REGISTRY, ), decoder_registry=DEPRECATED_DECODER_REGISTRY, class_decoder_registry=DEPRECATED_CLASS_DECODER_REGISTRY, ) self.assertEqual( branin_problem.f(1, 2), new_branin_problem.f(1, 2), branin(1, 2) ) self.assertEqual(sum_problem.f([1, 2]), new_sum_problem.f([1, 2]), 3) # Test using `from_botorch`. ackley_problem = SimpleBenchmarkProblem( f=from_botorch(Ackley()), noise_sd=0.0, minimize=True ) new_ackley_problem = object_from_json( object_to_json( ackley_problem, encoder_registry=DEPRECATED_ENCODER_REGISTRY, class_encoder_registry=DEPRECATED_CLASS_ENCODER_REGISTRY, ), decoder_registry=DEPRECATED_DECODER_REGISTRY, class_decoder_registry=DEPRECATED_CLASS_DECODER_REGISTRY, ) self.assertEqual( ackley_problem.f(1, 2), new_ackley_problem.f(1, 2), ackley(1, 2) )
def from_json_snapshot(serialized: Dict[str, Any], **kwargs) -> "AxClient": """Recreate an `AxClient` from a JSON snapshot.""" experiment = object_from_json(serialized.pop("experiment")) serialized_generation_strategy = serialized.pop("generation_strategy") ax_client = AxClient( generation_strategy=generation_strategy_from_json( generation_strategy_json=serialized_generation_strategy) if serialized_generation_strategy is not None else None, enforce_sequential_optimization=serialized.pop( "_enforce_sequential_optimization"), **kwargs, ) ax_client._experiment = experiment return ax_client
def table_S1_data(): with open('../benchmarks/results/all_aggregated_results.json', 'r') as fin: res = object_from_json(json.load(fin)) for D in [100, 1000]: pname = f'Hartmann6, D={D}' print('-----', pname) for m, ts in res[pname].gen_times.items(): # Get average total time for fit and gen t = np.mean(ts) t += np.mean(res[pname].fit_times[m]) # Divide by 200 to be time per iteration t /= 200. print(f'{m}: {t}')
def metric_from_sqa_util(self, metric_sqa: SQAMetric) -> Metric: """Convert SQLAlchemy Metric to Ax Metric""" metric_class = REVERSE_METRIC_REGISTRY.get(metric_sqa.metric_type) if metric_class is None: raise SQADecodeError( f"Cannot decode SQAMetric because {metric_sqa.metric_type} " f"is an invalid type.") args = dict(object_from_json(metric_sqa.properties) or {}) args["name"] = metric_sqa.name args["lower_is_better"] = metric_sqa.lower_is_better args = metric_class.deserialize_init_args(args=args) metric = metric_class(**args) metric.db_id = metric_sqa.id return metric
def testEncodeDecodeTorchTensor(self): x = torch.tensor( [[1.0, 2.0], [3.0, 4.0]], dtype=torch.float64, device=torch.device("cpu") ) expected_json = { "__type": "Tensor", "value": [[1.0, 2.0], [3.0, 4.0]], "dtype": {"__type": "torch_dtype", "value": "torch.float64"}, "device": {"__type": "torch_device", "value": "cpu"}, } x_json = object_to_json(x) self.assertEqual(expected_json, x_json) x2 = object_from_json(x_json) self.assertTrue(torch.equal(x, x2))
def load_experiment( filepath: str, decoder_registry: Dict[str, Type], class_decoder_registry: Dict[str, Callable[[Dict[str, Any]], Any]], ) -> Experiment: """Load experiment from file. 1) Read file. 2) Convert dictionary to Ax experiment instance. """ with open(filepath, "r") as file: json_experiment = json.loads(file.read()) return object_from_json( json_experiment, decoder_registry, class_decoder_registry )
def generation_strategy_from_sqa( self, gs_sqa: SQAGenerationStrategy) -> GenerationStrategy: """Convert SQALchemy generation strategy to Ax `GenerationStrategy`.""" steps = object_from_json(gs_sqa.steps) gs = GenerationStrategy(name=gs_sqa.name, steps=steps) gs._curr = gs._steps[gs_sqa.curr_index] gs._generator_runs = [ self.generator_run_from_sqa(gr) for gr in gs_sqa.generator_runs ] if len(gs._generator_runs) > 0: # Generation strategy had an initialized model. # pyre-ignore[16]: SQAGenerationStrategy does not have `experiment` attr. gs._experiment = self.experiment_from_sqa(gs_sqa.experiment) gs._restore_model_from_generator_run() gs._db_id = gs_sqa.id return gs
def testEncodeDecodeNumpy(self): arr = np.array([[1, 2, 3], [4, 5, 6]]) self.assertTrue( np.array_equal( arr, object_from_json( object_to_json( arr, encoder_registry=DEPRECATED_ENCODER_REGISTRY, class_encoder_registry=DEPRECATED_CLASS_ENCODER_REGISTRY, ), decoder_registry=DEPRECATED_DECODER_REGISTRY, class_decoder_registry=DEPRECATED_CLASS_DECODER_REGISTRY, ), ) )
def compile_ablation_benchmarks(): all_results = {} for rep in range(100): with open(f'results/ablation_rep_{rep}.json', 'r') as fin: res_i = object_from_json(json.load(fin)) all_results = merge_benchmark_results(all_results, res_i) problems = [branin_100] res = { p.name+'_ablation': aggregate_problem_results(runs=all_results[p.name], problem=p) for p in problems } # Save with open(f'results/ablation_aggregated_results.json', "w") as fout: json.dump(object_to_json(res), fout)
def extract_ablation_results(): with open(f'../benchmarks/results/ablation_aggregated_results.json', 'r') as fin: res = object_from_json(json.load(fin)) # Results ys = { 'ALEBO': res['Branin, D=100_ablation'].objective_at_true_best['ALEBO, base'], 'Ablation: Matern kernel': res['Branin, D=100_ablation']. objective_at_true_best['ALEBO, kernel ablation'], 'Ablation: Normal projection': res['Branin, D=100_ablation']. objective_at_true_best['ALEBO, projection ablation'], } return ys
def make_nasbench_figure(): with open('../benchmarks/results/nasbench_aggregated_results.json', 'r') as fin: res = object_from_json(json.load(fin)) # A map from method idx in plot_method_names to the name used in res method_idx_to_res_name = { 1: 'REMBO', 3: 'HeSBO', 9: 'cmaes', 10: 'turbo', 11: 'Sobol', 0: 'ALEBO', } plot_method_names[3] = 'HeSBO' plot_colors['HeSBO'] = plt.cm.tab20(3) fig = plt.figure(figsize=(2.96, 2.0)) ax1 = fig.add_subplot(111) method_idx = 1 method_names_used = [] for i, m in method_idx_to_res_name.items(): f = np.nanmean(res[m], axis=0) sem = np.nanstd(res[m], axis=0) / np.sqrt(res[m].shape[0]) x = np.arange(1, 51) mname = plot_method_names[i] color = plot_colors[mname] ax1.plot(x, f, color=color, label=mname) ax1.errorbar(x, f, yerr=2 * sem, color=color, alpha=0.5, ls='') ax1.set_xlim([0, 51]) ax1.set_ylabel('Best feasible test accuracy', fontsize=9) ax1.set_xlabel('Function evaluations', fontsize=9) #ax1.legend(bbox_to_anchor=(1.0, 1.24), ncol=4, fontsize=6, columnspacing=1.65) ax1.legend(ncol=2, loc='lower right', fontsize=7) ax1.set_ylim([0.92, 0.936]) ax1.set_yticks([0.92, 0.925, 0.93, 0.935]) ax1.set_yticklabels(['92.0\%', '92.5\%', '93.0\%', '93.5\%']) ax1.grid(alpha=0.2, zorder=-10) plt.subplots_adjust(right=0.98, bottom=0.17, left=0.19, top=0.98) plt.savefig(f'pdfs/nas.pdf', pad_inches=0)
def testEncodeDecodeTorchTensor(self): x = torch.tensor( [[1.0, 2.0], [3.0, 4.0]], dtype=torch.float64, device=torch.device("cpu") ) expected_json = { "__type": "Tensor", "value": [[1.0, 2.0], [3.0, 4.0]], "dtype": {"__type": "torch_dtype", "value": "torch.float64"}, "device": {"__type": "torch_device", "value": "cpu"}, } x_json = object_to_json( x, encoder_registry=DEPRECATED_ENCODER_REGISTRY, class_encoder_registry=DEPRECATED_CLASS_ENCODER_REGISTRY, ) self.assertEqual(expected_json, x_json) x2 = object_from_json( x_json, decoder_registry=DEPRECATED_DECODER_REGISTRY, class_decoder_registry=DEPRECATED_CLASS_DECODER_REGISTRY, ) self.assertTrue(torch.equal(x, x2))
def testEncodeDecode(self): for class_, fake_func in TEST_CASES: # Can't load trials from JSON, because a batch needs an experiment # in order to be initialized if class_ == "BatchTrial" or class_ == "Trial": continue # Can't load parameter constraints from JSON, because they require # a SearchSpace in order to be initialized if class_ == "OrderConstraint" or class_ == "SumConstraint": continue original_object = fake_func() # Can't load generation strategy from JSON, because it requires an # experiment to reload. if class_ == "GenerationStrategy": with self.assertRaisesRegex(ValueError, ".* custom .* cannot "): original_object._uses_registered_models = False object_to_json(original_object) continue json_object = object_to_json(original_object) converted_object = object_from_json(json_object) if class_ == "SimpleExperiment": # Evaluation functions will be different, so need to do # this so equality test passes with self.assertRaises(Exception): converted_object.evaluation_function() original_object.evaluation_function = None converted_object.evaluation_function = None self.assertEqual( original_object, converted_object, msg=f"Error encoding/decoding {class_}.", )
def test_encode_decode_numpy(self): arr = np.array([[1, 2, 3], [4, 5, 6]]) self.assertTrue(np.array_equal(arr, object_from_json(object_to_json(arr))))
def make_fig_5(): # Load in the benchmark results res = {} for fname in ['hartmann6_1000', 'branin_gramacy_100']: with open(f'../benchmarks/results/{fname}_aggregated_results.json', 'r') as fin: res.update(object_from_json(json.load(fin))) # A map from method idx in plot_method_names to the name used in res method_idx_to_res_name = { 0: 'ALEBO', 1: 'REMBO', 2: 'HeSBO, d=d', 3: 'HeSBO, d=2d', 4: 'rrembos_standard_kPsi', 5: 'rrembos_reverse_kPsi', 6: 'ebo', 7: 'addgpucb', 8: 'smac', 9: 'cmaes', 10: 'turbo', 11: 'Sobol', 12: 'coordinatelinebo', 13: 'randomlinebo', 14: 'descentlinebo', } # Make the figure fig = plt.figure(figsize=(5.5, 3.7)) ####### Branin, D=100 ax1 = fig.add_subplot(231) ax2 = fig.add_subplot(234) res_h = res['Branin, D=100'] for idx, m in enumerate(plot_method_names): res_name = method_idx_to_res_name[idx] if res_name not in res_h.objective_at_true_best: continue # Not run on this problem Y = res_h.objective_at_true_best[res_name] f = Y.mean(axis=0) x = np.arange(1, 51) color = plot_colors[m] ax1.plot(x, f, color=color, label=m) parts = ax2.violinplot(positions=[idx], dataset=Y[:, 49], showmeans=True) for pc in parts['bodies']: pc.set_facecolor(color) pc.set_edgecolor(color) for field in ['cmeans', 'cmaxes', 'cmins', 'cbars']: parts[field].set_color(color) ax1.set_xlim([0, 51]) ax1.set_ylabel('Best value found', fontsize=7) ax1.set_xlabel('Function evaluations', fontsize=7) ax1.axhline(y=0.397887, c='gray', ls='--') ax1.grid(alpha=0.2, zorder=-10) ax1.set_ylim([0, 7]) ax2.set_xticks(range(12)) ax2.set_xticklabels([]) ax2.set_ylabel('Final value', fontsize=7) ax2.grid(alpha=0.2, zorder=-10) ax2.set_xticklabels([plot_method_names[i] for i in range(12)], fontsize=6) ax2.xaxis.set_tick_params(rotation=90) # Make the legend custom_lines = [] names = [] for i in range(12): names.append(plot_method_names[i]) custom_lines.append( Line2D([0], [0], color=plot_colors[plot_method_names[i]], lw=2)) order = range(12) names = [names[o] for o in order] custom_lines = [custom_lines[o] for o in order] ax1.legend(custom_lines, names, ncol=6, fontsize=5.5, bbox_to_anchor=(3.52, -2.26)) ax1.set_title('Branin, $d$=2, $D$=100', fontsize=8) ####### Hartmann6, D=1000 ax1 = fig.add_subplot(232) ax2 = fig.add_subplot(235) res_h = res['Hartmann6, D=1000'] for idx, m in enumerate(plot_method_names): res_name = method_idx_to_res_name[idx] if res_name not in res_h.objective_at_true_best: continue # Not run on this problem Y = res_h.objective_at_true_best[res_name] f = Y.mean(axis=0) x = np.arange(1, 201) color = plot_colors[m] ax1.plot(x, f, color=color, label=m) parts = ax2.violinplot(positions=[idx], dataset=Y[:, 199], showmeans=True) for pc in parts['bodies']: pc.set_facecolor(color) pc.set_edgecolor(color) for field in ['cmeans', 'cmaxes', 'cmins', 'cbars']: parts[field].set_color(color) ax1.set_xlim([0, 201]) #ax1.set_ylabel('Best value found', fontsize=9) ax1.set_xlabel('Function evaluations', fontsize=7) ax1.axhline(y=-3.32237, c='gray', ls='--') ax1.grid(alpha=0.2, zorder=-10) ax1.set_ylim([-3.5, -0.5]) ax2.set_xticks(range(12)) ax2.set_xticklabels([]) #ax2.set_ylabel('Final value', fontsize=9) ax2.grid(alpha=0.2, zorder=-10) ax2.set_xticklabels([plot_method_names[i] for i in range(12)], fontsize=6) ax2.xaxis.set_tick_params(rotation=90) ax1.set_title('Hartmann6, $d$=6, $D$=1000', fontsize=8) ####### Gramacy, D=100 ax1 = fig.add_subplot(233) ax2 = fig.add_subplot(236) res_h = res['Gramacy, D=100'] for idx, m in enumerate(plot_method_names): res_name = method_idx_to_res_name[idx] if res_name not in res_h.objective_at_true_best: continue # Not run on this problem Y = res_h.objective_at_true_best[res_name] f = Y.mean(axis=0) x = np.arange(1, 51) color = plot_colors[m] ax1.plot(x, f, color=color, label=m) parts = ax2.violinplot(positions=[idx], dataset=Y[:, 49], showmeans=True) for pc in parts['bodies']: pc.set_facecolor(color) pc.set_edgecolor(color) for field in ['cmeans', 'cmaxes', 'cmins', 'cbars']: parts[field].set_color(color) ax1.set_xlim([0, 51]) #ax1.set_ylabel('Best value found', fontsize=9) ax1.set_xlabel('Function evaluations', fontsize=7) ax1.set_ylim([0.58, 1]) ax1.axhline(y=0.5998, c='gray', ls='--') ax1.grid(alpha=0.2, zorder=-10) ax2.set_xticks(range(12)) ax2.set_xticklabels([plot_method_names[i] for i in range(12)], fontsize=6) ax2.xaxis.set_tick_params(rotation=90) #ax2.set_ylabel('Final value', fontsize=9) ax2.grid(alpha=0.2, zorder=-10) ax1.set_title('Gramacy, $d$=2, $D$=100', fontsize=8) plt.subplots_adjust(right=0.995, bottom=0.3, left=0.07, top=0.94, wspace=0.25, hspace=0.45) plt.savefig('pdfs/benchmark_results_t.pdf', pad_inches=0)