def _build_kde_wrapper(self, configspace: ConfigurationSpace) -> KdeWrapper: kde_vartypes = '' vartypes = [] for h in configspace.get_hyperparameters(): if hasattr(h, 'sequence'): raise RuntimeError( 'This version on dswizard does not support ordinal hyperparameters. ' f'Please encode {h.name} as an integer parameter!') if hasattr(h, 'choices'): kde_vartypes += 'u' vartypes += [len(h.choices)] else: kde_vartypes += 'c' vartypes += [0] vartypes = np.array(vartypes, dtype=int) return KdeWrapper(kde_vartypes, vartypes)
def _transform_continuous_designs( self, design: np.ndarray, origin: str, cs: ConfigurationSpace) -> typing.List[Configuration]: params = cs.get_hyperparameters() for idx, param in enumerate(params): if isinstance(param, NumericalHyperparameter): continue elif isinstance(param, Constant): # add a vector with zeros design_ = np.zeros(np.array(design.shape) + np.array((0, 1))) design_[:, :idx] = design[:, :idx] design_[:, idx + 1:] = design[:, idx:] design = design_ elif isinstance(param, CategoricalHyperparameter): v_design = design[:, idx] v_design[v_design == 1] = 1 - 10**-10 design[:, idx] = np.array(v_design * len(param.choices), dtype=np.int) elif isinstance(param, OrdinalHyperparameter): v_design = design[:, idx] v_design[v_design == 1] = 1 - 10**-10 design[:, idx] = np.array(v_design * len(param.sequence), dtype=np.int) else: raise ValueError("Hyperparameter not supported in LHD") self.logger.debug("Initial Design") configs = [] for vector in design: try: conf = deactivate_inactive_hyperparameters( configuration=None, configuration_space=cs, vector=vector) except ForbiddenValueError: continue conf.origin = origin configs.append(conf) self.logger.debug(conf) self.logger.debug("Size of initial design: %d" % (len(configs))) return configs
def get_default_initial_configs(phps: ConfigurationSpace, n_configs) -> List[Configuration]: None_name = "None:NoneType" phps = deepcopy(phps) for config in phps.get_hyperparameters(): name: str = config.name if name.startswith("preprocessing") and name.endswith( "__choice__") and (None_name in config.choices): config.default_value = None_name model_choice = phps.get_hyperparameter("estimator:__choice__") ans = [] for choice in model_choice.choices: cur_phps = deepcopy(phps) cur_phps.get_hyperparameter( "estimator:__choice__").default_value = choice default = cur_phps.get_default_configuration() ans.append(default) if len(ans) < n_configs: ans.extend(phps.sample_configuration(n_configs - len(ans))) return ans
def replace_phps(phps: ConfigurationSpace, key, value): for hp in phps.get_hyperparameters(): if hp.__class__.__name__ == "Constant" and hp.name.endswith(key): hp.value = _encode(value)
class fANOVA(AbstractEvaluator): def __init__(self, scenario, cs, model, to_evaluate: int, runhist: RunHistory, rng, n_pairs=5, minimize=True, pairwise=True, preprocessed_X=None, preprocessed_y=None, **kwargs): super().__init__(scenario, cs, model, to_evaluate, rng, **kwargs) self.name = 'fANOVA' self.logger = 'pimp.' + self.name # Turn all Constants into Categoricals (fANOVA cannot handle Constants) self.cs_contained_constant = False if any( [isinstance(hp, Constant) for hp in self.cs.get_hyperparameters()]): self.logger.debug( "Replacing configspace's hyperparameter Constants by one-value Categoricals." ) new_hyperparameters = [ CategoricalHyperparameter(hp.name, [hp.value]) if isinstance( hp, Constant) else hp for hp in self.cs.get_hyperparameters() ] self.cs = ConfigurationSpace() self.cs.add_hyperparameters(new_hyperparameters) self.cs_contained_constant = True # This way the instance features in X are ignored and a new forest is constructed if self.model.instance_features is None: self.logger.info('No preprocessing necessary') if preprocessed_X is not None and preprocessed_y is not None: self.X = preprocessed_X self.y = preprocessed_y else: self._preprocess(runhist) else: self._preprocess(runhist) cutoffs = (-np.inf, np.inf) if minimize: cutoffs = ( -np.inf, self.model.predict_marginalized_over_instances( np.array([ impute_inactive_values( self.cs.get_default_configuration()).get_array() ]))[0].flatten()[0]) elif minimize is False: cutoffs = (self.model.predict_marginalized_over_instances( np.array([ impute_inactive_values( self.cs.get_default_configuration()).get_array() ]))[0].flatten()[0], np.inf) self.evaluator = fanova_pyrfr(X=self.X, Y=self.y.flatten(), config_space=self.cs, seed=self.rng.randint(2**31 - 1), cutoffs=cutoffs) self.n_most_imp_pairs = n_pairs self.num_single = None self.pairwise = pairwise self.evaluated_parameter_importance_uncertainty = OrderedDict() def _preprocess(self, runhistory): """ Method to marginalize over instances such that fANOVA can determine the parameter importance without having to deal with instance features. :param runhistory: RunHistory that knows all configurations that were run. For all these configurations we have to marginalize away the instance features with which fANOVA will make it's predictions """ self.logger.info( 'PREPROCESSING PREPROCESSING PREPROCESSING PREPROCESSING PREPROCESSING PREPROCESSING' ) self.logger.info('Marginalizing away all instances!') configs = runhistory.get_all_configs() if self.cs_contained_constant: configs = [ Configuration(self.cs, vector=c.get_array()) for c in configs ] X_non_hyper, X_prime = [], [] for config in configs: config = impute_inactive_values(config).get_array() X_prime.append(config) X_non_hyper.append(config) for idx, param in enumerate(self.cs.get_hyperparameters()): if not (isinstance(param, CategoricalHyperparameter) or isinstance(param, Constant)): X_non_hyper[-1][idx] = param._transform( X_non_hyper[-1][idx]) X_non_hyper = np.array(X_non_hyper) X_prime = np.array(X_prime) y_prime = np.array( self.model.predict_marginalized_over_instances(X_prime)[0]) self.X = X_non_hyper self.y = y_prime self.logger.info('Size of training X after preprocessing: %s' % str(self.X.shape)) self.logger.info('Size of training y after preprocessing: %s' % str(self.y.shape)) self.logger.info('Finished Preprocessing') def plot_result(self, name='fANOVA', show=True): if not os.path.exists(name): os.mkdir(name) if self.scenario.run_obj == 'runtime': label = 'runtime [sec]' elif self.scenario.run_obj == 'quality': label = 'cost' else: label = '%s' % self.scenario.run_obj vis = Visualizer(self.evaluator, self.cs, directory=name, y_label=label) self.logger.info('Getting Marginals!') pbar = tqdm(range(self.to_evaluate), ascii=True, disable=not self.verbose) for i in pbar: plt.close('all') plt.clf() param = list(self.evaluated_parameter_importance.keys())[i] outfile_name = os.path.join(name, param.replace(os.sep, "_") + ".png") vis.plot_marginal(self.cs.get_idx_by_hyperparameter_name(param), log_scale=False, show=False) fig = plt.gcf() fig.savefig(outfile_name) plt.close('all') plt.clf() outfile_name = os.path.join( name, param.replace(os.sep, "_") + "_log.png") vis.plot_marginal(self.cs.get_idx_by_hyperparameter_name(param), log_scale=True, show=False) fig = plt.gcf() fig.savefig(outfile_name) plt.close('all') plt.clf() if show: plt.show() pbar.set_description('Creating fANOVA plot: {: <.30s}'.format( outfile_name.split(os.path.sep)[-1])) if self.pairwise: self.logger.info('Plotting Pairwise-Marginals!') most_important_ones = list( self.evaluated_parameter_importance.keys( ))[:min(self.num_single, self.n_most_imp_pairs)] try: vis.create_most_important_pairwise_marginal_plots( most_important_ones) except TypeError: self.logger.warning('Could not create pairwise plots!') plt.close('all') def run(self) -> OrderedDict: try: params = self.cs.get_hyperparameters() tmp_res = [] for idx, param in enumerate(params): imp = self.evaluator.quantify_importance([idx])[(idx, )] self.logger.debug('{:>02d} {:<30s}: {:>02.4f}'.format( idx, param.name, imp['total importance'])) tmp_res.append(imp) tmp_res_sort_keys = [ i[0] for i in sorted(enumerate(tmp_res), key=lambda x: x[1]['total importance'], reverse=True) ] self.logger.debug(tmp_res_sort_keys) count = 0 for idx in tmp_res_sort_keys: if count >= self.to_evaluate: break self.logger.info('{:>02d} {:<30s}: {:>02.4f}'.format( idx, params[idx].name, tmp_res[idx]['total importance'])) self.evaluated_parameter_importance[ params[idx].name] = tmp_res[idx]['total importance'] try: self.evaluated_parameter_importance_uncertainty[ params[idx].name] = tmp_res[idx]['total std'] except KeyError as e: self.logger.debug( "std not available yet for this fanova version") count += 1 self.num_single = len( list(self.evaluated_parameter_importance.keys())) if self.pairwise: self.logger.info( 'Computing most important pairwise marginals using at most' ' the %d most important ones.' % min(self.n_most_imp_pairs, self.num_single)) pairs = [ x for x in it.combinations( list(self.evaluated_parameter_importance.keys()) [:self.n_most_imp_pairs], 2) ] for pair in pairs: imp = self.evaluator.quantify_importance(pair)[pair] try: mean, std = imp['individual importance'], imp[ 'individual std'] self.evaluated_parameter_importance_uncertainty[str( list(pair))] = std except KeyError as e: self.logger.debug( "std not available yet for this fanova version") mean = imp['individual importance'] self.evaluated_parameter_importance[str(list(pair))] = mean a, b = pair if len(a) > 13: a = str(a)[:5] + '...' + str(a)[-5:] if len(b) > 13: b = str(b)[:5] + '...' + str(b)[-5:] self.logger.info('{:>02d} {:<30s}: {:>02.4f}'.format( -1, a + ' <> ' + b, mean)) all_res = { 'imp': self.evaluated_parameter_importance, 'order': list(self.evaluated_parameter_importance.keys()) } return all_res except ZeroDivisionError: with open('fANOVA_crash_data.pkl', 'wb') as fh: pickle.dump([self.X, self.y, self.cs], fh) raise Exception( 'fANOVA crashed with a "float division by zero" error. Dumping the data to disk' )
def build_matrix(feature_pd:pd.DataFrame, perf_pd:pd.DataFrame, configs:list, cs:ConfigurationSpace, n_insts:int=None): insts = list(feature_pd.index) if n_insts is not None and n_insts < len(insts): insts = random.sample(insts, n_insts) config_matrix = convert_configurations_to_array(configs) # one hot encode categorical parameters n_values = [] mask_array = [] parameters = cs.get_hyperparameters() for param in parameters: if isinstance(param, (CategoricalHyperparameter)): n_values.append(len(param.choices)) mask_array.append(True) else: mask_array.append(False) n_values = np.array(n_values) mask_array = np.array(mask_array) ohe = OneHotEncoder(n_values=n_values, categorical_features=mask_array, sparse=False) config_matrix = ohe.fit_transform(config_matrix) train_config_indices = random.sample(range(len(configs)), int(len(configs)/2)) valid_config_indices = random.sample(train_config_indices, int(len(train_config_indices)/2)) train_inst_indices = random.sample(range(len(insts)), int(len(insts)/2)) valid_inst_indices = random.sample(train_inst_indices, int(len(train_inst_indices)/2)) # convert in X matrix and y vector X_I, X_II, X_III, X_IV = [[],[],[],[]], [], [], [] y_I, y_II, y_III, y_IV = [[],[],[],[]], [], [], [] for i_idx, inst in enumerate(insts): feat_vector = feature_pd.loc[inst].values perf_vector = perf_pd.loc[inst].values for c_idx in range(len(configs)): config_vec = config_matrix[c_idx,:] perf = perf_vector[c_idx] if i_idx in train_inst_indices and c_idx in train_config_indices: if i_idx in valid_inst_indices and c_idx in valid_config_indices: X_I[3].append(np.concatenate((config_vec, feat_vector))) y_I[3].append(perf) elif i_idx not in valid_inst_indices and c_idx in valid_config_indices: X_I[2].append(np.concatenate((config_vec, feat_vector))) y_I[2].append(perf) elif i_idx in valid_inst_indices and c_idx not in valid_config_indices: X_I[1].append(np.concatenate((config_vec, feat_vector))) y_I[1].append(perf) else: X_I[0].append(np.concatenate((config_vec, feat_vector))) y_I[0].append(perf) elif i_idx not in train_inst_indices and c_idx in train_config_indices: X_II.append(np.concatenate((config_vec, feat_vector))) y_II.append(perf) elif i_idx in train_inst_indices and c_idx not in train_config_indices: X_III.append(np.concatenate((config_vec, feat_vector))) y_III.append(perf) else: X_IV.append(np.concatenate((config_vec, feat_vector))) y_IV.append(perf) X_II, X_III, X_IV = np.array(X_II), np.array(X_III), np.array(X_IV) y_II, y_III, y_IV = np.array(y_II), np.array(y_III), np.array(y_IV) X_I = np.array([np.array(X_I[0]),np.array(X_I[1]),np.array(X_I[2]),np.array(X_I[3])]) y_I = np.array([np.array(y_I[0]),np.array(y_I[1]),np.array(y_I[2]),np.array(y_I[3])]) print(X_I.shape, X_II.shape, X_III.shape, X_IV.shape) print(y_I.shape, y_II.shape, y_III.shape, y_IV.shape) return X_I, X_II, X_III, X_IV, y_I, y_II, y_III, y_IV