def parse_config(config, config_space, cell_type): """Function that converts a ConfigSpace representation of the architecture to a Genotype. """ cell = [] config = ConfigSpace.Configuration(config_space, config) edges = custom_sorted( list( filter( re.compile('.*edge_{}*.'.format(cell_type)).match, config_space.get_active_hyperparameters(config)))).__iter__() nodes = custom_sorted( list( filter( re.compile('.*inputs_node_{}*.'.format(cell_type)).match, config_space.get_active_hyperparameters(config)))).__iter__() op_1 = config[next(edges)] op_2 = config[next(edges)] cell.extend([(op_1, 0), (op_2, 1)]) for node in nodes: op_1 = config[next(edges)] op_2 = config[next(edges)] input_1, input_2 = map(int, config[node].split('_')) cell.extend([(op_1, input_1), (op_2, input_2)]) return cell
def _check_and_cast_configuration(configuration: Union[Dict, ConfigSpace.Configuration], configuration_space: ConfigSpace.ConfigurationSpace) \ -> ConfigSpace.Configuration: """ Helper-function to evaluate the given configuration. Cast it to a ConfigSpace.Configuration and evaluate if it violates its boundaries. Note: We remove inactive hyperparameters from the given configuration. Inactive hyperparameters are hyperparameters that are not relevant for a configuration, e.g. hyperparameter A is only relevant if hyperparameter B=1 and if B!=1 then A is inactive and will be removed from the configuration. Since the authors of the benchmark removed those parameters explicitly, they should also handle the cases that inactive parameters are not present in the input-configuration. """ if isinstance(configuration, dict): configuration = ConfigSpace.Configuration(configuration_space, configuration, allow_inactive_with_values=True) elif isinstance(configuration, ConfigSpace.Configuration): configuration = configuration else: raise TypeError(f'Configuration has to be from type List, np.ndarray, dict, or ' f'ConfigSpace.Configuration but was {type(configuration)}') all_hps = set(configuration_space.get_hyperparameter_names()) active_hps = configuration_space.get_active_hyperparameters(configuration) inactive_hps = all_hps - active_hps if len(inactive_hps) != 0: logger.debug(f'There are inactive {len(inactive_hps)} hyperparameter: {inactive_hps}' 'Going to remove them from the configuration.') configuration = deactivate_inactive_hyperparameters(configuration, configuration_space) configuration_space.check_configuration(configuration) return configuration
def new_result(self, job, config_info): # pylint: disable=unused-argument if job.exception is not None: self.logger.warning(f"job {job.id} failed with exception\n{job.exception}") if job.result is None: # One could skip crashed results, but we decided to # assign a +inf loss and count them as bad configurations loss = np.inf else: # same for non numeric losses. # Note that this means losses of minus infinity will count as bad! loss = job.result["loss"] if np.isfinite(job.result["loss"]) else np.inf config = ConfigSpace.Configuration(self.configspace, job.kwargs["config"]) self.configs.append(config) self.losses.append(loss) if self.has_model: # TODO: include old X = convert_configurations_to_array(self.configs) Y = np.array(self.losses, dtype=np.float64) self.model.train(X, Y) self.acquisition_func.update( model=self.model, eta=min(self.losses), )
def get_config(self, budget): """ Function to sample a new configuration This function is called inside Hyperband to query a new configuration Parameters: ----------- budget: float the budget for which this configuration is scheduled returns: config should return a valid configuration """ # No observations available for this budget sample from the prior if len(self.kde_models.keys()) == 0: return self.configspace.sample_configuration().get_dictionary() # If we haven't seen anything with this budget, we sample from the kde trained on the highest budget if budget not in self.kde_models.keys(): budget = sorted(self.kde_models.keys())[-1] # TODO: This only works in continuous space and with gaussian kernels kde = self.kde_models[budget] idx = np.random.randint(0, len(self.kde_models[budget].data)) vector = [sps.truncnorm.rvs(-m/bw,(1-m)/bw, loc=m, scale=bw) for m,bw in zip(self.kde_models[budget].data[idx], kde.bw)] if np.any(np.array(vector)>1) or np.any(np.array(vector)<0): raise RuntimeError("truncated normal sampling problems!") sample = ConfigSpace.Configuration(self.configspace, vector=vector) return sample.get_dictionary(), {}
def new_result(self, job, config_info): # pylint: disable=unused-argument if job.exception is not None: self.logger.warning( f"job {job.id} failed with exception\n{job.exception}") if job.result is None: # One could skip crashed results, but we decided to # assign a +inf loss and count them as bad configurations loss = np.inf else: # same for non numeric losses. # Note that this means losses of minus infinity will count as bad! loss = job.result["loss"] if np.isfinite( job.result["loss"]) else np.inf # We want to get a numerical representation of the configuration in the # original space conf = ConfigSpace.Configuration(self.configspace, job.kwargs["config"]) self.configs.append(conf.get_array()) self.losses.append(loss) # only fit kdes if enough points are available if self.has_model: good_kde, bad_kde = _fit_kde( np.array(self.configs), np.array(self.losses), self.kde_var_types, self.min_bandwidth, self.min_points_in_model, self.top_n_percent, ) self.kde_models = {"good": good_kde, "bad": bad_kde}
def get_fANOVA_data(self, config_space, budgets=None, loss_fn=lambda r: r.loss, failed_loss=None): import numpy as np import ConfigSpace as CS id2conf = self.get_id2config_mapping() if budgets is None: budgets = self.HB_config["budgets"] if len(budgets) > 1: config_space.add_hyperparameter( CS.UniformFloatHyperparameter("budget", min(budgets), max(budgets), log=True)) hp_names = config_space.get_hyperparameter_names() hps = config_space.get_hyperparameters() needs_transform = list( map(lambda h: isinstance(h, CS.CategoricalHyperparameter), hps)) all_runs = self.get_all_runs(only_largest_budget=False) all_runs = list(filter(lambda r: r.budget in budgets, all_runs)) X = [] y = [] for r in all_runs: if r.loss is None: if failed_loss is None: continue else: y.append(failed_loss) else: y.append(loss_fn(r)) config = id2conf[r.config_id]["config"] if len(budgets) > 1: config["budget"] = r.budget config = CS.Configuration(config_space, config) x = [] for (name, hp, transform) in zip(hp_names, hps, needs_transform): if transform: # pylint: disable=protected-access x.append(hp._inverse_transform(config[name])) # pylint: enable=protected-access else: x.append(config[name]) X.append(x) return np.array(X), np.array(y), config_space
def get_fANOVA_data(self, config_space, budgets=None): import numpy as np import ConfigSpace as CS id2conf = self.get_id2config_mapping() if budgets is None: budgets = self.HB_config['budgets'] if len(budgets)>1: config_space.add_hyperparameter(CS.UniformFloatHyperparameter('budget', min(budgets), max(budgets), log=True)) hp_names = list(map( lambda hp: hp.name, config_space.get_hyperparameters())) all_runs = self.get_all_runs(only_largest_budget=False) all_runs=list(filter( lambda r: r.budget in budgets, all_runs)) X = [] y = [] for r in all_runs: if r.loss is None: continue config = id2conf[r.config_id]['config'] if len(budgets)>1: config['budget'] = r.budget config = CS.Configuration(config_space, config) X.append([config[n] for n in hp_names]) y.append(r.loss) return(np.array(X), np.array(y), config_space)
def new_result(self, job): """ function to register finished runs Every time a run has finished, this function should be called to register it with the result logger. If overwritten, make sure to call this method from the base class to ensure proper logging. Parameters: ----------- job_id: dict a dictionary containing all the info about the run job_result: dict contains all the results of the job, i.e. it's a dict with the keys 'loss' and 'info' """ super().new_result(job) conf = ConfigSpace.Configuration(self.config_space, job.kwargs['config']).get_array() epochs = len(job.result["info"]["learning_curve"]) budget = int(job.kwargs["budget"]) t_idx = np.linspace(budget / epochs, budget, epochs) / self.max_budget x_new = np.repeat(conf[None, :], t_idx.shape[0], axis=0) x_new = np.concatenate((x_new, t_idx[:, None]), axis=1) # Smooth learning curve lc = smoothing(job.result["info"]["learning_curve"]) # Flip learning curves since LC-Net wants increasing curves lc_new = [1 - y for y in lc] if self.train is None: self.train = x_new self.train_targets = lc_new else: self.train = np.append(self.train, x_new, axis=0) self.train_targets = np.append(self.train_targets, lc_new, axis=0) if self.counter >= self.n_points: self.lock.acquire() y_min = np.min(self.train_targets) y_max = np.max(self.train_targets) train_targets = (self.train_targets - y_min) / (y_max - y_min) self.model.train(self.train, train_targets) self.is_trained = True self.counter = 0 self.lock.release() else: self.counter += epochs
def compute_reward(self, sample): config = ConfigSpace.Configuration( self.space.get_configuration_space(), vector=sample) y, c = self.space.objective_function( self.nasbench, config, budget=self.budget) fitness = float(y) return fitness
def EvaluateNasbench(theta, search_space, logger, NASbenchName): # get result log stdout_backup = sys.stdout result_path = os.path.join(cfg.OUT_DIR, "result.log") log_file = open(result_path, "w") sys.stdout = log_file if NASbenchName == "nasbench201": geotype = search_space.genotype(theta) index = api_nasben201.query_index_by_arch(geotype) api_nasben201.show(index) else: current_best = np.argmax(theta, axis=1) config = ConfigSpace.Configuration( search_space.search_space.get_configuration_space(), vector=current_best) adjacency_matrix, node_list = search_space.search_space.convert_config_to_nasbench_format( config) node_list = [ INPUT, *node_list, OUTPUT ] if search_space.search_space.search_space_number == 3 else [ INPUT, *node_list, CONV1X1, OUTPUT ] adjacency_list = adjacency_matrix.astype(np.int).tolist() model_spec = api.ModelSpec(matrix=adjacency_list, ops=node_list) nasbench_data = nasbench.query(model_spec, epochs=108) print("the test accuracy in {}".format(NASbenchName)) print(nasbench_data['test_accuracy']) log_file.close() sys.stdout = stdout_backup
def data_to_state(data: dict) -> TuningJobState: cs = CS.ConfigurationSpace() cs_names = ['x{}'.format(i) for i in range(len(data['ss_limits']))] cs.add_hyperparameters([ CSH.UniformFloatHyperparameter(name=name, lower=lims['min'], upper=lims['max']) for name, lims in zip(cs_names, data['ss_limits']) ]) _evaluations = [] x_mult = [] x_add = [] for lim in data['ss_limits']: mn, mx = lim['min'], lim['max'] x_mult.append(mx - mn) x_add.append(mn) x_mult = np.array(x_mult) x_add = np.array(x_add) for x, y in zip(data['train_inputs'], data['train_targets']): x_decoded = x * x_mult + x_add config_dct = dict(zip(cs_names, x_decoded)) config = CS.Configuration(cs, values=config_dct) _evaluations.append( CandidateEvaluation(config, dictionarize_objective(y))) return TuningJobState(hp_ranges=HyperparameterRanges_CS(cs), candidate_evaluations=_evaluations, failed_candidates=[], pending_evaluations=[])
def _sample_from_ei(kde_models, num_samples, vartypes, min_bandwidth, bw_factor, configspace): good_pdf = kde_models["good"].pdf bad_pdf = kde_models["bad"].pdf candidates = [ _generate_candidate(bw_factor, kde_models["good"], min_bandwidth, vartypes) for _ in range(num_samples) ] values = [ei_tpe(candidate, good_pdf, bad_pdf) for candidate in candidates] best_vector = None for value, candidate in sorted(zip(values, candidates), reverse=True): if not np.isfinite(value): # right now, this happens because a KDE does not contain all values for a # categorical parameter this cannot be fixed with the statsmodels KDE, so # for now, we are just going to evaluate this one if the good_kde has a # finite value, i.e. there is no config with that value in the bad kde, so # it shouldn't be terrible. if np.isfinite(good_pdf(candidate)): best_vector = candidate break else: best_vector = candidate break if best_vector is None: raise ValueError("EI optimization failed") best_vector = _parse_categorical(best_vector, configspace) return ConfigSpace.Configuration(configspace, vector=best_vector)
def get_config(self, budget): """ Function to sample a new configuration This function is called inside Hyperband to query a new configuration Parameters: ----------- budget: float the budget for which this configuration is scheduled returns: config should return a valid configuration """ sample = None info_dict = {} # If no model is available, sample from prior # also mix in a fraction of random configs if len(self.kde_models.keys() ) == 0 or np.random.rand() < self.random_fraction: sample = self.configspace.sample_configuration() info_dict['model_based_pick'] = False if sample is None: try: #import pdb; pdb.set_trace() samples = self.kde_models[budget]['good'].sample( self.num_samples) ei = self.kde_models[budget]['good'].pdf( samples) / self.kde_models[budget]['bad'].pdf(samples) best_idx = np.argmax(ei) best_vector = samples[best_idx] sample = ConfigSpace.Configuration(self.configspace, vector=best_vector) sample = ConfigSpace.util.deactivate_inactive_hyperparameters( configuration_space=self.configspace, configuration=sample.get_dictionary()) info_dict['model_based_pick'] = True except Exception as e: self.logger.warning(("="*50 + "\n")*3 +\ "Error sampling a configuration!\n"+\ "Models for budgets: %s"%(self.kde_models.keys()) +\ "\n here is a traceback:" +\ traceback.format_exc()) for b, l in self.losses.items(): self.logger.debug("budget: {}\nlosses:{}".format(b, l)) sample = self.configspace.sample_configuration() info_dict['model_based_pick'] = False return sample.get_dictionary(), info_dict
def _update_trackers(self, traj, runtime, history): if len(self.traj) == self.inc_run: incumbent = ConfigSpace.Configuration( self.cs, vector=traj[1]).get_dictionary() for key in list(incumbent.keys()): if key != "agent": incumbent[key[2:]] = incumbent.pop(key) traj_obj = { "cpu_time": time.time() - self.start, "wall_clocktime": time.time() - self.start, "evaluations": len(self.traj), "cost": traj[0], "incumbent": incumbent, "budget": 0, "origin": "DEHB" } self.smac_trajectory.append(traj_obj) self.smac_history["data"].append(( [ len(self.traj) + 1, # config id None, # instance id 0, # seed... history[-2], # budget ], [ history[1], # cost / fitness history[2], # time / cost { # status "__enum__": "StatusType.SUCCESS" }, time.time() - history[2], # start time time.time(), # end time, history[-1] # info ])) self.smac_history["config_origins"][str(len(self.traj) + 1)] = "DEHB" config = ConfigSpace.Configuration(self.cs, vector=history[0]).get_dictionary() for key in list(config.keys()): if key != "agent": config[key[2:]] = config.pop(key) self.smac_history["configs"][str(len(self.traj) + 1)] = config self.traj.append(traj[0]) self.runtime.append(runtime) self.history.append(history)
def tuple_to_config( self, config_tpl: tuple, as_dict: bool = False, keys=None) -> \ Union[CS.Configuration, dict]: if keys is None: keys = self.keys_sorted config = dict(zip(keys, config_tpl)) if not as_dict: config = CS.Configuration(self.config_space, values=config) return config
def remove_resource( self, config_ext: CS.Configuration, as_dict: bool=False) -> Union[CS.Configuration, dict]: x_dct = copy.copy(config_ext.get_dictionary()) del x_dct[self.resource_attr_name] if as_dict: return x_dct else: return CS.Configuration(self.hp_ranges.config_space, values=x_dct)
def compute(self, config, budget, *args, **kwargs): cfg = CS.Configuration(cs, values=config) loss, info = objective_function(cfg, epoch=int(budget)) return ({ 'loss': loss, 'info': {"runtime": info["cost"], "lc": info["learning_curve"]} })
def get_config_dictionary(x, cs): config = dict() for i, hp_value in enumerate(x): if isinstance(cs.get_hyperparameter(cs.get_hyperparameter_by_idx(i)), CS.hyperparameters.CategoricalHyperparameter): x[i] = int(np.rint(x[i])) config = CS.Configuration(cs, vector=x).get_dictionary() # config[cs.get_hyperparameter_by_idx(i)] = cs.get_hyperparameter(cs.get_hyperparameter_by_idx(i))._transform(x[i]) return config
def objective_function(config): config_copy = deepcopy(config) c = ConfigSpace.Configuration(cs, values=config_copy) y, cost = search_space.objective_function(nasbench, c, budget=108) return { 'config': config_copy, 'loss': 1 - float(y), 'cost': cost, 'status': STATUS_OK }
def get(self, config: CS.Configuration, resource: int) -> CS.Configuration: """ Create extended config with resource added. :param config: :param resource: :return: Extended config """ values = copy.deepcopy(config.get_dictionary()) values[self.resource_attr_name] = resource return CS.Configuration(self.hp_ranges_ext.config_space, values=values)
def get_accuracy(self, sample): # return test_accuracy of a sample config = ConfigSpace.Configuration( self.space.get_configuration_space(), vector=sample) adjacency_matrix, node_list = self.space.convert_config_to_nasbench_format( config) node_list = [INPUT, *node_list, OUTPUT] if self.space.search_space_number == 3 else [INPUT, *node_list, CONV1X1, OUTPUT] adjacency_list = adjacency_matrix.astype(np.int).tolist() model_spec = api.ModelSpec(matrix=adjacency_list, ops=node_list) nasbench_data = self.nasbench.query(model_spec, epochs=self.budget) return nasbench_data['test_accuracy']
def select_new_parent(self, budget): assert len( self.init_compute_queue ) == 0, 'A new parent should only be chosen once the compute queue is empty.' # wrap the trained architectures in the current population with the # Baselearner class population_baselearners = { x.config_id: load_baselearner(model_id=model_seeds(x.config_id, x.config_id, self.scheme), load_nn_module=False, working_dir=x.info['dest_dir']) for x in self.population } # randomly choose one severity from self.severity_list to evaluate the # baselearners when running ForwardSelect severity = random.choice(self.severity_list) # select the ensemple of size self.pop_sample_size based on the ensemble # selection algorithm (esa), i.e. in this case ForwardSelect pop_sample = run_esa(M=self.pop_sample_size, population=population_baselearners, esa=esas_registry[self.esa], val_severity=severity) self.logger.info('ESA: {}'.format(pop_sample)) # append current population to history and save that self.history.append(list(population_baselearners.keys())) with open(os.path.join(self.working_dir, 'history.json'), 'w') as g: json.dump(self.history, g) # select one random model in pop_sample (selected ensemble from esa) parent_id = random.choice(pop_sample) parent_baselearner = list( filter(lambda x: x.config_id == parent_id, self.population)) assert len(parent_baselearner) == 1 parent_baselearner = parent_baselearner[0] # wrap the parent_baselearner dict with a ConfigSpace.Configuration # object (needed for hpbandster and the mutations) parent_config = ConfigSpace.Configuration( self.configspace, parent_baselearner.info['config']) # mutate the parent configuration and return the child. # self.configspace is the search space object mutated_config_dict = self.mutate_arch(self.configspace, parent_config).get_dictionary() # Evaluate the mutated parent next return mutated_config_dict
def genotype(self, theta): sample = np.argmax(theta, axis=1) config = ConfigSpace.Configuration( self.search_space.get_configuration_space(), vector=sample) adjacency_matrix, node_list = self.search_space.convert_config_to_nasbench_format( config) if self.search_space.search_space_number == 3: node_list = [INPUT, *node_list, OUTPUT] else: node_list = [INPUT, *node_list, CONV1X1, OUTPUT] result = "adjacency_matrix:" + str( adjacency_matrix) + "node_list:" + str(node_list) return result
def from_dict(self, config_dct: dict) -> CS.Configuration: """ Converts dict into CS.Configuration config (extended or normal, depending on whether the dict contains a resource attribute). :param config_dct: :return: """ # Note: Here, the key for resource is resource_attr_key, not # resource_attr_name hp_ranges = self.hp_ranges_ext if self.resource_attr_key in config_dct \ else self.hp_ranges return CS.Configuration(hp_ranges.config_space, values=config_dct)
def compute(self, config, budget, **kwargs): if self.config_as_array: c = CS.Configuration(self.configspace, values=config) else: c = config kwargs = {self.budget_name: self.budget_preprocessor(budget)} res = self.benchmark.objective_function(c, **kwargs) if self.measure_test_loss: del kwargs[self.budget_name] res['test_loss'] = self.benchmark.objective_function_test( c, **kwargs)['function_value'] return ({'loss': res['function_value'], 'info': res})
def wrapper(self, configuration: Union[np.ndarray, ConfigSpace.Configuration, Dict], **kwargs): try: if isinstance(configuration, np.ndarray): config_dict = {k: configuration[i] for (i, k) in enumerate(self.configuration_space)} config = ConfigSpace.Configuration(self.configuration_space, config_dict) elif isinstance(configuration, dict): config = ConfigSpace.Configuration(self.configuration_space, configuration) elif isinstance(configuration, ConfigSpace.Configuration): config = configuration else: config = None except Exception as e: logger.error('Error during the conversion of the provided configuration ' 'into a ConfigSpace.Configuration object') raise e if config is None: raise TypeError(f'Configuration has to be from type np.ndarray, dict, or ConfigSpace.Configuration but ' f'was {type(configuration)}') self.configuration_space.check_configuration(config) return foo(self, configuration, **kwargs)
def split(self, config_ext: CS.Configuration, as_dict: bool=False) -> \ (Union[CS.Configuration, dict], int): """ Split extended config into normal config and resource value. :param config_ext: Extended config :param as_dict: Return config as dict? :return: (config, resource_value) """ x_res = copy.copy(config_ext.get_dictionary()) resource_value = int(x_res[self.resource_attr_name]) del x_res[self.resource_attr_name] if not as_dict: x_res = CS.Configuration(self.hp_ranges.config_space, values=x_res) return x_res, resource_value
def Eval_nasbench1shot1(theta, search_space, logger): nasbench1shot1_path = 'benchmark/nasbench_full.tfrecord' nasbench = api.NASBench(nasbench1shot1_path) current_best = np.argmax(theta, axis=1) config = ConfigSpace.Configuration( search_space.search_space.get_configuration_space(), vector=current_best) adjacency_matrix, node_list = search_space.search_space.convert_config_to_nasbench_format( config) node_list = [INPUT, *node_list, OUTPUT] if search_space.search_space.search_space_number == 3 else [ INPUT, *node_list, CONV1X1, OUTPUT] adjacency_list = adjacency_matrix.astype(np.int).tolist() model_spec = api.ModelSpec(matrix=adjacency_list, ops=node_list) nasbench_data = nasbench.query(model_spec, epochs=108) logger.info("test accuracy = {}".format(nasbench_data['test_accuracy']))
def _check_and_cast_configuration(configuration: Union[Dict, ConfigSpace.Configuration], configuration_space: ConfigSpace.ConfigurationSpace) \ -> ConfigSpace.Configuration: """ Helper-function to evaluate the given configuration. Cast it to a ConfigSpace.Configuration and evaluate if it violates some boundaries. """ if isinstance(configuration, dict): configuration = ConfigSpace.Configuration(configuration_space, configuration) elif isinstance(configuration, ConfigSpace.Configuration): configuration = configuration else: raise TypeError(f'Configuration has to be from type List, np.ndarray, dict, or ' f'ConfigSpace.Configuration but was {type(configuration)}') configuration_space.check_configuration(configuration) return configuration
def wrapper(self, configuration, **kwargs): if not isinstance(configuration, ConfigSpace.Configuration): try: squirtle = { k: configuration[i] for (i, k) in enumerate(self.configuration_space) } wartortle = ConfigSpace.Configuration( self.configuration_space, squirtle) except Exception as e: raise Exception( 'Error during the conversion of the provided ' 'into a ConfigSpace.Configuration object') from e else: wartortle = configuration self.configuration_space.check_configuration(wartortle) return (foo(self, configuration, **kwargs))