def fit_selector(self, scenario: ASlibScenario, config: Configuration): ''' fits an algorithm selector for a given scenario wrt a given configuration Arguments --------- scenario: aslib_scenario.aslib_scenario.ASlibScenario aslib scenario at hand config: Configuration parameter configuration ''' if config.get("selector") == "PairwiseClassifier": clf_class = None if config.get("classifier") == "RandomForest": clf_class = RandomForest if config.get("classifier") == "XGBoost": clf_class = XGBoost selector = PairwiseClassifier(classifier_class=clf_class) selector.fit(scenario=scenario, config=config) if config.get("selector") == "PairwiseRegressor": reg_class = None if config.get("regressor") == "RandomForestRegressor": reg_class = RandomForestRegressor selector = PairwiseRegression(regressor_class=reg_class) selector.fit(scenario=scenario, config=config) return selector
def create_configuration(self, values: Union[None, Dict[str, Union[str, float, int]]] = None, vector: Union[None, np.ndarray] = None) -> Configuration: """ Method that creates new BRISE Configuration using ConfigSpace notions of `vector` and `values`. If both, `vector` and `values` are provided - create from `vector`, ignore `values`. :param values: dict. Optional parameter. Should be set to get Configuration from pre-defined values :param vector: np.ndarray. Optional parameter. Should be set to get Configuration from set vector :return: sampled Configuration """ try: if vector is not None: cs_vector = [None]*len(self.get_hyperparameter_names()) for cs_idx, cs_param in enumerate(self._config_space.get_hyperparameter_names()): for idx, param in enumerate(self.get_hyperparameter_names()): if cs_param == param: cs_vector[cs_idx] = vector[idx] cs_configuration = CSConfiguration(self._config_space, vector=cs_vector) elif values is not None: cs_configuration = CSConfiguration(self._config_space, values=values) else: raise TypeError("A new Configuration creation requires either parameter 'vector' or parameter 'values'.") except ForbiddenValueError as error: self.logger.error(f"Tried to sample forbidden configuration: {error}") raise error # todo: its a work-around similar we had in original hh if cs_configuration.get("low level heuristic", "") == "jMetalPy.EvolutionStrategy": if cs_configuration['py.ES|lambda_'] < cs_configuration['py.ES|mu']: self.logger.warning(f"Hyperparameter 'lambda_' was altered from {cs_configuration['py.ES|lambda_']} to" f" {cs_configuration['py.ES|mu']}!") cs_configuration['py.ES|lambda_'] = cs_configuration['py.ES|mu'] parameters = [] for parameter_name in self.__hyperparameter_names: parameters.append(cs_configuration.get(parameter_name)) return Configuration(parameters, Configuration.Type.FROM_SELECTOR)
def _add_in_old_format(self, train_perf: float, incumbent_id: int, incumbent: Configuration, ta_time_used: float, wallclock_time: float): """Adds entries to old SMAC2-like trajectory file Parameters ---------- train_perf: float Estimated performance on training (sub)set incumbent_id: int Id of incumbent incumbent: Configuration() Current incumbent configuration ta_time_used: float CPU time used by the target algorithm wallclock_time: float Wallclock time used so far """ conf = [] for p in incumbent: if not incumbent.get(p) is None: conf.append("%s='%s'" % (p, repr(incumbent[p]))) txt = ("%f, %f, %f, %d, %f, %s\n" % ( ta_time_used, train_perf, wallclock_time, incumbent_id, wallclock_time - ta_time_used, ", ".join(conf) )) self.file_system.write_txt(self.old_traj_fn, txt, append=True)
def _add_in_aclib_format(self, train_perf: float, incumbent_id: int, incumbent: Configuration, ta_time_used: float, wallclock_time: float): """Adds entries to AClib2-like trajectory file Parameters ---------- train_perf: float Estimated performance on training (sub)set incumbent_id: int Id of incumbent incumbent: Configuration() Current incumbent configuration ta_time_used: float CPU time used by the target algorithm wallclock_time: float Wallclock time used so far """ conf = [] for p in incumbent: if not incumbent.get(p) is None: conf.append("%s='%s'" % (p, repr(incumbent[p]))) traj_entry = { "cpu_time": ta_time_used, "total_cpu_time": None, # TODO: fix this "wallclock_time": wallclock_time, "evaluations": self.stats.ta_runs, "cost": train_perf, "incumbent": conf }
def _add_in_aclib_format(self, train_perf: float, incumbent_id: int, incumbent: Configuration, ta_time_used: float, wallclock_time: float) -> None: """Adds entries to AClib2-like trajectory file Parameters ---------- train_perf: float Estimated performance on training (sub)set incumbent_id: int Id of incumbent incumbent: Configuration() Current incumbent configuration ta_time_used: float CPU time used by the target algorithm wallclock_time: float Wallclock time used so far """ conf = [] for p in incumbent: if not incumbent.get(p) is None: conf.append("%s='%s'" % (p, repr(incumbent[p]))) traj_entry = {"cpu_time": ta_time_used, "wallclock_time": wallclock_time, "evaluations": self.stats.ta_runs, "cost": train_perf, "incumbent": conf, "origin": incumbent.origin, } with open(self.aclib_traj_fn, "a") as fp: json.dump(traj_entry, fp) fp.write("\n")
def _add_in_old_format(self, train_perf: Union[float, np.ndarray], incumbent_id: int, incumbent: Configuration, ta_time_used: float, wallclock_time: float) -> None: """Adds entries to old SMAC2-like trajectory file Parameters ---------- train_perf: float or list of floats Estimated performance on training (sub)set incumbent_id: int Id of incumbent incumbent: Configuration() Current incumbent configuration ta_time_used: float CPU time used by the target algorithm wallclock_time: float Wallclock time used so far """ conf = [] for p in incumbent: if not incumbent.get(p) is None: conf.append("%s='%s'" % (p, repr(incumbent[p]))) if isinstance(train_perf, float): # Make it compatible with old format with open(self.old_traj_fn, "a") as fp: fp.write( f"{ta_time_used:f}, {train_perf:f}, {wallclock_time:f}, {incumbent_id:d}, " f"{wallclock_time - ta_time_used:f}, {','.join(conf):s}\n") else: # We recommend to use pandas to read this csv file with open(self.old_traj_fn, "a") as fp: fp.write( f"{ta_time_used:f}, {train_perf}, {wallclock_time:f}, {incumbent_id:d}, " f"{wallclock_time - ta_time_used:f}, {','.join(conf):s}\n")
def fit_selector(self, scenario: ASlibScenario, config: Configuration): ''' fits an algorithm selector for a given scenario wrt a given configuration Arguments --------- scenario: autofolio.data.aslib_scenario.ASlibScenario aslib scenario at hand config: Configuration parameter configuration ''' if config.get("selector") == "PairwiseClassifier": clf_class = None if config.get("classifier") == "RandomForest": clf_class = RandomForest selector = PairwiseClassifier(classifier_class=clf_class) selector.fit(scenario=scenario, config=config) return selector
def _add_in_aclib_format(self, train_perf: float, incumbent_id: int, incumbent: Configuration, ta_time_used: float, wallclock_time: float): """Adds entries to AClib2-like trajectory file Parameters ---------- train_perf: float Estimated performance on training (sub)set incumbent_id: int Id of incumbent incumbent: Configuration() Current incumbent configuration ta_time_used: float CPU time used by the target algorithm wallclock_time: float Wallclock time used so far """ conf = [] for p in incumbent: if not incumbent.get(p) is None: conf.append("%s='%s'" % (p, repr(incumbent[p]))) traj_entry = { "cpu_time": ta_time_used, "total_cpu_time": None, # TODO: fix this "wallclock_time": wallclock_time, "evaluations": self.stats.ta_runs, "cost": train_perf, "incumbent": conf } try: traj_entry["origin"] = incumbent.origin except AttributeError: traj_entry["origin"] = "UNKNOWN" txt = json.dumps(traj_entry) + "\n" self.file_system.write_txt(self.aclib_traj_fn, txt, True)