def fit_selector(self, scenario: ASlibScenario, config: Configuration):
'''
fits an algorithm selector for a given scenario wrt a given configuration
Arguments
---------
scenario: aslib_scenario.aslib_scenario.ASlibScenario
aslib scenario at hand
config: Configuration
parameter configuration
'''
if config.get("selector") == "PairwiseClassifier":
clf_class = None
if config.get("classifier") == "RandomForest":
clf_class = RandomForest
if config.get("classifier") == "XGBoost":
clf_class = XGBoost
selector = PairwiseClassifier(classifier_class=clf_class)
selector.fit(scenario=scenario, config=config)
if config.get("selector") == "PairwiseRegressor":
reg_class = None
if config.get("regressor") == "RandomForestRegressor":
reg_class = RandomForestRegressor
selector = PairwiseRegression(regressor_class=reg_class)
selector.fit(scenario=scenario, config=config)
return selector
def create_configuration(self, values: Union[None, Dict[str, Union[str, float, int]]] = None,
vector: Union[None, np.ndarray] = None) -> Configuration:
"""
Method that creates new BRISE Configuration using ConfigSpace notions of `vector` and `values`.
If both, `vector` and `values` are provided - create from `vector`, ignore `values`.
:param values: dict. Optional parameter. Should be set to get Configuration from pre-defined values
:param vector: np.ndarray. Optional parameter. Should be set to get Configuration from set vector
:return: sampled Configuration
"""
try:
if vector is not None:
cs_vector = [None]*len(self.get_hyperparameter_names())
for cs_idx, cs_param in enumerate(self._config_space.get_hyperparameter_names()):
for idx, param in enumerate(self.get_hyperparameter_names()):
if cs_param == param:
cs_vector[cs_idx] = vector[idx]
cs_configuration = CSConfiguration(self._config_space, vector=cs_vector)
elif values is not None:
cs_configuration = CSConfiguration(self._config_space, values=values)
else:
raise TypeError("A new Configuration creation requires either parameter 'vector' or parameter 'values'.")
except ForbiddenValueError as error:
self.logger.error(f"Tried to sample forbidden configuration: {error}")
raise error
# todo: its a work-around similar we had in original hh
if cs_configuration.get("low level heuristic", "") == "jMetalPy.EvolutionStrategy":
if cs_configuration['py.ES|lambda_'] < cs_configuration['py.ES|mu']:
self.logger.warning(f"Hyperparameter 'lambda_' was altered from {cs_configuration['py.ES|lambda_']} to"
f" {cs_configuration['py.ES|mu']}!")
cs_configuration['py.ES|lambda_'] = cs_configuration['py.ES|mu']
parameters = []
for parameter_name in self.__hyperparameter_names:
parameters.append(cs_configuration.get(parameter_name))
return Configuration(parameters, Configuration.Type.FROM_SELECTOR)
def _add_in_old_format(self, train_perf: float, incumbent_id: int,
incumbent: Configuration, ta_time_used: float,
wallclock_time: float):
"""Adds entries to old SMAC2-like trajectory file
Parameters
----------
train_perf: float
Estimated performance on training (sub)set
incumbent_id: int
Id of incumbent
incumbent: Configuration()
Current incumbent configuration
ta_time_used: float
CPU time used by the target algorithm
wallclock_time: float
Wallclock time used so far
"""
conf = []
for p in incumbent:
if not incumbent.get(p) is None:
conf.append("%s='%s'" % (p, repr(incumbent[p])))
txt = ("%f, %f, %f, %d, %f, %s\n" % (
ta_time_used,
train_perf,
wallclock_time,
incumbent_id,
wallclock_time - ta_time_used,
", ".join(conf)
))
self.file_system.write_txt(self.old_traj_fn, txt, append=True)
def _add_in_aclib_format(self, train_perf: float, incumbent_id: int,
incumbent: Configuration, ta_time_used: float,
wallclock_time: float):
"""Adds entries to AClib2-like trajectory file
Parameters
----------
train_perf: float
Estimated performance on training (sub)set
incumbent_id: int
Id of incumbent
incumbent: Configuration()
Current incumbent configuration
ta_time_used: float
CPU time used by the target algorithm
wallclock_time: float
Wallclock time used so far
"""
conf = []
for p in incumbent:
if not incumbent.get(p) is None:
conf.append("%s='%s'" % (p, repr(incumbent[p])))
traj_entry = {
"cpu_time": ta_time_used,
"total_cpu_time": None, # TODO: fix this
"wallclock_time": wallclock_time,
"evaluations": self.stats.ta_runs,
"cost": train_perf,
"incumbent": conf
}
def _add_in_aclib_format(self, train_perf: float, incumbent_id: int,
incumbent: Configuration, ta_time_used: float,
wallclock_time: float) -> None:
"""Adds entries to AClib2-like trajectory file
Parameters
----------
train_perf: float
Estimated performance on training (sub)set
incumbent_id: int
Id of incumbent
incumbent: Configuration()
Current incumbent configuration
ta_time_used: float
CPU time used by the target algorithm
wallclock_time: float
Wallclock time used so far
"""
conf = []
for p in incumbent:
if not incumbent.get(p) is None:
conf.append("%s='%s'" % (p, repr(incumbent[p])))
traj_entry = {"cpu_time": ta_time_used,
"wallclock_time": wallclock_time,
"evaluations": self.stats.ta_runs,
"cost": train_perf,
"incumbent": conf,
"origin": incumbent.origin,
}
with open(self.aclib_traj_fn, "a") as fp:
json.dump(traj_entry, fp)
fp.write("\n")
def _add_in_old_format(self, train_perf: Union[float, np.ndarray],
incumbent_id: int, incumbent: Configuration,
ta_time_used: float, wallclock_time: float) -> None:
"""Adds entries to old SMAC2-like trajectory file
Parameters
----------
train_perf: float or list of floats
Estimated performance on training (sub)set
incumbent_id: int
Id of incumbent
incumbent: Configuration()
Current incumbent configuration
ta_time_used: float
CPU time used by the target algorithm
wallclock_time: float
Wallclock time used so far
"""
conf = []
for p in incumbent:
if not incumbent.get(p) is None:
conf.append("%s='%s'" % (p, repr(incumbent[p])))
if isinstance(train_perf, float):
# Make it compatible with old format
with open(self.old_traj_fn, "a") as fp:
fp.write(
f"{ta_time_used:f}, {train_perf:f}, {wallclock_time:f}, {incumbent_id:d}, "
f"{wallclock_time - ta_time_used:f}, {','.join(conf):s}\n")
else:
# We recommend to use pandas to read this csv file
with open(self.old_traj_fn, "a") as fp:
fp.write(
f"{ta_time_used:f}, {train_perf}, {wallclock_time:f}, {incumbent_id:d}, "
f"{wallclock_time - ta_time_used:f}, {','.join(conf):s}\n")
def fit_selector(self, scenario: ASlibScenario, config: Configuration):
'''
fits an algorithm selector for a given scenario wrt a given configuration
Arguments
---------
scenario: autofolio.data.aslib_scenario.ASlibScenario
aslib scenario at hand
config: Configuration
parameter configuration
'''
if config.get("selector") == "PairwiseClassifier":
clf_class = None
if config.get("classifier") == "RandomForest":
clf_class = RandomForest
selector = PairwiseClassifier(classifier_class=clf_class)
selector.fit(scenario=scenario, config=config)
return selector
def _add_in_aclib_format(self, train_perf: float, incumbent_id: int,
incumbent: Configuration, ta_time_used: float,
wallclock_time: float):
"""Adds entries to AClib2-like trajectory file
Parameters
----------
train_perf: float
Estimated performance on training (sub)set
incumbent_id: int
Id of incumbent
incumbent: Configuration()
Current incumbent configuration
ta_time_used: float
CPU time used by the target algorithm
wallclock_time: float
Wallclock time used so far
"""
conf = []
for p in incumbent:
if not incumbent.get(p) is None:
conf.append("%s='%s'" % (p, repr(incumbent[p])))
traj_entry = {
"cpu_time": ta_time_used,
"total_cpu_time": None, # TODO: fix this
"wallclock_time": wallclock_time,
"evaluations": self.stats.ta_runs,
"cost": train_perf,
"incumbent": conf
}
try:
traj_entry["origin"] = incumbent.origin
except AttributeError:
traj_entry["origin"] = "UNKNOWN"
txt = json.dumps(traj_entry) + "\n"
self.file_system.write_txt(self.aclib_traj_fn, txt, True)