def get_configurations(
self,
runhistory: RunHistory,
budget_subset: typing.Optional[typing.List] = None,
) -> np.ndarray:
"""Returns vector representation of only the configurations.
Instance features are not appended and cost values are not taken into account.
Parameters
----------
runhistory : smac.runhistory.runhistory.RunHistory
Runhistory containing all evaluated configurations/instances
budget_subset : list of budgets to consider
Returns
-------
numpy.ndarray
"""
s_runs = self._get_s_run_dict(runhistory, budget_subset)
s_config_ids = set(s_run.config_id for s_run in s_runs)
t_runs = self._get_t_run_dict(runhistory, budget_subset)
t_config_ids = set(t_run.config_id for t_run in t_runs)
config_ids = s_config_ids | t_config_ids
configurations = [
runhistory.ids_config[config_id] for config_id in config_ids
]
configs_array = convert_configurations_to_array(configurations)
return configs_array
def _get_initial_points(
self,
num_points: int,
runhistory: RunHistory,
additional_start_points: Optional[List[Tuple[float, Configuration]]],
) -> List[Configuration]:
if runhistory.empty():
init_points = self.config_space.sample_configuration(
size=num_points)
else:
# initiate local search
configs_previous_runs = runhistory.get_all_configs()
# configurations with the highest previous EI
configs_previous_runs_sorted = self._sort_configs_by_acq_value(
configs_previous_runs)
configs_previous_runs_sorted = [
conf[1] for conf in configs_previous_runs_sorted[:num_points]
]
# configurations with the lowest predictive cost, check for None to make unit tests work
if self.acquisition_function.model is not None:
conf_array = convert_configurations_to_array(
configs_previous_runs)
costs = self.acquisition_function.model.predict_marginalized_over_instances(
conf_array)[0]
# From here
# http://stackoverflow.com/questions/20197990/how-to-make-argsort-result-to-be-random-between-equal-values
random = self.rng.rand(len(costs))
# Last column is primary sort key!
indices = np.lexsort((random.flatten(), costs.flatten()))
# Cannot use zip here because the indices array cannot index the
# rand_configs list, because the second is a pure python list
configs_previous_runs_sorted_by_cost = [
configs_previous_runs[ind] for ind in indices
][:num_points]
else:
configs_previous_runs_sorted_by_cost = []
if additional_start_points is not None:
additional_start_points = [
asp[1] for asp in additional_start_points[:num_points]
]
else:
additional_start_points = []
init_points = []
init_points_as_set = set() # type: Set[Configuration]
for cand in itertools.chain(
configs_previous_runs_sorted,
configs_previous_runs_sorted_by_cost,
additional_start_points,
):
if cand not in init_points_as_set:
init_points.append(cand)
init_points_as_set.add(cand)
return init_points
def new_result(self, job, config_info): # pylint: disable=unused-argument
if job.exception is not None:
self.logger.warning(f"job {job.id} failed with exception\n{job.exception}")
if job.result is None:
# One could skip crashed results, but we decided to
# assign a +inf loss and count them as bad configurations
loss = np.inf
else:
# same for non numeric losses.
# Note that this means losses of minus infinity will count as bad!
loss = job.result["loss"] if np.isfinite(job.result["loss"]) else np.inf
config = ConfigSpace.Configuration(self.configspace, job.kwargs["config"])
self.configs.append(config)
self.losses.append(loss)
if self.has_model:
# TODO: include old
X = convert_configurations_to_array(self.configs)
Y = np.array(self.losses, dtype=np.float64)
self.model.train(X, Y)
self.acquisition_func.update(
model=self.model,
eta=min(self.losses),
)
def _sort_configs_by_acq_value(self, configs):
"""Sort the given configurations by acquisition value
Parameters
----------
configs : list(Configuration)
Returns
-------
list: (acquisition value, Candidate solutions),
ordered by their acquisition function value
"""
config_array = convert_configurations_to_array(configs)
acq_values = self.acquisition_func(config_array)
# From here
# http://stackoverflow.com/questions/20197990/how-to-make-argsort-result-to-be-random-between-equal-values
random = self.rng.rand(len(acq_values))
# Last column is primary sort key!
indices = np.lexsort((random.flatten(), acq_values.flatten()))
# Cannot use zip here because the indices array cannot index the
# rand_configs list, because the second is a pure python list
return [(acq_values[ind][0], configs[ind]) for ind in indices[::-1]]
def _train(self, X: np.ndarray, Y: np.ndarray) -> AbstractEPM:
meta_data = dict()
for id_ in self.training_data:
configs = self.training_data[id_]['configurations']
X_ = convert_configurations_to_array(configs)
X_ = self._preprocess(X_)
meta_data[id_] = (
X_,
self.training_data[id_]['y'].flatten(),
None,
)
X = self._preprocess(X)
for i in range(10):
try:
if self.nn is None:
self.nn = Net(
num_tasks=len(self.training_data) + 1,
n_attributes=X.shape[1],
meta_data=meta_data,
alpha_min=alpha_min,
alpha_max=alpha_max,
beta_min=beta_min,
beta_max=beta_max,
)
self.nn.train(X, Y)
break
except Exception as e:
print('Training failed %d/%d!' % (i + 1, 10))
print(e)
self.nn = None
return self
def _build_matrix(self,
run_dict: typing.Mapping[RunKey, RunValue],
runhistory: RunHistory,
instances: typing.List[str] = None,
return_time_as_y: bool = False,
store_statistics: bool = False):
"""TODO"""
if return_time_as_y:
raise NotImplementedError()
if store_statistics:
raise NotImplementedError()
# First build nan-matrix of size #configs x #params+1
n_rows = len(run_dict)
n_cols = self.num_params
X = np.ones([n_rows, n_cols + self.n_feats]) * np.nan
y = np.ones([n_rows, 2])
# Then populate matrix
for row, (key, run) in enumerate(run_dict.items()):
# Scaling is automatically done in configSpace
conf = runhistory.ids_config[key.config_id]
conf_vector = convert_configurations_to_array([conf])[0]
if self.n_feats:
feats = self.instance_features[key.instance_id]
X[row, :] = np.hstack((conf_vector, feats))
else:
X[row, :] = conf_vector
y[row, 0] = run.cost
y[row, 1] = 1 + run.time
y = self.transform_response_values(values=y)
return X, y
def _build_matrix(self,
run_dict: typing.Mapping[RunKey, RunValue],
runhistory: RunHistory,
instances: typing.List[str] = None,
par_factor: int = 1):
"""TODO"""
# First build nan-matrix of size #configs x #params+1
n_rows = len(run_dict)
n_cols = self.num_params
X = np.ones([n_rows, n_cols + self.n_feats]) * np.nan
Y = np.ones([n_rows, 2])
# Then populate matrix
for row, (key, run) in enumerate(run_dict.items()):
# Scaling is automatically done in configSpace
conf = runhistory.ids_config[key.config_id]
conf_vector = convert_configurations_to_array([conf])[0]
if self.n_feats:
feats = self.instance_features[key.instance_id]
X[row, :] = np.hstack((conf_vector, feats))
else:
X[row, :] = conf_vector
# run_array[row, -1] = instances[row]
Y[row, 0] = run.cost
Y[row, 1] = np.log(1 + run.time)
return X, Y
def _build_matrix(
self,
run_dict: typing.Mapping[RunKey, RunValue],
runhistory: RunHistory,
instances: list = None,
return_time_as_y: bool = False,
store_statistics: bool = False
) -> typing.Tuple[np.ndarray, np.ndarray]:
""""Builds X,y matrixes from selected runs from runhistory
Parameters
----------
run_dict: dict: RunKey -> RunValue
dictionary from RunHistory.RunKey to RunHistory.RunValue
runhistory: RunHistory
runhistory object
instances: list
list of instances
return_time_as_y: bool
Return the time instead of cost as y value. Necessary to access the raw y values for imputation.
store_statistics: bool
Whether to store statistics about the data (to be used at subsequent calls)
Returns
-------
X: np.ndarray
Y: np.ndarray
"""
# First build nan-matrix of size #configs x #params+1
n_rows = len(run_dict)
n_cols = self.num_params
X = np.ones([n_rows, n_cols + self.n_feats]) * np.nan
y = np.ones([n_rows, 1])
# Then populate matrix
for row, (key, run) in enumerate(run_dict.items()):
# Scaling is automatically done in configSpace
conf = runhistory.ids_config[key.config_id]
conf_vector = convert_configurations_to_array([conf])[0]
if self.n_feats:
feats = self.instance_features[key.instance_id]
X[row, :] = np.hstack((conf_vector, feats))
else:
X[row, :] = conf_vector
# run_array[row, -1] = instances[row]
if return_time_as_y:
y[row, 0] = run.time
else:
y[row, 0] = run.cost
if y.size > 0:
if store_statistics:
self.perc = np.percentile(y, self.scale_perc)
self.min_y = np.min(y)
self.max_y = np.max(y)
y = self.transform_response_values(values=y)
return X, y
def _build_matrix(self,
run_dict: typing.Mapping[RunKey, RunValue],
runhistory: RunHistory,
instances: typing.List[str] = None,
par_factor: int = 1):
""""Builds X,y matrixes from selected runs from runhistory
Parameters
----------
run_dict: dict: RunKey -> RunValue
dictionary from RunHistory.RunKey to RunHistory.RunValue
runhistory: RunHistory
runhistory object
instances: list
list of instances
par_factor: int
penalization factor for censored runtime data
Returns
-------
X: np.ndarray
Y: np.ndarray
"""
# First build nan-matrix of size #configs x #params+1
n_rows = len(run_dict)
n_cols = self.num_params
X = np.ones([n_rows, n_cols + self.n_feats]) * np.nan
y = np.ones([n_rows, 1])
# Then populate matrix
for row, (key, run) in enumerate(run_dict.items()):
# Scaling is automatically done in configSpace
conf = runhistory.ids_config[key.config_id]
conf_vector = convert_configurations_to_array([conf])[0]
if self.n_feats:
feats = self.instance_features[key.instance_id]
X[row, :] = np.hstack((conf_vector, feats))
else:
X[row, :] = conf_vector
# run_array[row, -1] = instances[row]
if self.scenario.run_obj == "runtime":
if run.status != StatusType.SUCCESS:
y[row, 0] = run.time * par_factor
else:
y[row, 0] = run.time
else:
y[row, 0] = run.cost
return X, y
def _build_matrix(
self,
run_dict: typing.Mapping[RunKey, RunValue],
runhistory: RunHistory,
return_time_as_y: bool = False,
store_statistics: bool = False,
) -> typing.Tuple[np.ndarray, np.ndarray]:
"""TODO"""
if return_time_as_y:
raise NotImplementedError()
if store_statistics:
# store_statistics is currently not necessary
pass
# First build nan-matrix of size #configs x #params+1
n_rows = len(run_dict)
n_cols = self.num_params
X = np.ones([n_rows, n_cols + self.n_feats]) * np.nan
y = np.ones([n_rows, 2])
# Then populate matrix
for row, (key, run) in enumerate(run_dict.items()):
# Scaling is automatically done in configSpace
conf = runhistory.ids_config[key.config_id]
conf_vector = convert_configurations_to_array([conf])[0]
if self.n_feats:
feats = self.instance_features[key.instance_id]
X[row, :] = np.hstack((conf_vector, feats))
else:
X[row, :] = conf_vector
if self.num_obj > 1:
assert self.multi_objective_algorithm is not None
# Let's normalize y here
# We use the objective_bounds calculated by the runhistory
y_ = normalize_costs([run.cost], runhistory.objective_bounds)
y_ = self.multi_objective_algorithm(y_)
y[row, 0] = y_
else:
y[row, 0] = run.cost
y[row, 1] = 1 + run.time
y = self.transform_response_values(values=y)
return X, y
def predict_perf_of_traj(traj, smac: SMAC):
'''
predict the performance of all entries in the trajectory
marginalized across all instances
Arguments
---------
smac: SMAC()
SMAC Facade object
traj: typing.List
list of trajectory entries (dictionaries)
Returns
-------
perfs: typing.List[float]
list of performance values
time_stamps: typing.List[float]
list of time stamps -- in the same order as perfs
'''
logger.info("Predict performance of %d entries in trajectory." %
(len(traj)))
time_stamps = []
perfs = []
for entry in traj:
config = entry["incumbent"]
wc_time = entry["wallclock_time"]
config_array = convert_configurations_to_array([config])
m, v = smac.solver.model.predict_marginalized_over_instances(
X=config_array)
if smac.solver.scenario.run_obj == "runtime":
p = 10**m[0, 0]
else:
p = m[0, 0]
perfs.append(p)
time_stamps.append(wc_time)
return perfs, time_stamps
def __init__(
self,
configspace,
random_fraction=1 / 2,
logger=None,
configs=None,
losses=None,
):
self.logger = logger
self.random_fraction = random_fraction
self.configspace = configspace
self.min_points_in_model = len(self.configspace.get_hyperparameters())
rng = np.random.RandomState(random.randint(0, 100))
self.model = _construct_model(configspace, rng)
self.acquisition_func = EI(model=self.model)
self.acq_optimizer = LocalSearch(
acquisition_function=self.acquisition_func,
config_space=configspace,
rng=rng)
self.runhistory = RunHistory()
self.configs = configs or list()
self.losses = losses or list()
if self.has_model:
for config, cost in zip(self.configs, self.losses):
self.runhistory.add(config, cost, 0, StatusType.SUCCESS)
X = convert_configurations_to_array(self.configs)
Y = np.array(self.losses, dtype=np.float64)
self.model.train(X, Y)
self.acquisition_func.update(
model=self.model,
eta=min(self.losses),
)
def test_impute_inactive_hyperparameters(self):
cs = ConfigurationSpace()
a = cs.add_hyperparameter(CategoricalHyperparameter('a', [0, 1]))
b = cs.add_hyperparameter(CategoricalHyperparameter('b', [0, 1]))
c = cs.add_hyperparameter(UniformFloatHyperparameter('c', 0, 1))
cs.add_condition(EqualsCondition(b, a, 1))
cs.add_condition(EqualsCondition(c, a, 0))
cs.seed(1)
configs = cs.sample_configuration(size=100)
config_array = convert_configurations_to_array(configs)
for line in config_array:
if line[0] == 0:
self.assertTrue(np.isnan(line[1]))
elif line[0] == 1:
self.assertTrue(np.isnan(line[2]))
gp = get_gp(3, np.random.RandomState(1))
config_array = gp._impute_inactive(config_array)
for line in config_array:
if line[0] == 0:
self.assertEqual(line[1], -1)
elif line[0] == 1:
self.assertEqual(line[2], -1)
def local_search(self, start_point: Configuration):
"""Starts a local search from the given startpoint and quits
if either the max number of steps is reached or no neighbor
with an higher improvement was found.
Parameters:
----------
start_point: Configuration
The point from where the local search starts
Returns:
-------
incumbent: Configuration
The best found configuration
"""
self.intensifier.minR = self.fast_race_minR # be aggressive here!
self.intensifier.Adaptive_Capping_Slackfactor = self.fast_race_adaptive_capping_factor
incumbent = start_point
local_search_steps = 0
neighbors_looked_at = 0
time_n = []
while True:
local_search_steps += 1
# Get neighborhood of the current incumbent
# by randomly drawing configurations
changed_inc = False
# Get one exchange neighborhood returns an iterator (in contrast of
# the previously returned list).
all_neighbors = list(
get_one_exchange_neighbourhood(incumbent,
seed=self.rng.seed()))
neighbors_array = convert_configurations_to_array(all_neighbors)
acq_val = self.acquisition_func(neighbors_array)
sorted_neighbors = sorted(zip(all_neighbors, acq_val),
key=lambda x: x[1],
reverse=True)
prev_incumbent = incumbent
for neighbor in all_neighbors[:self.max_neighbors]:
neighbors_looked_at += 1
neighbor.origin = "SLS"
self.logger.debug("Intensify")
incumbent, inc_perf = self.intensifier.intensify(
challengers=[neighbor],
incumbent=incumbent,
run_history=self.runhistory,
aggregate_func=self.aggregate_func,
time_bound=0.01,
log_traj=False)
# first improvement SLS
if incumbent != prev_incumbent:
changed_inc = True
break
if not changed_inc:
self.logger.info(
"Local search took %d steps and looked at %d configurations."
% (local_search_steps, neighbors_looked_at))
break
return incumbent
def validate_epm(
self,
config_mode: Union[str, typing.List[Configuration]] = 'def',
instance_mode: Union[str, typing.List[str]] = 'test',
repetitions: int = 1,
runhistory: typing.Optional[RunHistory] = None,
output_fn: typing.Optional[str] = None,
reuse_epm: bool = True,
) -> RunHistory:
"""
Use EPM to predict costs/runtimes for unknown config/inst-pairs.
side effect: if output is specified, saves runhistory to specified
output directory.
Parameters
----------
output_fn: str
path to runhistory to be saved. if the suffix is not '.json', will
be interpreted as directory and filename will be
'validated_runhistory_EPM.json'
config_mode: str or list<Configuration>
string or directly a list of Configuration, string from [def, inc, def+inc, wallclock_time, cpu_time, all].
time evaluates at cpu- or wallclock-timesteps of:
[max_time/2^0, max_time/2^1, max_time/2^3, ..., default] with max_time being the highest recorded time
instance_mode: str or list<str>
what instances to use for validation, either from
[train, test, train+test] or directly a list of instances
repetitions: int
number of repetitions in nondeterministic algorithms
runhistory: RunHistory
optional, RunHistory-object to reuse runs
reuse_epm: bool
if true (and if `self.epm`), reuse epm to validate runs
Returns
-------
runhistory: RunHistory
runhistory with predicted runs
"""
if not isinstance(runhistory, RunHistory) and (self.epm is None
or not reuse_epm):
raise ValueError(
"No runhistory specified for validating with EPM!")
elif not reuse_epm or self.epm is None:
# Create RandomForest
types, bounds = get_types(
self.scen.cs, self.scen.feature_array
) # type: ignore[attr-defined] # noqa F821
epm = RandomForestWithInstances(
configspace=self.scen.
cs, # type: ignore[attr-defined] # noqa F821
types=types,
bounds=bounds,
instance_features=self.scen.feature_array,
seed=self.rng.randint(MAXINT),
ratio_features=1.0,
)
# Use imputor if objective is runtime
imputor = None
impute_state = None
impute_censored_data = False
if self.scen.run_obj == 'runtime':
threshold = self.scen.cutoff * self.scen.par_factor # type: ignore[attr-defined] # noqa F821
imputor = RFRImputator(
rng=self.rng,
cutoff=self.scen.
cutoff, # type: ignore[attr-defined] # noqa F821
threshold=threshold,
model=epm)
impute_censored_data = True
impute_state = [StatusType.CAPPED]
success_states = [
StatusType.SUCCESS,
]
else:
success_states = [
StatusType.SUCCESS, StatusType.CRASHED, StatusType.MEMOUT
]
# Transform training data (from given rh)
rh2epm = RunHistory2EPM4Cost(
num_params=len(self.scen.cs.get_hyperparameters()
), # type: ignore[attr-defined] # noqa F821
scenario=self.scen,
rng=self.rng,
impute_censored_data=impute_censored_data,
imputor=imputor,
impute_state=impute_state,
success_states=success_states)
assert runhistory is not None # please mypy
X, y = rh2epm.transform(runhistory)
self.logger.debug("Training model with data of shape X: %s, y:%s",
str(X.shape), str(y.shape))
# Train random forest
epm.train(X, y)
else:
epm = typing.cast(RandomForestWithInstances, self.epm)
# Predict desired runs
runs, rh_epm = self._get_runs(config_mode, instance_mode, repetitions,
runhistory)
feature_array_size = len(self.scen.cs.get_hyperparameters()
) # type: ignore[attr-defined] # noqa F821
if self.scen.feature_array is not None:
feature_array_size += self.scen.feature_array.shape[1]
X_pred = np.empty((len(runs), feature_array_size))
for idx, run in enumerate(runs):
if self.scen.feature_array is not None and run.inst is not None:
X_pred[idx] = np.hstack([
convert_configurations_to_array([run.config])[0],
self.scen.feature_dict[run.inst]
])
else:
X_pred[idx] = convert_configurations_to_array([run.config])[0]
self.logger.debug("Predicting desired %d runs, data has shape %s",
len(runs), str(X_pred.shape))
y_pred = epm.predict(X_pred)
self.epm = epm
# Add runs to runhistory
for run, pred in zip(runs, y_pred[0]):
rh_epm.add(
config=run.config,
cost=float(pred),
time=float(pred),
status=StatusType.SUCCESS,
instance_id=run.inst,
seed=-1,
additional_info={"additional_info": "ESTIMATED USING EPM!"})
if output_fn:
self._save_results(rh_epm,
output_fn,
backup_fn="validated_runhistory_EPM.json")
return rh_epm
def _get_mean_var_time(self, validator, traj, use_epm, rh):
"""
Parameters
----------
validator: Validator
validator (smac-based)
traj: List[Configuraton]
trajectory to set in validator
use_epm: bool
validated or not (no need to use epm if validated)
rh: RunHistory
??
Returns
-------
mean, var
times: List[float]
times to plot (x-values)
configs
"""
# TODO kinda important: docstrings, what is this function doing?
if validator:
validator.traj = traj # set trajectory
time, configs = [], []
if use_epm and not self.block_epm:
for entry in traj:
time.append(entry["wallclock_time"])
configs.append(entry["incumbent"])
# self.logger.debug('Time: %d Runs: %d', time[-1], len(rh.get_runs_for_config(configs[-1])))
self.logger.debug(
"Using %d samples (%d distinct) from trajectory.", len(time),
len(set(configs)))
# Initialize EPM
if validator.epm: # not log as validator epm is trained on cost, not log cost
epm = validator.epm
else:
self.logger.debug(
"No EPM passed! Training new one from runhistory.")
# Train random forest and transform training data (from given rh)
# Not using validator because we want to plot uncertainties
rh2epm = RunHistory2EPM4Cost(num_params=len(
self.scenario.cs.get_hyperparameters()),
scenario=self.scenario)
X, y = rh2epm.transform(rh)
self.logger.debug(
"Training model with data of shape X: %s, y: %s",
str(X.shape), str(y.shape))
types, bounds = get_types(self.scenario.cs,
self.scenario.feature_array)
epm = RandomForestWithInstances(
self.scenario.cs,
types=types,
bounds=bounds,
seed=self.rng.randint(MAXINT),
instance_features=self.scenario.feature_array,
ratio_features=1.0)
epm.train(X, y)
config_array = convert_configurations_to_array(configs)
mean, var = epm.predict_marginalized_over_instances(config_array)
var = np.zeros(mean.shape)
# We don't want to show the uncertainty of the model but uncertainty over multiple optimizer runs
# This variance is computed in an outer loop.
else:
mean, var = [], []
for entry in traj:
#self.logger.debug(entry)
time.append(entry["wallclock_time"])
configs.append(entry["incumbent"])
costs = _cost(configs[-1], rh,
rh.get_runs_for_config(configs[-1]))
# self.logger.debug(len(costs), time[-1]
if not costs:
time.pop()
else:
mean.append(np.mean(costs))
var.append(0) # No variance over instances
mean, var = np.array(mean).reshape(-1, 1), np.array(var).reshape(
-1, 1)
return mean, var, time, configs
def maximize(self, start_point, *args):
"""
Starts a local search from the given startpoint and quits
if either the max number of steps is reached or no neighbor
with an higher improvement was found.
Parameters:
----------
start_point: np.array(1, D):
The point from where the local search starts
*args :
Additional parameters that will be passed to the
acquisition function
Returns:
-------
incumbent np.array(1, D):
The best found configuration
acq_val_incumbent np.array(1,1) :
The acquisition value of the incumbent
"""
incumbent = start_point
# Compute the acquisition value of the incumbent
incumbent_array = convert_configurations_to_array([incumbent])
acq_val_incumbent = self.acquisition_function(incumbent_array, *args)
local_search_steps = 0
neighbors_looked_at = 0
time_n = []
while True:
local_search_steps += 1
if local_search_steps % 1000 == 0:
self.logger.warn(
"Local search took already %d iterations."
"Is it maybe stuck in a infinite loop?",
local_search_steps)
# Get neighborhood of the current incumbent
# by randomly drawing configurations
changed_inc = False
# Get one exchange neighborhood returns an iterator (in contrast of
# the previously returned list).
all_neighbors = get_one_exchange_neighbourhood(
incumbent, seed=self.rng.seed())
for neighbor in all_neighbors:
s_time = time.time()
neighbor_array_ = convert_configurations_to_array([neighbor])
acq_val = self.acquisition_function(neighbor_array_, *args)
neighbors_looked_at += 1
time_n.append(time.time() - s_time)
if acq_val > acq_val_incumbent + self.epsilon:
self.logger.debug("Switch to one of the neighbors")
incumbent = neighbor
acq_val_incumbent = acq_val
changed_inc = True
break
if (not changed_inc) or (self.max_iterations != None
and local_search_steps
== self.max_iterations):
self.logger.debug(
"Local search took %d steps and looked at %d configurations. "
"Computing the acquisition value for one "
"configuration took %f seconds on average.",
local_search_steps, neighbors_looked_at, np.mean(time_n))
break
return incumbent, acq_val_incumbent
def _get_initial_points(
self,
num_points: int,
runhistory: RunHistory,
additional_start_points: Optional[List[Tuple[float, Configuration]]],
) -> List[Configuration]:
if runhistory.empty():
init_points = self.config_space.sample_configuration(
size=num_points)
else:
# initiate local search
configs_previous_runs = runhistory.get_all_configs()
# configurations with the highest previous EI
configs_previous_runs_sorted = self._sort_configs_by_acq_value(
configs_previous_runs)
configs_previous_runs_sorted = [
conf[1] for conf in configs_previous_runs_sorted[:num_points]
]
# configurations with the lowest predictive cost, check for None to make unit tests work
if self.acquisition_function.model is not None:
conf_array = convert_configurations_to_array(
configs_previous_runs)
costs = self.acquisition_function.model.predict_marginalized_over_instances(
conf_array)[0]
assert len(conf_array) == len(costs), (conf_array.shape,
costs.shape)
# In case of the predictive model returning the prediction for more than one objective per configuration
# (for example multi-objective or EIPS) it is not immediately clear how to sort according to the cost
# of a configuration. Therefore, we simply follow the ParEGO approach and use a random scalarization.
if len(costs.shape) == 2 and costs.shape[1] > 1:
weights = np.array(
[self.rng.rand() for _ in range(costs.shape[1])])
weights = weights / np.sum(weights)
costs = costs @ weights
# From here
# http://stackoverflow.com/questions/20197990/how-to-make-argsort-result-to-be-random-between-equal-values
random = self.rng.rand(len(costs))
# Last column is primary sort key!
indices = np.lexsort((random.flatten(), costs.flatten()))
# Cannot use zip here because the indices array cannot index the
# rand_configs list, because the second is a pure python list
configs_previous_runs_sorted_by_cost = [
configs_previous_runs[ind] for ind in indices
][:num_points]
else:
configs_previous_runs_sorted_by_cost = []
if additional_start_points is not None:
additional_start_points = [
asp[1] for asp in additional_start_points[:num_points]
]
else:
additional_start_points = []
init_points = []
init_points_as_set = set() # type: Set[Configuration]
for cand in itertools.chain(
configs_previous_runs_sorted,
configs_previous_runs_sorted_by_cost,
additional_start_points,
):
if cand not in init_points_as_set:
init_points.append(cand)
init_points_as_set.add(cand)
return init_points
def __init__(
self,
configspace,
random_fraction=1 / 2,
logger=None,
previous_results=None
):
self.logger = logger
self.random_fraction = random_fraction
self.runhistory = RunHistory()
self.configs = list()
self.losses = list()
rng = np.random.RandomState(random.randint(0, 100))
if previous_results is not None and len(previous_results.batch_results) > 0:
# Assume same-task changing-configspace trajectory for now
results_previous_adjustment = previous_results.batch_results[-1]
configspace_previous = results_previous_adjustment.configspace
# Construct combined config space
configspace_combined = ConfigSpace.ConfigurationSpace()
development_step = CSH.CategoricalHyperparameter("development_step", choices=["old", "new"])
configspace_combined.add_hyperparameter(
development_step
)
configspace_only_old, configspace_both, configspace_only_new = get_configspace_partitioning_cond(configspace, configspace_previous)
configspace_combined.add_hyperparameters(configspace_both.get_hyperparameters())
configspace_combined.add_hyperparameters(configspace_only_old.get_hyperparameters())
configspace_combined.add_hyperparameters(configspace_only_new.get_hyperparameters())
for hyperparameter in configspace_only_old.get_hyperparameters():
configspace_combined.add_condition(
ConfigSpace.EqualsCondition(hyperparameter, development_step, "old")
)
for hyperparameter in configspace_only_new.get_hyperparameters():
configspace_combined.add_condition(
ConfigSpace.EqualsCondition(hyperparameter, development_step, "new")
)
# Read old configs and losses
result_previous = results_previous_adjustment.results[0]
all_runs = result_previous.get_all_runs(only_largest_budget=False)
self.losses_old = [run.loss for run in all_runs]
self.configs_old = [run.config_id for run in all_runs]
id2conf = result_previous.get_id2config_mapping()
self.configs_old = [id2conf[id_]["config"] for id_ in self.configs_old]
# Map old configs to combined space
for config in self.configs_old:
config["development_step"] = "old"
self.configs_old = [ConfigSpace.Configuration(configspace_combined, config) for config in self.configs_old]
for config, cost in zip(self.configs_old, self.losses_old):
self.runhistory.add(config, cost, 0, StatusType.SUCCESS)
# Construct and fit model
self.configspace = configspace_combined
self.model = _construct_model(self.configspace, rng)
self.acquisition_func = EI(model=self.model)
self.acq_optimizer = LocalSearch(acquisition_function=self.acquisition_func,
config_space=self.configspace, rng=rng)
X = convert_configurations_to_array(self.configs_old)
Y = np.array(self.losses_old, dtype=np.float64)
self.model.train(X, Y)
self.acquisition_func.update(
model=self.model,
eta=min(self.losses_old),
)
else:
self.configspace = configspace
self.model = _construct_model(self.configspace, rng)
self.acquisition_func = EI(model=self.model)
self.acq_optimizer = LocalSearch(acquisition_function=self.acquisition_func,
config_space=self.configspace, rng=rng)
self.min_points_in_model = len(self.configspace.get_hyperparameters()) # TODO
def _build_matrix(
self,
run_dict: typing.Mapping[RunKey, RunValue],
runhistory: RunHistory,
return_time_as_y: bool = False,
store_statistics: bool = False,
) -> typing.Tuple[np.ndarray, np.ndarray]:
""" "Builds X,y matrixes from selected runs from runhistory
Parameters
----------
run_dict: dict: RunKey -> RunValue
dictionary from RunHistory.RunKey to RunHistory.RunValue
runhistory: RunHistory
runhistory object
return_time_as_y: bool
Return the time instead of cost as y value. Necessary to access the raw y values for imputation.
store_statistics: bool
Whether to store statistics about the data (to be used at subsequent calls)
Returns
-------
X: np.ndarray
Y: np.ndarray
"""
# First build nan-matrix of size #configs x #params+1
n_rows = len(run_dict)
n_cols = self.num_params
X = np.ones([n_rows, n_cols + self.n_feats]) * np.nan
# For now we keep it as 1
# TODO: Extend for native multi-objective
y = np.ones([n_rows, 1])
# Then populate matrix
for row, (key, run) in enumerate(run_dict.items()):
# Scaling is automatically done in configSpace
conf = runhistory.ids_config[key.config_id]
conf_vector = convert_configurations_to_array([conf])[0]
if self.n_feats:
feats = self.instance_features[key.instance_id]
X[row, :] = np.hstack((conf_vector, feats))
else:
X[row, :] = conf_vector
# run_array[row, -1] = instances[row]
if self.num_obj > 1:
assert self.multi_objective_algorithm is not None
# Let's normalize y here
# We use the objective_bounds calculated by the runhistory
y_ = normalize_costs([run.cost], runhistory.objective_bounds)
y_ = self.multi_objective_algorithm(y_)
y[row] = y_
else:
if return_time_as_y:
y[row, 0] = run.time
else:
y[row] = run.cost
if y.size > 0:
if store_statistics:
self.perc = np.percentile(y, self.scale_perc, axis=0)
self.min_y = np.min(y, axis=0)
self.max_y = np.max(y, axis=0)
y = self.transform_response_values(values=y)
return X, y
def plot_cost_over_time(self,
rh,
traj,
output="performance_over_time.png",
validator=None):
""" Plot performance over time, using all trajectory entries
with max_time = wallclock_limit or (if inf) the highest
recorded time
Parameters
----------
rh: RunHistory
runhistory to use
traj: List
trajectory to take times/incumbents from
output: str
path to output-png
epm: RandomForestWithInstances
emperical performance model (expecting trained on all runs)
"""
self.logger.debug("Estimating costs over time for best run.")
validator.traj = traj # set trajectory
time, configs = [], []
for entry in traj:
time.append(entry["wallclock_time"])
configs.append(entry["incumbent"])
self.logger.debug("Using %d samples (%d distinct) from trajectory.",
len(time), len(set(configs)))
if validator.epm: # not log as validator epm is trained on cost, not log cost
epm = validator.epm
else:
self.logger.debug(
"No EPM passed! Training new one from runhistory.")
# Train random forest and transform training data (from given rh)
# Not using validator because we want to plot uncertainties
rh2epm = RunHistory2EPM4Cost(num_params=len(
self.scenario.cs.get_hyperparameters()),
scenario=self.scenario)
X, y = rh2epm.transform(rh)
self.logger.debug("Training model with data of shape X: %s, y:%s",
str(X.shape), str(y.shape))
types, bounds = get_types(self.scenario.cs,
self.scenario.feature_array)
epm = RandomForestWithInstances(
types=types,
bounds=bounds,
instance_features=self.scenario.feature_array,
#seed=self.rng.randint(MAXINT),
ratio_features=1.0)
epm.train(X, y)
## not necessary right now since the EPM only knows the features
## of the training instances
# use only training instances
#=======================================================================
# if self.scenario.feature_dict:
# feat_array = []
# for inst in self.scenario.train_insts:
# feat_array.append(self.scenario.feature_dict[inst])
# backup_features_epm = epm.instance_features
# epm.instance_features = np.array(feat_array)
#=======================================================================
# predict performance for all configurations in trajectory
config_array = convert_configurations_to_array(configs)
mean, var = epm.predict_marginalized_over_instances(config_array)
#=======================================================================
# # restore feature array in epm
# if self.scenario.feature_dict:
# epm.instance_features = backup_features_epm
#=======================================================================
mean = mean[:, 0]
var = var[:, 0]
uncertainty_upper = mean + np.sqrt(var)
uncertainty_lower = mean - np.sqrt(var)
if self.scenario.run_obj == 'runtime': # We have to clip at 0 as we want to put y on the logscale
uncertainty_lower[uncertainty_lower < 0] = 0
uncertainty_upper[uncertainty_upper < 0] = 0
# plot
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_ylabel('performance')
ax.set_xlabel('time [sec]')
ax.plot(time, mean, 'r-', label="estimated performance")
ax.fill_between(time,
uncertainty_upper,
uncertainty_lower,
alpha=0.8,
label="standard deviation")
ax.set_xscale("log", nonposx='clip')
if self.scenario.run_obj == 'runtime':
ax.set_yscale('log')
# ax.set_ylim(min(mean)*0.8, max(mean)*1.2)
# start after 1% of the configuration budget
ax.set_xlim(min(time) + (max(time) - min(time)) * 0.01, max(time))
ax.legend()
plt.tight_layout()
fig.savefig(output)
plt.close(fig)
