def test_predict_marginalized_over_instances_mocked(self, rf_mock):
"""Use mock to count the number of calls to predict()"""
class SideEffect(object):
def __call__(self, X):
# Numpy array of number 0 to X.shape[0]
rval = np.array(list(range(X.shape[0]))).reshape((-1, 1))
# Return mean and variance
return rval, rval
rf_mock.side_effect = SideEffect()
rs = np.random.RandomState(1)
F = rs.rand(10, 5)
model = RandomForestWithInstances(
configspace=self._get_cs(10),
types=np.zeros((15, ), dtype=np.uint),
instance_features=F,
bounds=list(map(lambda x: (0, 10), range(10))),
seed=1,
)
X = rs.rand(20, 10)
F = rs.rand(10, 5)
Y = rs.randint(1, size=(len(X) * len(F), 1)) * 1.
X_ = rs.rand(200, 15)
model.train(X_, Y)
means, vars = model.predict_marginalized_over_instances(rs.rand(
11, 10))
# expected to be 0 as the predict is replaced by manual unloggin the trees
self.assertEqual(rf_mock.call_count, 0)
self.assertEqual(means.shape, (11, 1))
self.assertEqual(vars.shape, (11, 1))
for i in range(11):
self.assertEqual(means[i], 0.)
self.assertEqual(vars[i], 1.e-10)
def test_predict_marginalized_over_instances_mocked(self, rf_mock):
"""Use mock to count the number of calls to predict()"""
class SideEffect(object):
def __call__(self, X):
# Numpy array of number 0 to X.shape[0]
rval = np.array(list(range(X.shape[0]))).reshape((-1, 1))
# Return mean and variance
return rval, rval
rf_mock.side_effect = SideEffect()
rs = np.random.RandomState(1)
F = rs.rand(10, 5)
model = RandomForestWithInstances(np.zeros((15, ), dtype=np.uint),
instance_features=F,
bounds=np.array(list(
map(lambda x: (0, 10),
range(10))),
dtype=object))
means, vars = model.predict_marginalized_over_instances(rs.rand(
11, 10))
self.assertEqual(rf_mock.call_count, 11)
self.assertEqual(means.shape, (11, 1))
self.assertEqual(vars.shape, (11, 1))
for i in range(11):
self.assertEqual(means[i], 4.5)
self.assertEqual(vars[i], 4.5)
def test_predict_marginalized_over_instances(self):
rs = np.random.RandomState(1)
X = rs.rand(20, 10)
F = rs.rand(10, 5)
Y = rs.rand(len(X) * len(F), 1)
X_ = rs.rand(200, 15)
model = RandomForestWithInstances(np.zeros((15, ), dtype=np.uint),
instance_features=F)
model.train(X_, Y)
means, vars = model.predict_marginalized_over_instances(X)
self.assertEqual(means.shape, (20, 1))
self.assertEqual(vars.shape, (20, 1))
def test_predict_marginalized_over_instances(self):
rs = np.random.RandomState(1)
X = rs.rand(20, 10)
F = rs.rand(10, 5)
Y = rs.rand(len(X) * len(F), 1)
X_ = rs.rand(200, 15)
model = RandomForestWithInstances(
configspace=self._get_cs(10),
types=np.zeros((15, ), dtype=np.uint),
instance_features=F,
bounds=list(map(lambda x: (0, 10), range(10))),
seed=1,
)
model.train(X_, Y)
means, vars = model.predict_marginalized_over_instances(X)
self.assertEqual(means.shape, (20, 1))
self.assertEqual(vars.shape, (20, 1))
def _get_mean_var_time(self, validator, traj, use_epm, rh):
"""
Parameters
----------
validator: Validator
validator (smac-based)
traj: List[Configuraton]
trajectory to set in validator
use_epm: bool
validated or not (no need to use epm if validated)
rh: RunHistory
??
Returns
-------
mean, var
times: List[float]
times to plot (x-values)
configs
"""
# TODO kinda important: docstrings, what is this function doing?
if validator:
validator.traj = traj # set trajectory
time, configs = [], []
if use_epm and not self.block_epm:
for entry in traj:
time.append(entry["wallclock_time"])
configs.append(entry["incumbent"])
# self.logger.debug('Time: %d Runs: %d', time[-1], len(rh.get_runs_for_config(configs[-1])))
self.logger.debug(
"Using %d samples (%d distinct) from trajectory.", len(time),
len(set(configs)))
# Initialize EPM
if validator.epm: # not log as validator epm is trained on cost, not log cost
epm = validator.epm
else:
self.logger.debug(
"No EPM passed! Training new one from runhistory.")
# Train random forest and transform training data (from given rh)
# Not using validator because we want to plot uncertainties
rh2epm = RunHistory2EPM4Cost(num_params=len(
self.scenario.cs.get_hyperparameters()),
scenario=self.scenario)
X, y = rh2epm.transform(rh)
self.logger.debug(
"Training model with data of shape X: %s, y: %s",
str(X.shape), str(y.shape))
types, bounds = get_types(self.scenario.cs,
self.scenario.feature_array)
epm = RandomForestWithInstances(
self.scenario.cs,
types=types,
bounds=bounds,
seed=self.rng.randint(MAXINT),
instance_features=self.scenario.feature_array,
ratio_features=1.0)
epm.train(X, y)
config_array = convert_configurations_to_array(configs)
mean, var = epm.predict_marginalized_over_instances(config_array)
var = np.zeros(mean.shape)
# We don't want to show the uncertainty of the model but uncertainty over multiple optimizer runs
# This variance is computed in an outer loop.
else:
mean, var = [], []
for entry in traj:
#self.logger.debug(entry)
time.append(entry["wallclock_time"])
configs.append(entry["incumbent"])
costs = _cost(configs[-1], rh,
rh.get_runs_for_config(configs[-1]))
# self.logger.debug(len(costs), time[-1]
if not costs:
time.pop()
else:
mean.append(np.mean(costs))
var.append(0) # No variance over instances
mean, var = np.array(mean).reshape(-1, 1), np.array(var).reshape(
-1, 1)
return mean, var, time, configs
def get_pred_surface(self, rh, X_scaled, conf_list: list,
contour_step_size):
"""fit epm on the scaled input dimension and
return data to plot a contour plot of the empirical performance
Parameters
----------
rh: RunHistory
runhistory
X_scaled: np.array
configurations in scaled 2dim
conf_list: list
list of Configuration objects
Returns
-------
contour_data: (np.array, np.array, np.array)
x, y, Z for contour plots
"""
# use PCA to reduce features to also at most 2 dims
scen = copy.deepcopy(self.scenario) # pca changes feats
if scen.feature_array.shape[1] > 2:
self.logger.debug(
"Use PCA to reduce features to from %d dim to 2 dim",
scen.feature_array.shape[1])
# perform PCA
insts = scen.feature_dict.keys()
feature_array = np.array([scen.feature_dict[i] for i in insts])
feature_array = StandardScaler().fit_transform(feature_array)
feature_array = PCA(n_components=2).fit_transform(feature_array)
# inject in scenario-object
scen.feature_array = feature_array
scen.feature_dict = dict([(inst, feature_array[idx, :])
for idx, inst in enumerate(insts)])
scen.n_features = 2
# convert the data to train EPM on 2-dim featurespace (for contour-data)
self.logger.debug("Convert data for epm.")
X, y, types = convert_data_for_epm(scenario=scen,
runhistory=rh,
logger=self.logger)
types = np.array(np.zeros((2 + scen.feature_array.shape[1])),
dtype=np.uint)
num_params = len(scen.cs.get_hyperparameters())
# impute missing values in configs and insert MDS'ed (2dim) configs to the right positions
conf_dict = {}
for idx, c in enumerate(conf_list):
conf_list[idx] = impute_inactive_values(c)
conf_dict[str(conf_list[idx].get_array())] = X_scaled[idx, :]
X_trans = []
for x in X:
x_scaled_conf = conf_dict[str(x[:num_params])]
# append scaled config + pca'ed features (total of 4 values) per config/feature-sample
X_trans.append(
np.concatenate((x_scaled_conf, x[num_params:]), axis=0))
X_trans = np.array(X_trans)
self.logger.debug("Train random forest for contour-plot.")
bounds = np.array([(0, np.nan), (0, np.nan)], dtype=object)
model = RandomForestWithInstances(types=types,
bounds=bounds,
instance_features=np.array(
scen.feature_array),
ratio_features=1.0)
start = time.time()
model.train(X_trans, y)
self.logger.debug("Fitting random forest took %f time",
time.time() - start)
x_min, x_max = X_scaled[:, 0].min() - 1, X_scaled[:, 0].max() + 1
y_min, y_max = X_scaled[:, 1].min() - 1, X_scaled[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, contour_step_size),
np.arange(y_min, y_max, contour_step_size))
self.logger.debug("x_min: %f, x_max: %f, y_min: %f, y_max: %f", x_min,
x_max, y_min, y_max)
self.logger.debug(
"Predict on %d samples in grid to get surface (step-size: %f)",
np.c_[xx.ravel(), yy.ravel()].shape[0], contour_step_size)
start = time.time()
Z, _ = model.predict_marginalized_over_instances(np.c_[xx.ravel(),
yy.ravel()])
Z = Z.reshape(xx.shape)
self.logger.debug("Predicting random forest took %f time",
time.time() - start)
return xx, yy, Z
def get_pred_surface(self, rh, X_scaled, conf_list: list,
contour_step_size):
"""fit epm on the scaled input dimension and
return data to plot a contour plot of the empirical performance
Parameters
----------
rh: RunHistory
runhistory
X_scaled: np.array
configurations in scaled 2dim
conf_list: list
list of Configuration objects
contour_step_size: float
step-size for contour
Returns
-------
contour_data: (np.array, np.array, np.array)
x, y, Z for contour plots
"""
# use PCA to reduce features to also at most 2 dims
scen = copy.deepcopy(self.scenario) # pca changes feats
if scen.feature_array.shape[1] > 2:
self.logger.debug(
"Use PCA to reduce features to from %d dim to 2 dim",
scen.feature_array.shape[1])
# perform PCA
insts = scen.feature_dict.keys()
feature_array = np.array([scen.feature_dict[i] for i in insts])
feature_array = StandardScaler().fit_transform(feature_array)
feature_array = PCA(n_components=2).fit_transform(feature_array)
# inject in scenario-object
scen.feature_array = feature_array
scen.feature_dict = dict([(inst, feature_array[idx, :])
for idx, inst in enumerate(insts)])
scen.n_features = 2
# convert the data to train EPM on 2-dim featurespace (for contour-data)
self.logger.debug("Convert data for epm.")
X, y, types = convert_data_for_epm(scenario=scen,
runhistory=rh,
impute_inactive_parameters=True,
logger=self.logger)
types = np.array(np.zeros((2 + scen.feature_array.shape[1])),
dtype=np.uint)
num_params = len(scen.cs.get_hyperparameters())
# impute missing values in configs and insert MDS'ed (2dim) configs to the right positions
conf_dict = {}
# Remove forbidden clauses (this is necessary to enable the impute_inactive_values-method, see #226)
cs_no_forbidden = copy.deepcopy(conf_list[0].configuration_space)
cs_no_forbidden.forbidden_clauses = []
for idx, c in enumerate(conf_list):
c.configuration_space = cs_no_forbidden
conf_list[idx] = impute_inactive_values(c)
conf_dict[str(conf_list[idx].get_array())] = X_scaled[idx, :]
# Debug compare elements:
c1, c2 = {str(z) for z in X}, {str(z) for z in conf_dict.keys()}
self.logger.debug(
"{} elements not in both sets, {} elements in both sets, X (len {}) and conf_dict (len {}) "
"(might be a problem related to forbidden clauses?)".format(
len(c1 ^ c2), len(c1 & c2), len(c1 ^ c2), len(c1), len(c2)))
# self.logger.debug("Elements: {}".format(str(c1 ^ c2)))
X_trans = [
] # X_trans is the same as X but with reduced 2-dim features (so shape is (N, 2) instead of (N, M))
for x in X:
x_scaled_conf = conf_dict[str(x[:num_params])]
# append scaled config + pca'ed features (total of 4 values) per config/feature-sample
X_trans.append(
np.concatenate((x_scaled_conf, x[num_params:]), axis=0))
X_trans = np.array(X_trans)
self.logger.debug(
"Train random forest for contour-plot. Shape of X: {}, shape of X_trans: {}"
.format(X.shape, X_trans.shape))
self.logger.debug("Faking configspace to be able to train rf...")
# We need to fake config-space bypass imputation of inactive values in random forest implementation
fake_cs = ConfigurationSpace(name="fake-cs-for-configurator-footprint")
bounds = np.array([(0, np.nan), (0, np.nan)], dtype=object)
model = RandomForestWithInstances(fake_cs,
types,
bounds,
seed=self.rng.randint(MAXINT),
instance_features=np.array(
scen.feature_array),
ratio_features=1.0)
start = time.time()
model.train(X_trans, y)
self.logger.debug("Fitting random forest took %f time",
time.time() - start)
x_min, x_max = X_scaled[:, 0].min() - 1, X_scaled[:, 0].max() + 1
y_min, y_max = X_scaled[:, 1].min() - 1, X_scaled[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, contour_step_size),
np.arange(y_min, y_max, contour_step_size))
self.logger.debug("x_min: %f, x_max: %f, y_min: %f, y_max: %f", x_min,
x_max, y_min, y_max)
self.logger.debug(
"Predict on %d samples in grid to get surface (step-size: %f)",
np.c_[xx.ravel(), yy.ravel()].shape[0], contour_step_size)
start = time.time()
Z, _ = model.predict_marginalized_over_instances(np.c_[xx.ravel(),
yy.ravel()])
Z = Z.reshape(xx.shape)
self.logger.debug("Predicting random forest took %f time",
time.time() - start)
return xx, yy, Z
def get_pred_surface(self, X_scaled, conf_list: list):
'''
fit epm on the scaled input dimension and
return data to plot a contour plot
Parameters
----------
X_scaled: np.array
configurations in scaled 2dim
conf_list: list
list of Configuration objects
Returns
-------
np.array, np.array, np.array
x,y,Z for contour plots
'''
# use PCA to reduce features to also at most 2 dims
n_feats = self.scenario.feature_array.shape[1]
if n_feats > 2:
self.logger.debug("Use PCA to reduce features to 2dim")
insts = self.scenario.feature_dict.keys()
feature_array = np.array([self.scenario.feature_dict[inst] for inst in insts])
ss = StandardScaler()
self.scenario.feature_array = ss.fit_transform(feature_array)
pca = PCA(n_components=2)
feature_array = pca.fit_transform(feature_array)
n_feats = feature_array.shape[1]
self.scenario.feature_array = feature_array
self.scenario.feature_dict = dict([(inst, feature_array[idx,:]) for idx, inst in enumerate(insts)])
self.scenario.n_features = 2
# Create new rh with only wanted configs
new_rh = RunHistory(average_cost)
for rh in self.runhistories:
for key, value in rh.data.items():
config = rh.ids_config[key.config_id]
if config in self.configs_to_plot:
config_id, instance, seed = key
cost, time, status, additional_info = value
new_rh.add(config, cost, time, status, instance_id=instance,
seed=seed, additional_info=additional_info)
self.relevant_rh = new_rh
X, y, types = convert_data(scenario=self.scenario,
runhistory=new_rh)
types = np.array(np.zeros((2+n_feats)), dtype=np.uint)
num_params = len(self.scenario.cs.get_hyperparameters())
# impute missing values in configs
conf_dict = {}
for idx, c in enumerate(conf_list):
conf_list[idx] = impute_inactive_values(c)
conf_dict[str(conf_list[idx].get_array())] = X_scaled[idx, :]
X_trans = []
for x in X:
x_scaled_conf = conf_dict[str(x[:num_params])]
x_new = np.concatenate(
(x_scaled_conf, x[num_params:]), axis=0)
X_trans.append(x_new)
X_trans = np.array(X_trans)
bounds = np.array([(0, np.nan), (0, np.nan)], dtype=object)
model = RandomForestWithInstances(types=types, bounds=bounds,
instance_features=np.array(self.scenario.feature_array),
ratio_features=1.0)
model.train(X_trans, y)
self.logger.debug("RF fitted")
plot_step = self.contour_step_size
x_min, x_max = X_scaled[:, 0].min() - 1, X_scaled[:, 0].max() + 1
y_min, y_max = X_scaled[:, 1].min() - 1, X_scaled[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, plot_step),
np.arange(y_min, y_max, plot_step))
self.logger.debug("x_min: %f, x_max: %f, y_min: %f, y_max: %f" %(x_min, x_max, y_min, y_max))
self.logger.debug("Predict on %d samples in grid to get surface" %(np.c_[xx.ravel(), yy.ravel()].shape[0]))
Z, _ = model.predict_marginalized_over_instances(
np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)
return xx, yy, Z
def plot_cost_over_time(self,
rh,
traj,
output="performance_over_time.png",
validator=None):
""" Plot performance over time, using all trajectory entries
with max_time = wallclock_limit or (if inf) the highest
recorded time
Parameters
----------
rh: RunHistory
runhistory to use
traj: List
trajectory to take times/incumbents from
output: str
path to output-png
epm: RandomForestWithInstances
emperical performance model (expecting trained on all runs)
"""
self.logger.debug("Estimating costs over time for best run.")
validator.traj = traj # set trajectory
time, configs = [], []
for entry in traj:
time.append(entry["wallclock_time"])
configs.append(entry["incumbent"])
self.logger.debug("Using %d samples (%d distinct) from trajectory.",
len(time), len(set(configs)))
if validator.epm: # not log as validator epm is trained on cost, not log cost
epm = validator.epm
else:
self.logger.debug(
"No EPM passed! Training new one from runhistory.")
# Train random forest and transform training data (from given rh)
# Not using validator because we want to plot uncertainties
rh2epm = RunHistory2EPM4Cost(num_params=len(
self.scenario.cs.get_hyperparameters()),
scenario=self.scenario)
X, y = rh2epm.transform(rh)
self.logger.debug("Training model with data of shape X: %s, y:%s",
str(X.shape), str(y.shape))
types, bounds = get_types(self.scenario.cs,
self.scenario.feature_array)
epm = RandomForestWithInstances(
types=types,
bounds=bounds,
instance_features=self.scenario.feature_array,
#seed=self.rng.randint(MAXINT),
ratio_features=1.0)
epm.train(X, y)
## not necessary right now since the EPM only knows the features
## of the training instances
# use only training instances
#=======================================================================
# if self.scenario.feature_dict:
# feat_array = []
# for inst in self.scenario.train_insts:
# feat_array.append(self.scenario.feature_dict[inst])
# backup_features_epm = epm.instance_features
# epm.instance_features = np.array(feat_array)
#=======================================================================
# predict performance for all configurations in trajectory
config_array = convert_configurations_to_array(configs)
mean, var = epm.predict_marginalized_over_instances(config_array)
#=======================================================================
# # restore feature array in epm
# if self.scenario.feature_dict:
# epm.instance_features = backup_features_epm
#=======================================================================
mean = mean[:, 0]
var = var[:, 0]
uncertainty_upper = mean + np.sqrt(var)
uncertainty_lower = mean - np.sqrt(var)
if self.scenario.run_obj == 'runtime': # We have to clip at 0 as we want to put y on the logscale
uncertainty_lower[uncertainty_lower < 0] = 0
uncertainty_upper[uncertainty_upper < 0] = 0
# plot
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_ylabel('performance')
ax.set_xlabel('time [sec]')
ax.plot(time, mean, 'r-', label="estimated performance")
ax.fill_between(time,
uncertainty_upper,
uncertainty_lower,
alpha=0.8,
label="standard deviation")
ax.set_xscale("log", nonposx='clip')
if self.scenario.run_obj == 'runtime':
ax.set_yscale('log')
# ax.set_ylim(min(mean)*0.8, max(mean)*1.2)
# start after 1% of the configuration budget
ax.set_xlim(min(time) + (max(time) - min(time)) * 0.01, max(time))
ax.legend()
plt.tight_layout()
fig.savefig(output)
plt.close(fig)
