def test_train_regressor(self, space, fspace):
"""Test training different models"""
array = flatten_numpy(to_numpy(data, space), fspace)
model = train_regressor("AdaBoostRegressor", array)
assert isinstance(model, AdaBoostRegressor)
model = train_regressor("BaggingRegressor", array)
assert isinstance(model, BaggingRegressor)
model = train_regressor("ExtraTreesRegressor", array)
assert isinstance(model, ExtraTreesRegressor)
model = train_regressor("GradientBoostingRegressor", array)
assert isinstance(model, GradientBoostingRegressor)
model = train_regressor("RandomForestRegressor", array)
assert isinstance(model, RandomForestRegressor)
def test_train_regressor_kwargs(self, space, fspace):
"""Test training models with kwargs"""
array = flatten_numpy(to_numpy(data, space), fspace)
model = train_regressor("RandomForestRegressor",
array,
max_depth=2,
max_features="sqrt")
assert model.max_depth == 2
assert model.max_features == "sqrt"
def test_make_grid():
"""Test grid has correct format"""
trials = to_numpy(data, space)
model = train_regressor("RandomForestRegressor", trials)
best_point = trials[numpy.argmin(trials[:, -1])]
grid = make_grid(best_point, space, model, 4)
# Are fixed to anchor value
numpy.testing.assert_equal(grid[0][:, 1], best_point[1])
numpy.testing.assert_equal(grid[1][:, 0], best_point[0])
# Is a grid in search space
numpy.testing.assert_equal(grid[0][:, 0], [0, 2, 4, 6])
numpy.testing.assert_equal(grid[1][:, 1], [0, 1, 2, 3])
def test_make_grid_predictor(monkeypatch):
"""Test grid contains corresponding predictions from the model"""
trials = to_numpy(data, space)
model = train_regressor("RandomForestRegressor", trials)
best_point = trials[numpy.argmin(trials[:, -1])]
# Make sure model is not predicting exactly the original objective
with numpy.testing.assert_raises(AssertionError):
numpy.testing.assert_equal(
best_point[-1], model.predict(best_point[:-1].reshape(1, -1))
)
grid = make_grid(best_point, space, model, 4)
# Verify that grid predictions are those of the model
numpy.testing.assert_equal(grid[0][:, -1], model.predict(grid[0][:, :-1]))
numpy.testing.assert_equal(grid[1][:, -1], model.predict(grid[1][:, :-1]))
# Verify model predictions differ on different points
with numpy.testing.assert_raises(AssertionError):
numpy.testing.assert_equal(grid[0][:, -1], grid[1][:, -1])
def lpi(
trials,
space,
mode="best",
model="RandomForestRegressor",
n_points=20,
n_runs=10,
**kwargs
):
"""
Calculates the Local Parameter Importance for a collection of
:class:`orion.core.worker.trial.Trial`.
For more information on the metric, see original paper at
https://ml.informatik.uni-freiburg.de/papers/18-LION12-CAVE.pdf.
Biedenkapp, André, et al. "Cave: Configuration assessment, visualization and evaluation."
International Conference on Learning and Intelligent Optimization. Springer, Cham, 2018.
Parameters
----------
trials: DataFrame or dict
A dataframe of trials containing, at least, the columns 'objective' and 'id'. Or a dict
equivalent.
space: Space object
A space object from an experiment.
mode: str
Mode to compute the LPI.
- ``best``: Take the best trial found as the anchor for the LPI
- ``linear``: Recompute LPI for all values on a grid
model: str
Name of the regression model to use. Can be one of
- AdaBoostRegressor
- BaggingRegressor
- ExtraTreesRegressor
- GradientBoostingRegressor
- RandomForestRegressor (Default)
n_points: int
Number of points to compute the variances. Default is 20.
n_runs: int
Number of runs to compute the standard error of the LPI. Default is 10.
``**kwargs``
Arguments for the regressor model.
Returns
-------
DataFrame
LPI value for each parameter. If ``mode`` is `linear`, then a list of
param values and LPI metrics are returned in a DataFrame format.
"""
flattened_space = build_required_space(
space,
dist_requirement="linear",
type_requirement="numerical",
shape_requirement="flattened",
)
if trials.empty or trials.shape[0] == 0:
return pd.DataFrame(
data=[0] * len(flattened_space),
index=flattened_space.keys(),
columns=["LPI"],
)
data = to_numpy(trials, space)
data = flatten_numpy(data, flattened_space)
best_point = data[numpy.argmin(data[:, -1])]
rng = numpy.random.RandomState(kwargs.pop("random_state", None))
results = numpy.zeros((n_runs, len(flattened_space)))
for i in range(n_runs):
trained_model = train_regressor(
model, data, random_state=rng.randint(2 ** 32 - 1), **kwargs
)
results[i] = modes[mode](best_point, flattened_space, trained_model, n_points)
averages = results.mean(0)
standard_errors = results.std(0)
frame = pd.DataFrame(
data=numpy.array([averages, standard_errors]).T,
index=flattened_space.keys(),
columns=["LPI", "STD"],
)
return frame
def partial_dependency(trials,
space,
params=None,
model="RandomForestRegressor",
n_grid_points=10,
n_samples=50,
**kwargs):
"""
Calculates the partial dependency of parameters in a collection of :class:`Trial`.
Parameters
----------
trials: DataFrame or dict
A dataframe of trials containing, at least, the columns 'objective' and 'id'. Or a dict
equivalent.
space: Space object
A space object from an experiment.
params: list of str, optional
The parameters to include in the computation. All parameters are included by default.
model: str
Name of the regression model to use. Can be one of
- AdaBoostRegressor
- BaggingRegressor
- ExtraTreesRegressor
- GradientBoostingRegressor
- RandomForestRegressor (Default)
n_grid_points: int
Number of points in the grid to compute partial dependency. Default is 10.
n_samples: int
Number of samples to randomly generate the grid used to compute the partial dependency.
Default is 50.
**kwargs
Arguments for the regressor model.
Returns
-------
dict
Dictionary of DataFrames. Each combination of parameters as keys (dim1.name, dim2.name)
and for each parameters individually (dim1.name). Columns are
(dim1.name, dim2.name, objective) or (dim1.name, objective).
"""
params = flatten_params(space, params)
flattened_space = build_required_space(
space,
dist_requirement="linear",
type_requirement="numerical",
shape_requirement="flattened",
)
if trials.empty or trials.shape[0] == 0:
return {}
data = to_numpy(trials, space)
data = flatten_numpy(data, flattened_space)
model = train_regressor(model, data, **kwargs)
data = flattened_space.sample(n_samples)
data = pandas.DataFrame(data, columns=flattened_space.keys())
partial_dependencies = dict()
for x_i, x_name in enumerate(params):
grid, averages, stds = partial_dependency_grid(flattened_space, model,
[x_name], data,
n_grid_points)
grid = reverse(flattened_space, grid)
partial_dependencies[x_name] = (grid, averages, stds)
for y_i in range(x_i + 1, len(params)):
y_name = params[y_i]
grid, averages, stds = partial_dependency_grid(
flattened_space, model, [x_name, y_name], data, n_grid_points)
grid = reverse(flattened_space, grid)
partial_dependencies[(x_name, y_name)] = (grid, averages, stds)
return partial_dependencies
def test_train_regressor_invalid(self, space, fspace):
"""Test error message for invalid model names"""
array = flatten_numpy(to_numpy(data, space), fspace)
with pytest.raises(ValueError) as exc:
train_regressor("IDontExist", array)
assert exc.match("IDontExist is not a supported regressor")
def mock_train_regressor(*args, **kwargs):
nonlocal n_runs
n_runs += 1
seeds.add(kwargs["random_state"])
return train_regressor(*args, **kwargs)
