def test_capacity(self, space_each_type):
"""Check transformer space capacity"""
tspace = build_required_space(space_each_type, type_requirement="real")
assert tspace.cardinality == numpy.inf
space = Space()
probs = (0.1, 0.2, 0.3, 0.4)
categories = ("asdfa", 2, 3, 4)
dim = Categorical("yolo0",
OrderedDict(zip(categories, probs)),
shape=2)
space.register(dim)
dim = Integer("yolo2", "uniform", -3, 6)
space.register(dim)
tspace = build_required_space(space, type_requirement="integer")
assert tspace.cardinality == (4**2) * (6 + 1)
dim = Integer("yolo3", "uniform", -3, 6, shape=(2, 1))
space.register(dim)
tspace = build_required_space(space, type_requirement="integer")
assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1))
tspace = build_required_space(space,
type_requirement="integer",
shape_requirement="flattened")
assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1))
tspace = build_required_space(space,
type_requirement="integer",
dist_requirement="linear")
assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1))
def test_no_requirement(self, space_each_type):
"""Check what is built using 'None' requirement."""
tspace = build_required_space(None, space_each_type)
assert len(tspace) == 3
assert tspace[0].type == 'real'
assert tspace[1].type == 'categorical'
assert tspace[2].type == 'integer'
assert (
str(tspace) ==
"Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),\n" # noqa
" Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None),\n" # noqa
" Integer(name=yolo3, prior={randint: (3, 10), {}}, shape=(), default value=None)])"
) # noqa
tspace = build_required_space([], space_each_type)
assert len(tspace) == 3
assert tspace[0].type == 'real'
assert tspace[1].type == 'categorical'
assert tspace[2].type == 'integer'
assert (
str(tspace) ==
"Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),\n" # noqa
" Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None),\n" # noqa
" Integer(name=yolo3, prior={randint: (3, 10), {}}, shape=(), default value=None)])"
) # noqa
def test_flatten_requirement(self, space_each_type):
"""Check what is built using 'flatten' requirement."""
tspace = build_required_space(space_each_type,
shape_requirement="flattened")
# 1 integer + 1 categorical + 1 * (3, 2) shapes
assert len(tspace) == 1 + 1 + 3 * (3 * 2)
assert str(tspace).count("View") == 3 * (3 * 2)
i = 0
for _ in range(3 * 2):
assert tspace[i].type == "real"
i += 1
assert tspace[i].type == "categorical"
i += 1
assert tspace[i].type == "integer"
i += 1
for _ in range(3 * 2):
assert tspace[i].type == "real"
i += 1
for _ in range(3 * 2):
assert tspace[i].type == "integer"
i += 1
tspace = build_required_space(space_each_type,
shape_requirement="flattened",
type_requirement="real")
# 1 integer + 4 categorical + 1 * (3, 2) shapes
assert len(tspace) == 1 + 4 + 3 * (3 * 2)
assert str(tspace).count("View") == 4 + 3 * (3 * 2)
def test_cardinality(self, dim2):
"""Check cardinality of reshaped space"""
space = Space()
space.register(Real("yolo0", "uniform", 0, 2, shape=(2, 2)))
space.register(dim2)
rspace = build_required_space(space, shape_requirement="flattened")
assert rspace.cardinality == numpy.inf
rspace = build_required_space(space,
type_requirement="integer",
shape_requirement="flattened")
assert rspace.cardinality == (3**(2 * 2)) * 4
def test_no_requirement(self, space_each_type):
"""Check what is built using 'None' requirement."""
tspace = build_required_space(None, space_each_type)
assert len(tspace) == 3
assert tspace[0].type == 'real'
assert tspace[1].type == 'categorical'
assert tspace[2].type == 'integer'
assert str(tspace) == str(space_each_type)
tspace = build_required_space([], space_each_type)
assert len(tspace) == 3
assert tspace[0].type == 'real'
assert tspace[1].type == 'categorical'
assert tspace[2].type == 'integer'
assert str(tspace) == str(space_each_type)
def test_precision_with_linear(space, logdim, logintdim):
"""Test that precision isn't messed up by linearization."""
space.register(logdim)
space.register(logintdim)
# Force precision on all real or linearized dimensions
space["yolo0"].precision = 3
space["yolo4"].precision = 4
space["yolo5"].precision = 5
# Create a point
point = list(space.sample(1)[0])
real_index = list(space.keys()).index("yolo0")
logreal_index = list(space.keys()).index("yolo4")
logint_index = list(space.keys()).index("yolo5")
point[real_index] = 0.133333
point[logreal_index] = 0.1222222
point[logint_index] = 2
# Check first without linearization
tspace = build_required_space(space, type_requirement="numerical")
# Check that transform is fine
tpoint = tspace.transform(point)
assert tpoint[real_index] == 0.133
assert tpoint[logreal_index] == 0.1222
assert tpoint[logint_index] == 2
# Check that reserve does not break precision
rpoint = tspace.reverse(tpoint)
assert rpoint[real_index] == 0.133
assert rpoint[logreal_index] == 0.1222
assert rpoint[logint_index] == 2
# Check with linearization
tspace = build_required_space(space,
dist_requirement="linear",
type_requirement="real")
# Check that transform is fine
tpoint = tspace.transform(point)
assert tpoint[real_index] == 0.133
assert tpoint[logreal_index] == numpy.log(0.1222)
assert tpoint[logint_index] == numpy.log(2)
# Check that reserve does not break precision
rpoint = tspace.reverse(tpoint)
assert rpoint[real_index] == 0.133
assert rpoint[logreal_index] == 0.1222
assert rpoint[logint_index] == 2
def fspace(space):
return build_required_space(
space,
dist_requirement="linear",
type_requirement="numerical",
shape_requirement="flattened",
)
def transformed_space(space: Space):
return build_required_space(
space,
type_requirement="real",
shape_requirement="flattened",
dist_requirement="linear",
)
def test_unsupported_space(self):
"""Test tpe only work for supported search space"""
space = Space()
dim1 = Real("yolo1", "uniform", -10, 10)
space.register(dim1)
dim2 = Real("yolo2", "reciprocal", 10, 20)
space.register(dim2)
categories = ["a", 0.1, 2, "c"]
dim3 = Categorical("yolo3", categories)
space.register(dim3)
dim4 = Fidelity("epoch", 1, 9, 3)
space.register(dim4)
TPE(space)
space = Space()
dim = Real("yolo1", "norm", 0.9)
space.register(dim)
with pytest.raises(ValueError) as ex:
tpe = TPE(space)
tpe.space = build_required_space(
space, shape_requirement=TPE.requires_shape
)
assert (
"TPE now only supports uniform, loguniform, uniform discrete and choices"
in str(ex.value)
)
def test_cardinality(self, dim2):
"""Check cardinality of reshaped space"""
space = Space()
space.register(Real("yolo", "reciprocal", 0.1, 1, precision=1, shape=(2, 2)))
space.register(dim2)
rspace = build_required_space(space, shape_requirement="flattened")
assert rspace.cardinality == (10 ** (2 * 2)) * 4
space = Space()
space.register(Real("yolo", "uniform", 0, 2, shape=(2, 2)))
space.register(dim2)
rspace = build_required_space(
space, type_requirement="integer", shape_requirement="flattened"
)
assert rspace.cardinality == (3 ** (2 * 2)) * 4
def flatten_space(some_space):
"""Flatten a space"""
return build_required_space(
some_space,
dist_requirement="linear",
type_requirement="numerical",
shape_requirement="flattened",
)
def test_capacity(self, space_each_type):
"""Check transformer space capacity"""
tspace = build_required_space('real', space_each_type)
assert tspace.cardinality == numpy.inf
space = Space()
probs = (0.1, 0.2, 0.3, 0.4)
categories = ('asdfa', 2, 3, 4)
dim = Categorical('yolo', OrderedDict(zip(categories, probs)), shape=2)
space.register(dim)
dim = Integer('yolo2', 'uniform', -3, 6)
space.register(dim)
tspace = build_required_space('integer', space)
assert tspace.cardinality == (4 * 2) * 6
dim = Integer('yolo3', 'uniform', -3, 6, shape=(2, 1))
space.register(dim)
tspace = build_required_space('integer', space)
assert tspace.cardinality == (4 * 2) * 6 * 6 * (2 * 1)
def test_multidim_space(multidim_space):
"""Test that multidim is flattened"""
lower, upper = build_bounds(
build_required_space(
multidim_space,
type_requirement="real",
shape_requirement="flattened",
dist_requirement="linear",
))
numpy.testing.assert_equal(lower, numpy.array([2, 2, -3, 0]))
numpy.testing.assert_equal(upper, numpy.array([4, 4, 3, 1]))
def test_real_requirement(self, space_each_type):
"""Check what is built using 'real' requirement."""
tspace = build_required_space('real', space_each_type)
assert len(tspace) == 3
assert tspace[0].type == 'real'
assert tspace[1].type == 'real'
assert tspace[2].type == 'real'
assert(str(tspace) ==
"Space([Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None),\n" # noqa
" OneHotEncode(Enumerate(Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None))),\n" # noqa
" ReverseQuantize(Integer(name=yolo3, prior={randint: (3, 10), {}}, shape=(), default value=None))])") # noqa
def test_categorical(cat_space):
"""Test that categorical is mapped properly to vector embedding space"""
lower, upper = build_bounds(
build_required_space(
cat_space,
type_requirement="real",
shape_requirement="flattened",
dist_requirement="linear",
))
# First dimension is 2d category which is mapped to (0, 1) with < 0.5 threshold
# Three next dimensions are the one-hot dimensions of 3d category
numpy.testing.assert_equal(lower, numpy.array([0, 0, 0, 0, -3, 0]))
numpy.testing.assert_equal(upper, numpy.array([1, 1, 1, 1, 3, 1]))
def test_conversion_of_transformed():
array_orion_space = copy.deepcopy(orion_space)
array_orion_space["uns"] = "uniform(0, 1, shape=[2, 3])"
original_space = build_space(array_orion_space)
transformed_space = build_required_space(
original_space,
type_requirement=None,
shape_requirement="flattened",
dist_requirement=None,
)
cs = convert_space(transformed_space)
print(cs)
cs.sample_configuration()
def test_real_requirement(self, space_each_type):
"""Check what is built using 'real' requirement."""
tspace = build_required_space(space_each_type, type_requirement="real")
assert len(tspace) == 5
assert tspace[0].type == "real"
assert tspace[1].type == "real"
assert tspace[2].type == "real"
assert tspace[3].type == "real"
assert tspace[4].type == "real"
assert (str(tspace) == """\
Space([Precision(4, Real(name=yolo0, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
OneHotEncode(Enumerate(Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None))),
ReverseQuantize(Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None)),
Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None)),
ReverseQuantize(Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None))])\
""") # noqa
def test_no_requirement(self, space_each_type):
"""Check what is built using 'None' requirement."""
tspace = build_required_space(space_each_type)
assert len(tspace) == 5
assert tspace[0].type == "real"
assert tspace[1].type == "categorical"
# NOTE:HEAD
assert tspace[2].type == "integer"
assert tspace[3].type == "real"
assert tspace[4].type == "integer"
assert (str(tspace) == """\
Space([Precision(4, Real(name=yolo0, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None),
Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None),
Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None)),
Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None)])\
""") # noqa
def __init__(self, space, algorithm_config):
"""
Initialize the primary algorithm.
Parameters
----------
space : `orion.algo.space.Space`
The original definition of a problem's parameters space.
algorithm_config : dict
Configuration for the algorithm.
"""
self.algorithm = None
super(PrimaryAlgo, self).__init__(space, algorithm=algorithm_config)
requirements = self.algorithm.requires
self.transformed_space = build_required_space(requirements, self.space)
self.algorithm.space = self.transformed_space
def test_linear_requirement(self, space_each_type):
"""Check what is built using 'linear' requirement."""
tspace = build_required_space(space_each_type,
dist_requirement="linear")
assert len(tspace) == 5
assert tspace[0].type == "real"
assert tspace[1].type == "categorical"
assert tspace[2].type == "integer"
assert tspace[3].type == "real"
assert tspace[4].type == "real"
assert (str(tspace) == """\
Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),
Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=2),
Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(1,), default value=None),
Linearize(Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None))),
Linearize(ReverseQuantize(Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None)))])\
""") # noqa
def __init__(self, space, max_trials, seed=1):
self.space = space
self.linear_space = build_required_space('linear', space)
self.max_trials = max_trials
self.trials = []
n_per_dim = numpy.ceil(numpy.exp(numpy.log(max_trials) / len(space)))
dimensions = []
for name, dim in self.linear_space.items():
assert not dim.shape or len(dim.shape) == 1 and dim.shape[0] == 1
low, high = dim.interval()
dimensions.append(list(numpy.linspace(low, high, n_per_dim)))
self.params = list(itertools.product(*dimensions))
if len(self.params) > max_trials:
logger.warning(
f'GridSearch has more trials ({len(self.params)}) than the max ({max_trials})'
)
def create_algo(
algo_type: type[AlgoType],
space: Space,
**algo_kwargs,
) -> SpaceTransformAlgoWrapper[AlgoType]:
"""Creates an algorithm of the given type, taking care of transforming the space if needed."""
original_space = space
from orion.core.worker.transformer import build_required_space
# TODO: We could perhaps eventually *not* wrap the algorithm if it doesn't require any
# transformations. For now we just always wrap it.
transformed_space = build_required_space(
space,
type_requirement=algo_type.requires_type,
shape_requirement=algo_type.requires_shape,
dist_requirement=algo_type.requires_dist,
)
algorithm = algo_type(transformed_space, **algo_kwargs)
wrapped_algo = SpaceTransformAlgoWrapper(algorithm=algorithm,
space=original_space)
return wrapped_algo
def space():
# Use a search space with dimensions of any type.
original_space = SpaceBuilder().build({
"unr":
"uniform(0, 10)",
"uni":
"uniform(0, 20, discrete=True)",
"uns":
"uniform(0, 5, shape=[2, 3])",
"lur":
"loguniform(1e-5, 0.1)",
"lui":
"loguniform(1, 100, discrete=True)",
"cat":
'choices(["what", "ever", 0.33])',
})
return build_required_space(
original_space,
type_requirement=MOFA.requires_type,
shape_requirement=MOFA.requires_shape,
dist_requirement=MOFA.requires_dist,
)
def test_verify_trial(self, palgo, space):
trial = format_trials.tuple_to_trial((["asdfa", 2], 0, 3.5), space)
palgo._verify_trial(trial)
with pytest.raises(ValueError, match="not contained in space:"):
invalid_trial = format_trials.tuple_to_trial((("asdfa", 2), 10, 3.5), space)
palgo._verify_trial(invalid_trial)
# transform space
tspace = build_required_space(
space, type_requirement="real", shape_requirement="flattened"
)
# transform point
ttrial = tspace.transform(trial)
ttrial in tspace
# Transformed point is not in original space
with pytest.raises(ValueError, match="not contained in space:"):
palgo._verify_trial(ttrial)
# Transformed point is in transformed space
palgo._verify_trial(ttrial, space=tspace)
def __init__(
self,
max_trials: int = 100,
input_dir: Union[Path, str] = "profet_data",
checkpoint_dir: Union[Path, str] = None,
model_config: MetaModelConfig = None,
device: Union[str, Any] = None,
with_grad: bool = False,
):
super().__init__(max_trials=max_trials)
self.input_dir = Path(input_dir)
self.checkpoint_dir = Path(checkpoint_dir
or self.input_dir / "checkpoints")
# The config for the training of the meta-model.
# NOTE: the train config is used to determine the hash of the task.
if model_config is None:
# NOTE: This type error is safe to ignore: the benchmark argument will have been set in
# each ModelConfig subclass.
self.model_config = self.ModelConfig() # type: ignore
elif isinstance(model_config, dict):
self.model_config = self.ModelConfig(**model_config)
elif not isinstance(model_config, self.ModelConfig):
# If passed a model config, for example through deserializing the configuration,
# then convert it back to the right type, so the class attributes are correct.
self.model_config = self.ModelConfig(**asdict(model_config))
else:
self.model_config = model_config
assert isinstance(self.model_config, self.ModelConfig)
self.seed = self.model_config.seed
self.with_grad = with_grad
# The parameters that have an influence over the training of the meta-model are used to
# create the filename where the model will be saved.
task_hash_params = asdict(self.model_config)
logger.info(f"Task hash params: {task_hash_params}")
task_hash = compute_identity(**task_hash_params)
filename = f"{task_hash}.pkl"
self.checkpoint_file = self.checkpoint_dir / filename
logger.info(f"Checkpoint file for this task: {self.checkpoint_file}")
if isinstance(device, torch.device):
self.device = device
else:
self.device = torch.device(
device or ("cuda" if torch.cuda.is_available() else "cpu"))
# NOTE: Need to control the randomness that's happening inside *both* the training
# function, as well as the loading function (since `load_task_network`` instantiates a model
# and then loads the weights, it also affects the global rng state of pytorch).
with make_reproducible(self.seed):
if os.path.exists(self.checkpoint_file):
logger.info(
f"Model has already been trained: loading it from file {self.checkpoint_file}."
)
self.net, h = self.model_config.load_task_network(
self.checkpoint_file)
else:
warnings.warn(
RuntimeWarning(
f"Checkpoint file {self.checkpoint_file} doesn't exist: re-training the "
f"model. (This may take a *very* long time!)"))
logger.info(f"Task hash params: {task_hash_params}")
self.checkpoint_file.parent.mkdir(exist_ok=True, parents=True)
# Need to re-train the meta-model and sample this task.
self.net, h = self.model_config.get_task_network(
self.input_dir)
# Numpy random state. Currently only used in `sample()`
self._np_rng_state = np.random.RandomState(self.seed)
self.h: np.ndarray = np.array(h)
self.model_config.save_task_network(self.checkpoint_file, self.net,
self.h)
self.net = self.net.to(device=self.device, dtype=torch.float32)
self.net.eval()
self.h_tensor = torch.as_tensor(self.h,
dtype=torch.float32,
device=self.device)
self._space: Optional[Space] = None
self.name = (
f"profet.{type(self).__qualname__.lower()}_{self.model_config.task_id}"
)
self.transformed_space = transformer.build_required_space(
self.space,
type_requirement="real",
shape_requirement="flattened",
dist_requirement="linear",
)
def parallel_coordinates(experiment,
with_evc_tree=True,
order=None,
colorscale="YlOrRd",
**kwargs):
"""Plotly implementation of `orion.plotting.parallel_coordinates`"""
def build_frame():
"""Builds the dataframe for the plot"""
names = list(experiment.space.keys())
df = experiment.to_pandas(with_evc_tree=with_evc_tree)
df = df.loc[df["status"] == "completed"]
if df.empty:
return df
df[names] = df[names].transform(
functools.partial(_curate_params, space=experiment.space))
df = _flatten_dims(df, experiment.space)
return df
def infer_order(space, order):
"""Create order if not passed, otherwise verify it"""
params = orion.analysis.base.flatten_params(space, order)
if order is None:
fidelity_dims = [
dim for dim in experiment.space.values()
if isinstance(dim, Fidelity)
]
fidelity = fidelity_dims[0].name if fidelity_dims else None
if fidelity in params:
del params[params.index(fidelity)]
params.insert(0, fidelity)
return params
def get_dimension(data, name, dim):
dim_data = dict(label=name, values=data[name])
if dim.type == "categorical":
categories = dim.interval()
dim_data["tickvals"] = list(range(len(categories)))
dim_data["ticktext"] = categories
else:
dim_data["range"] = dim.interval()
return dim_data
if not experiment:
raise ValueError("Parameter 'experiment' is None")
df = build_frame()
if df.empty:
return go.Figure()
trial = experiment.fetch_trials_by_status("completed")[0]
flattened_space = build_required_space(experiment.space,
shape_requirement="flattened")
dimensions = [
get_dimension(df, name, flattened_space[name])
for name in infer_order(experiment.space, order)
]
objective_name = trial.objective.name
objectives = df["objective"]
omin = min(df["objective"])
omax = max(df["objective"])
dimensions.append(
dict(label=objective_name, range=(omin, omax), values=objectives))
fig = go.Figure(data=go.Parcoords(
line=dict(
color=objectives,
colorscale=colorscale,
showscale=True,
cmin=omin,
cmax=omax,
colorbar=dict(title=objective_name),
),
dimensions=dimensions,
))
fig.update_layout(
title=f"Parallel Coordinates Plot for experiment '{experiment.name}'")
return fig
def test_not_supported_requirement(self, space_each_type):
"""Require something which is not supported."""
with pytest.raises(TypeError) as exc:
build_required_space('fasdfasf', space_each_type)
assert 'Unsupported' in str(exc.value)
def partial_dependency(trials,
space,
params=None,
model="RandomForestRegressor",
n_grid_points=10,
n_samples=50,
**kwargs):
"""
Calculates the partial dependency of parameters in a collection of :class:`Trial`.
Parameters
----------
trials: DataFrame or dict
A dataframe of trials containing, at least, the columns 'objective' and 'id'. Or a dict
equivalent.
space: Space object
A space object from an experiment.
params: list of str, optional
The parameters to include in the computation. All parameters are included by default.
model: str
Name of the regression model to use. Can be one of
- AdaBoostRegressor
- BaggingRegressor
- ExtraTreesRegressor
- GradientBoostingRegressor
- RandomForestRegressor (Default)
n_grid_points: int
Number of points in the grid to compute partial dependency. Default is 10.
n_samples: int
Number of samples to randomly generate the grid used to compute the partial dependency.
Default is 50.
**kwargs
Arguments for the regressor model.
Returns
-------
dict
Dictionary of DataFrames. Each combination of parameters as keys (dim1.name, dim2.name)
and for each parameters individually (dim1.name). Columns are
(dim1.name, dim2.name, objective) or (dim1.name, objective).
"""
params = flatten_params(space, params)
flattened_space = build_required_space(
space,
dist_requirement="linear",
type_requirement="numerical",
shape_requirement="flattened",
)
if trials.empty or trials.shape[0] == 0:
return {}
data = to_numpy(trials, space)
data = flatten_numpy(data, flattened_space)
model = train_regressor(model, data, **kwargs)
data = flattened_space.sample(n_samples)
data = pandas.DataFrame(data, columns=flattened_space.keys())
partial_dependencies = dict()
for x_i, x_name in enumerate(params):
grid, averages, stds = partial_dependency_grid(flattened_space, model,
[x_name], data,
n_grid_points)
grid = reverse(flattened_space, grid)
partial_dependencies[x_name] = (grid, averages, stds)
for y_i in range(x_i + 1, len(params)):
y_name = params[y_i]
grid, averages, stds = partial_dependency_grid(
flattened_space, model, [x_name, y_name], data, n_grid_points)
grid = reverse(flattened_space, grid)
partial_dependencies[(x_name, y_name)] = (grid, averages, stds)
return partial_dependencies
def lpi(
trials,
space,
mode="best",
model="RandomForestRegressor",
n_points=20,
n_runs=10,
**kwargs
):
"""
Calculates the Local Parameter Importance for a collection of
:class:`orion.core.worker.trial.Trial`.
For more information on the metric, see original paper at
https://ml.informatik.uni-freiburg.de/papers/18-LION12-CAVE.pdf.
Biedenkapp, André, et al. "Cave: Configuration assessment, visualization and evaluation."
International Conference on Learning and Intelligent Optimization. Springer, Cham, 2018.
Parameters
----------
trials: DataFrame or dict
A dataframe of trials containing, at least, the columns 'objective' and 'id'. Or a dict
equivalent.
space: Space object
A space object from an experiment.
mode: str
Mode to compute the LPI.
- ``best``: Take the best trial found as the anchor for the LPI
- ``linear``: Recompute LPI for all values on a grid
model: str
Name of the regression model to use. Can be one of
- AdaBoostRegressor
- BaggingRegressor
- ExtraTreesRegressor
- GradientBoostingRegressor
- RandomForestRegressor (Default)
n_points: int
Number of points to compute the variances. Default is 20.
n_runs: int
Number of runs to compute the standard error of the LPI. Default is 10.
``**kwargs``
Arguments for the regressor model.
Returns
-------
DataFrame
LPI value for each parameter. If ``mode`` is `linear`, then a list of
param values and LPI metrics are returned in a DataFrame format.
"""
flattened_space = build_required_space(
space,
dist_requirement="linear",
type_requirement="numerical",
shape_requirement="flattened",
)
if trials.empty or trials.shape[0] == 0:
return pd.DataFrame(
data=[0] * len(flattened_space),
index=flattened_space.keys(),
columns=["LPI"],
)
data = to_numpy(trials, space)
data = flatten_numpy(data, flattened_space)
best_point = data[numpy.argmin(data[:, -1])]
rng = numpy.random.RandomState(kwargs.pop("random_state", None))
results = numpy.zeros((n_runs, len(flattened_space)))
for i in range(n_runs):
trained_model = train_regressor(
model, data, random_state=rng.randint(2 ** 32 - 1), **kwargs
)
results[i] = modes[mode](best_point, flattened_space, trained_model, n_points)
averages = results.mean(0)
standard_errors = results.std(0)
frame = pd.DataFrame(
data=numpy.array([averages, standard_errors]).T,
index=flattened_space.keys(),
columns=["LPI", "STD"],
)
return frame
def flatten_params(space, params=None):
"""Return the params of the corresponding flat space
If no params are passed, returns all flattened params.
If params are passed, returns the corresponding flattened params.
Parameters
----------
space: Space object
A space object from an experiment.
params: list of str, optional
The parameters to select from the search space. If the flattened search space
contains flattened params such as ('y' -> 'y[0]', 'y[1]'), passing 'y' in the list of
params will returned the flattened version ['y[0]', 'y[1]']
Examples
--------
If space has x~uniform(0, 1) and y~uniform(0, 1, shape=(1, 2)).
>>> flatten_params(space)
['x', 'y[0,0]', 'y[0,1]']
>>> flatten_params(space, params=['x'])
['x']
>>> flatten_params(space, params=['x', 'y'])
['x', 'y[0,0]', 'y[0,1]']
>>> flatten_params(space, params=['x', 'y[0,1]'])
['x', 'y[0,1]']
>>> flatten_params(space, params=['y[0,1]', 'x'])
['x', 'y[0,1]']
Raises
------
ValueError
If one of the parameter names passed is not in the flattened space.
"""
keys = set(space.keys())
flattened_keys = set(
build_required_space(
space,
dist_requirement="linear",
type_requirement="numerical",
shape_requirement="flattened",
).keys())
if params is None:
return sorted(flattened_keys)
flattened_params = []
for param in params:
if param not in flattened_keys and param not in keys:
raise ValueError(
f"Parameter {param} not contained in space: {flattened_keys}")
elif param not in flattened_keys and param in keys:
dim = space[param]
flattened_params += [
f'{dim.name}[{",".join(map(str, index))}]'
for index in itertools.product(*map(range, dim.shape))
]
else:
flattened_params.append(param)
return flattened_params
