def test_capacity(self, space_each_type): """Check transformer space capacity""" tspace = build_required_space(space_each_type, type_requirement="real") assert tspace.cardinality == numpy.inf space = Space() probs = (0.1, 0.2, 0.3, 0.4) categories = ("asdfa", 2, 3, 4) dim = Categorical("yolo0", OrderedDict(zip(categories, probs)), shape=2) space.register(dim) dim = Integer("yolo2", "uniform", -3, 6) space.register(dim) tspace = build_required_space(space, type_requirement="integer") assert tspace.cardinality == (4**2) * (6 + 1) dim = Integer("yolo3", "uniform", -3, 6, shape=(2, 1)) space.register(dim) tspace = build_required_space(space, type_requirement="integer") assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1)) tspace = build_required_space(space, type_requirement="integer", shape_requirement="flattened") assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1)) tspace = build_required_space(space, type_requirement="integer", dist_requirement="linear") assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1))
def test_no_requirement(self, space_each_type): """Check what is built using 'None' requirement.""" tspace = build_required_space(None, space_each_type) assert len(tspace) == 3 assert tspace[0].type == 'real' assert tspace[1].type == 'categorical' assert tspace[2].type == 'integer' assert ( str(tspace) == "Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),\n" # noqa " Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None),\n" # noqa " Integer(name=yolo3, prior={randint: (3, 10), {}}, shape=(), default value=None)])" ) # noqa tspace = build_required_space([], space_each_type) assert len(tspace) == 3 assert tspace[0].type == 'real' assert tspace[1].type == 'categorical' assert tspace[2].type == 'integer' assert ( str(tspace) == "Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)),\n" # noqa " Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None),\n" # noqa " Integer(name=yolo3, prior={randint: (3, 10), {}}, shape=(), default value=None)])" ) # noqa
def test_flatten_requirement(self, space_each_type): """Check what is built using 'flatten' requirement.""" tspace = build_required_space(space_each_type, shape_requirement="flattened") # 1 integer + 1 categorical + 1 * (3, 2) shapes assert len(tspace) == 1 + 1 + 3 * (3 * 2) assert str(tspace).count("View") == 3 * (3 * 2) i = 0 for _ in range(3 * 2): assert tspace[i].type == "real" i += 1 assert tspace[i].type == "categorical" i += 1 assert tspace[i].type == "integer" i += 1 for _ in range(3 * 2): assert tspace[i].type == "real" i += 1 for _ in range(3 * 2): assert tspace[i].type == "integer" i += 1 tspace = build_required_space(space_each_type, shape_requirement="flattened", type_requirement="real") # 1 integer + 4 categorical + 1 * (3, 2) shapes assert len(tspace) == 1 + 4 + 3 * (3 * 2) assert str(tspace).count("View") == 4 + 3 * (3 * 2)
def test_cardinality(self, dim2): """Check cardinality of reshaped space""" space = Space() space.register(Real("yolo0", "uniform", 0, 2, shape=(2, 2))) space.register(dim2) rspace = build_required_space(space, shape_requirement="flattened") assert rspace.cardinality == numpy.inf rspace = build_required_space(space, type_requirement="integer", shape_requirement="flattened") assert rspace.cardinality == (3**(2 * 2)) * 4
def test_no_requirement(self, space_each_type): """Check what is built using 'None' requirement.""" tspace = build_required_space(None, space_each_type) assert len(tspace) == 3 assert tspace[0].type == 'real' assert tspace[1].type == 'categorical' assert tspace[2].type == 'integer' assert str(tspace) == str(space_each_type) tspace = build_required_space([], space_each_type) assert len(tspace) == 3 assert tspace[0].type == 'real' assert tspace[1].type == 'categorical' assert tspace[2].type == 'integer' assert str(tspace) == str(space_each_type)
def test_precision_with_linear(space, logdim, logintdim): """Test that precision isn't messed up by linearization.""" space.register(logdim) space.register(logintdim) # Force precision on all real or linearized dimensions space["yolo0"].precision = 3 space["yolo4"].precision = 4 space["yolo5"].precision = 5 # Create a point point = list(space.sample(1)[0]) real_index = list(space.keys()).index("yolo0") logreal_index = list(space.keys()).index("yolo4") logint_index = list(space.keys()).index("yolo5") point[real_index] = 0.133333 point[logreal_index] = 0.1222222 point[logint_index] = 2 # Check first without linearization tspace = build_required_space(space, type_requirement="numerical") # Check that transform is fine tpoint = tspace.transform(point) assert tpoint[real_index] == 0.133 assert tpoint[logreal_index] == 0.1222 assert tpoint[logint_index] == 2 # Check that reserve does not break precision rpoint = tspace.reverse(tpoint) assert rpoint[real_index] == 0.133 assert rpoint[logreal_index] == 0.1222 assert rpoint[logint_index] == 2 # Check with linearization tspace = build_required_space(space, dist_requirement="linear", type_requirement="real") # Check that transform is fine tpoint = tspace.transform(point) assert tpoint[real_index] == 0.133 assert tpoint[logreal_index] == numpy.log(0.1222) assert tpoint[logint_index] == numpy.log(2) # Check that reserve does not break precision rpoint = tspace.reverse(tpoint) assert rpoint[real_index] == 0.133 assert rpoint[logreal_index] == 0.1222 assert rpoint[logint_index] == 2
def fspace(space): return build_required_space( space, dist_requirement="linear", type_requirement="numerical", shape_requirement="flattened", )
def transformed_space(space: Space): return build_required_space( space, type_requirement="real", shape_requirement="flattened", dist_requirement="linear", )
def test_unsupported_space(self): """Test tpe only work for supported search space""" space = Space() dim1 = Real("yolo1", "uniform", -10, 10) space.register(dim1) dim2 = Real("yolo2", "reciprocal", 10, 20) space.register(dim2) categories = ["a", 0.1, 2, "c"] dim3 = Categorical("yolo3", categories) space.register(dim3) dim4 = Fidelity("epoch", 1, 9, 3) space.register(dim4) TPE(space) space = Space() dim = Real("yolo1", "norm", 0.9) space.register(dim) with pytest.raises(ValueError) as ex: tpe = TPE(space) tpe.space = build_required_space( space, shape_requirement=TPE.requires_shape ) assert ( "TPE now only supports uniform, loguniform, uniform discrete and choices" in str(ex.value) )
def test_cardinality(self, dim2): """Check cardinality of reshaped space""" space = Space() space.register(Real("yolo", "reciprocal", 0.1, 1, precision=1, shape=(2, 2))) space.register(dim2) rspace = build_required_space(space, shape_requirement="flattened") assert rspace.cardinality == (10 ** (2 * 2)) * 4 space = Space() space.register(Real("yolo", "uniform", 0, 2, shape=(2, 2))) space.register(dim2) rspace = build_required_space( space, type_requirement="integer", shape_requirement="flattened" ) assert rspace.cardinality == (3 ** (2 * 2)) * 4
def flatten_space(some_space): """Flatten a space""" return build_required_space( some_space, dist_requirement="linear", type_requirement="numerical", shape_requirement="flattened", )
def test_capacity(self, space_each_type): """Check transformer space capacity""" tspace = build_required_space('real', space_each_type) assert tspace.cardinality == numpy.inf space = Space() probs = (0.1, 0.2, 0.3, 0.4) categories = ('asdfa', 2, 3, 4) dim = Categorical('yolo', OrderedDict(zip(categories, probs)), shape=2) space.register(dim) dim = Integer('yolo2', 'uniform', -3, 6) space.register(dim) tspace = build_required_space('integer', space) assert tspace.cardinality == (4 * 2) * 6 dim = Integer('yolo3', 'uniform', -3, 6, shape=(2, 1)) space.register(dim) tspace = build_required_space('integer', space) assert tspace.cardinality == (4 * 2) * 6 * 6 * (2 * 1)
def test_multidim_space(multidim_space): """Test that multidim is flattened""" lower, upper = build_bounds( build_required_space( multidim_space, type_requirement="real", shape_requirement="flattened", dist_requirement="linear", )) numpy.testing.assert_equal(lower, numpy.array([2, 2, -3, 0])) numpy.testing.assert_equal(upper, numpy.array([4, 4, 3, 1]))
def test_real_requirement(self, space_each_type): """Check what is built using 'real' requirement.""" tspace = build_required_space('real', space_each_type) assert len(tspace) == 3 assert tspace[0].type == 'real' assert tspace[1].type == 'real' assert tspace[2].type == 'real' assert(str(tspace) == "Space([Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None),\n" # noqa " OneHotEncode(Enumerate(Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None))),\n" # noqa " ReverseQuantize(Integer(name=yolo3, prior={randint: (3, 10), {}}, shape=(), default value=None))])") # noqa
def test_categorical(cat_space): """Test that categorical is mapped properly to vector embedding space""" lower, upper = build_bounds( build_required_space( cat_space, type_requirement="real", shape_requirement="flattened", dist_requirement="linear", )) # First dimension is 2d category which is mapped to (0, 1) with < 0.5 threshold # Three next dimensions are the one-hot dimensions of 3d category numpy.testing.assert_equal(lower, numpy.array([0, 0, 0, 0, -3, 0])) numpy.testing.assert_equal(upper, numpy.array([1, 1, 1, 1, 3, 1]))
def test_conversion_of_transformed(): array_orion_space = copy.deepcopy(orion_space) array_orion_space["uns"] = "uniform(0, 1, shape=[2, 3])" original_space = build_space(array_orion_space) transformed_space = build_required_space( original_space, type_requirement=None, shape_requirement="flattened", dist_requirement=None, ) cs = convert_space(transformed_space) print(cs) cs.sample_configuration()
def test_real_requirement(self, space_each_type): """Check what is built using 'real' requirement.""" tspace = build_required_space(space_each_type, type_requirement="real") assert len(tspace) == 5 assert tspace[0].type == "real" assert tspace[1].type == "real" assert tspace[2].type == "real" assert tspace[3].type == "real" assert tspace[4].type == "real" assert (str(tspace) == """\ Space([Precision(4, Real(name=yolo0, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)), OneHotEncode(Enumerate(Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None))), ReverseQuantize(Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None)), Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None)), ReverseQuantize(Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None))])\ """) # noqa
def test_no_requirement(self, space_each_type): """Check what is built using 'None' requirement.""" tspace = build_required_space(space_each_type) assert len(tspace) == 5 assert tspace[0].type == "real" assert tspace[1].type == "categorical" # NOTE:HEAD assert tspace[2].type == "integer" assert tspace[3].type == "real" assert tspace[4].type == "integer" assert (str(tspace) == """\ Space([Precision(4, Real(name=yolo0, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)), Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=None), Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(), default value=None), Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None)), Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None)])\ """) # noqa
def __init__(self, space, algorithm_config): """ Initialize the primary algorithm. Parameters ---------- space : `orion.algo.space.Space` The original definition of a problem's parameters space. algorithm_config : dict Configuration for the algorithm. """ self.algorithm = None super(PrimaryAlgo, self).__init__(space, algorithm=algorithm_config) requirements = self.algorithm.requires self.transformed_space = build_required_space(requirements, self.space) self.algorithm.space = self.transformed_space
def test_linear_requirement(self, space_each_type): """Check what is built using 'linear' requirement.""" tspace = build_required_space(space_each_type, dist_requirement="linear") assert len(tspace) == 5 assert tspace[0].type == "real" assert tspace[1].type == "categorical" assert tspace[2].type == "integer" assert tspace[3].type == "real" assert tspace[4].type == "real" assert (str(tspace) == """\ Space([Precision(4, Real(name=yolo, prior={norm: (0.9,), {}}, shape=(3, 2), default value=None)), Categorical(name=yolo2, prior={asdfa: 0.10, 2: 0.20, 3: 0.30, 4: 0.40}, shape=(), default value=2), Integer(name=yolo3, prior={uniform: (3, 7), {}}, shape=(1,), default value=None), Linearize(Precision(4, Real(name=yolo4, prior={reciprocal: (1.0, 10.0), {}}, shape=(3, 2), default value=None))), Linearize(ReverseQuantize(Integer(name=yolo5, prior={reciprocal: (1, 10), {}}, shape=(3, 2), default value=None)))])\ """) # noqa
def __init__(self, space, max_trials, seed=1): self.space = space self.linear_space = build_required_space('linear', space) self.max_trials = max_trials self.trials = [] n_per_dim = numpy.ceil(numpy.exp(numpy.log(max_trials) / len(space))) dimensions = [] for name, dim in self.linear_space.items(): assert not dim.shape or len(dim.shape) == 1 and dim.shape[0] == 1 low, high = dim.interval() dimensions.append(list(numpy.linspace(low, high, n_per_dim))) self.params = list(itertools.product(*dimensions)) if len(self.params) > max_trials: logger.warning( f'GridSearch has more trials ({len(self.params)}) than the max ({max_trials})' )
def create_algo( algo_type: type[AlgoType], space: Space, **algo_kwargs, ) -> SpaceTransformAlgoWrapper[AlgoType]: """Creates an algorithm of the given type, taking care of transforming the space if needed.""" original_space = space from orion.core.worker.transformer import build_required_space # TODO: We could perhaps eventually *not* wrap the algorithm if it doesn't require any # transformations. For now we just always wrap it. transformed_space = build_required_space( space, type_requirement=algo_type.requires_type, shape_requirement=algo_type.requires_shape, dist_requirement=algo_type.requires_dist, ) algorithm = algo_type(transformed_space, **algo_kwargs) wrapped_algo = SpaceTransformAlgoWrapper(algorithm=algorithm, space=original_space) return wrapped_algo
def space(): # Use a search space with dimensions of any type. original_space = SpaceBuilder().build({ "unr": "uniform(0, 10)", "uni": "uniform(0, 20, discrete=True)", "uns": "uniform(0, 5, shape=[2, 3])", "lur": "loguniform(1e-5, 0.1)", "lui": "loguniform(1, 100, discrete=True)", "cat": 'choices(["what", "ever", 0.33])', }) return build_required_space( original_space, type_requirement=MOFA.requires_type, shape_requirement=MOFA.requires_shape, dist_requirement=MOFA.requires_dist, )
def test_verify_trial(self, palgo, space): trial = format_trials.tuple_to_trial((["asdfa", 2], 0, 3.5), space) palgo._verify_trial(trial) with pytest.raises(ValueError, match="not contained in space:"): invalid_trial = format_trials.tuple_to_trial((("asdfa", 2), 10, 3.5), space) palgo._verify_trial(invalid_trial) # transform space tspace = build_required_space( space, type_requirement="real", shape_requirement="flattened" ) # transform point ttrial = tspace.transform(trial) ttrial in tspace # Transformed point is not in original space with pytest.raises(ValueError, match="not contained in space:"): palgo._verify_trial(ttrial) # Transformed point is in transformed space palgo._verify_trial(ttrial, space=tspace)
def __init__( self, max_trials: int = 100, input_dir: Union[Path, str] = "profet_data", checkpoint_dir: Union[Path, str] = None, model_config: MetaModelConfig = None, device: Union[str, Any] = None, with_grad: bool = False, ): super().__init__(max_trials=max_trials) self.input_dir = Path(input_dir) self.checkpoint_dir = Path(checkpoint_dir or self.input_dir / "checkpoints") # The config for the training of the meta-model. # NOTE: the train config is used to determine the hash of the task. if model_config is None: # NOTE: This type error is safe to ignore: the benchmark argument will have been set in # each ModelConfig subclass. self.model_config = self.ModelConfig() # type: ignore elif isinstance(model_config, dict): self.model_config = self.ModelConfig(**model_config) elif not isinstance(model_config, self.ModelConfig): # If passed a model config, for example through deserializing the configuration, # then convert it back to the right type, so the class attributes are correct. self.model_config = self.ModelConfig(**asdict(model_config)) else: self.model_config = model_config assert isinstance(self.model_config, self.ModelConfig) self.seed = self.model_config.seed self.with_grad = with_grad # The parameters that have an influence over the training of the meta-model are used to # create the filename where the model will be saved. task_hash_params = asdict(self.model_config) logger.info(f"Task hash params: {task_hash_params}") task_hash = compute_identity(**task_hash_params) filename = f"{task_hash}.pkl" self.checkpoint_file = self.checkpoint_dir / filename logger.info(f"Checkpoint file for this task: {self.checkpoint_file}") if isinstance(device, torch.device): self.device = device else: self.device = torch.device( device or ("cuda" if torch.cuda.is_available() else "cpu")) # NOTE: Need to control the randomness that's happening inside *both* the training # function, as well as the loading function (since `load_task_network`` instantiates a model # and then loads the weights, it also affects the global rng state of pytorch). with make_reproducible(self.seed): if os.path.exists(self.checkpoint_file): logger.info( f"Model has already been trained: loading it from file {self.checkpoint_file}." ) self.net, h = self.model_config.load_task_network( self.checkpoint_file) else: warnings.warn( RuntimeWarning( f"Checkpoint file {self.checkpoint_file} doesn't exist: re-training the " f"model. (This may take a *very* long time!)")) logger.info(f"Task hash params: {task_hash_params}") self.checkpoint_file.parent.mkdir(exist_ok=True, parents=True) # Need to re-train the meta-model and sample this task. self.net, h = self.model_config.get_task_network( self.input_dir) # Numpy random state. Currently only used in `sample()` self._np_rng_state = np.random.RandomState(self.seed) self.h: np.ndarray = np.array(h) self.model_config.save_task_network(self.checkpoint_file, self.net, self.h) self.net = self.net.to(device=self.device, dtype=torch.float32) self.net.eval() self.h_tensor = torch.as_tensor(self.h, dtype=torch.float32, device=self.device) self._space: Optional[Space] = None self.name = ( f"profet.{type(self).__qualname__.lower()}_{self.model_config.task_id}" ) self.transformed_space = transformer.build_required_space( self.space, type_requirement="real", shape_requirement="flattened", dist_requirement="linear", )
def parallel_coordinates(experiment, with_evc_tree=True, order=None, colorscale="YlOrRd", **kwargs): """Plotly implementation of `orion.plotting.parallel_coordinates`""" def build_frame(): """Builds the dataframe for the plot""" names = list(experiment.space.keys()) df = experiment.to_pandas(with_evc_tree=with_evc_tree) df = df.loc[df["status"] == "completed"] if df.empty: return df df[names] = df[names].transform( functools.partial(_curate_params, space=experiment.space)) df = _flatten_dims(df, experiment.space) return df def infer_order(space, order): """Create order if not passed, otherwise verify it""" params = orion.analysis.base.flatten_params(space, order) if order is None: fidelity_dims = [ dim for dim in experiment.space.values() if isinstance(dim, Fidelity) ] fidelity = fidelity_dims[0].name if fidelity_dims else None if fidelity in params: del params[params.index(fidelity)] params.insert(0, fidelity) return params def get_dimension(data, name, dim): dim_data = dict(label=name, values=data[name]) if dim.type == "categorical": categories = dim.interval() dim_data["tickvals"] = list(range(len(categories))) dim_data["ticktext"] = categories else: dim_data["range"] = dim.interval() return dim_data if not experiment: raise ValueError("Parameter 'experiment' is None") df = build_frame() if df.empty: return go.Figure() trial = experiment.fetch_trials_by_status("completed")[0] flattened_space = build_required_space(experiment.space, shape_requirement="flattened") dimensions = [ get_dimension(df, name, flattened_space[name]) for name in infer_order(experiment.space, order) ] objective_name = trial.objective.name objectives = df["objective"] omin = min(df["objective"]) omax = max(df["objective"]) dimensions.append( dict(label=objective_name, range=(omin, omax), values=objectives)) fig = go.Figure(data=go.Parcoords( line=dict( color=objectives, colorscale=colorscale, showscale=True, cmin=omin, cmax=omax, colorbar=dict(title=objective_name), ), dimensions=dimensions, )) fig.update_layout( title=f"Parallel Coordinates Plot for experiment '{experiment.name}'") return fig
def test_not_supported_requirement(self, space_each_type): """Require something which is not supported.""" with pytest.raises(TypeError) as exc: build_required_space('fasdfasf', space_each_type) assert 'Unsupported' in str(exc.value)
def partial_dependency(trials, space, params=None, model="RandomForestRegressor", n_grid_points=10, n_samples=50, **kwargs): """ Calculates the partial dependency of parameters in a collection of :class:`Trial`. Parameters ---------- trials: DataFrame or dict A dataframe of trials containing, at least, the columns 'objective' and 'id'. Or a dict equivalent. space: Space object A space object from an experiment. params: list of str, optional The parameters to include in the computation. All parameters are included by default. model: str Name of the regression model to use. Can be one of - AdaBoostRegressor - BaggingRegressor - ExtraTreesRegressor - GradientBoostingRegressor - RandomForestRegressor (Default) n_grid_points: int Number of points in the grid to compute partial dependency. Default is 10. n_samples: int Number of samples to randomly generate the grid used to compute the partial dependency. Default is 50. **kwargs Arguments for the regressor model. Returns ------- dict Dictionary of DataFrames. Each combination of parameters as keys (dim1.name, dim2.name) and for each parameters individually (dim1.name). Columns are (dim1.name, dim2.name, objective) or (dim1.name, objective). """ params = flatten_params(space, params) flattened_space = build_required_space( space, dist_requirement="linear", type_requirement="numerical", shape_requirement="flattened", ) if trials.empty or trials.shape[0] == 0: return {} data = to_numpy(trials, space) data = flatten_numpy(data, flattened_space) model = train_regressor(model, data, **kwargs) data = flattened_space.sample(n_samples) data = pandas.DataFrame(data, columns=flattened_space.keys()) partial_dependencies = dict() for x_i, x_name in enumerate(params): grid, averages, stds = partial_dependency_grid(flattened_space, model, [x_name], data, n_grid_points) grid = reverse(flattened_space, grid) partial_dependencies[x_name] = (grid, averages, stds) for y_i in range(x_i + 1, len(params)): y_name = params[y_i] grid, averages, stds = partial_dependency_grid( flattened_space, model, [x_name, y_name], data, n_grid_points) grid = reverse(flattened_space, grid) partial_dependencies[(x_name, y_name)] = (grid, averages, stds) return partial_dependencies
def lpi( trials, space, mode="best", model="RandomForestRegressor", n_points=20, n_runs=10, **kwargs ): """ Calculates the Local Parameter Importance for a collection of :class:`orion.core.worker.trial.Trial`. For more information on the metric, see original paper at https://ml.informatik.uni-freiburg.de/papers/18-LION12-CAVE.pdf. Biedenkapp, André, et al. "Cave: Configuration assessment, visualization and evaluation." International Conference on Learning and Intelligent Optimization. Springer, Cham, 2018. Parameters ---------- trials: DataFrame or dict A dataframe of trials containing, at least, the columns 'objective' and 'id'. Or a dict equivalent. space: Space object A space object from an experiment. mode: str Mode to compute the LPI. - ``best``: Take the best trial found as the anchor for the LPI - ``linear``: Recompute LPI for all values on a grid model: str Name of the regression model to use. Can be one of - AdaBoostRegressor - BaggingRegressor - ExtraTreesRegressor - GradientBoostingRegressor - RandomForestRegressor (Default) n_points: int Number of points to compute the variances. Default is 20. n_runs: int Number of runs to compute the standard error of the LPI. Default is 10. ``**kwargs`` Arguments for the regressor model. Returns ------- DataFrame LPI value for each parameter. If ``mode`` is `linear`, then a list of param values and LPI metrics are returned in a DataFrame format. """ flattened_space = build_required_space( space, dist_requirement="linear", type_requirement="numerical", shape_requirement="flattened", ) if trials.empty or trials.shape[0] == 0: return pd.DataFrame( data=[0] * len(flattened_space), index=flattened_space.keys(), columns=["LPI"], ) data = to_numpy(trials, space) data = flatten_numpy(data, flattened_space) best_point = data[numpy.argmin(data[:, -1])] rng = numpy.random.RandomState(kwargs.pop("random_state", None)) results = numpy.zeros((n_runs, len(flattened_space))) for i in range(n_runs): trained_model = train_regressor( model, data, random_state=rng.randint(2 ** 32 - 1), **kwargs ) results[i] = modes[mode](best_point, flattened_space, trained_model, n_points) averages = results.mean(0) standard_errors = results.std(0) frame = pd.DataFrame( data=numpy.array([averages, standard_errors]).T, index=flattened_space.keys(), columns=["LPI", "STD"], ) return frame
def flatten_params(space, params=None): """Return the params of the corresponding flat space If no params are passed, returns all flattened params. If params are passed, returns the corresponding flattened params. Parameters ---------- space: Space object A space object from an experiment. params: list of str, optional The parameters to select from the search space. If the flattened search space contains flattened params such as ('y' -> 'y[0]', 'y[1]'), passing 'y' in the list of params will returned the flattened version ['y[0]', 'y[1]'] Examples -------- If space has x~uniform(0, 1) and y~uniform(0, 1, shape=(1, 2)). >>> flatten_params(space) ['x', 'y[0,0]', 'y[0,1]'] >>> flatten_params(space, params=['x']) ['x'] >>> flatten_params(space, params=['x', 'y']) ['x', 'y[0,0]', 'y[0,1]'] >>> flatten_params(space, params=['x', 'y[0,1]']) ['x', 'y[0,1]'] >>> flatten_params(space, params=['y[0,1]', 'x']) ['x', 'y[0,1]'] Raises ------ ValueError If one of the parameter names passed is not in the flattened space. """ keys = set(space.keys()) flattened_keys = set( build_required_space( space, dist_requirement="linear", type_requirement="numerical", shape_requirement="flattened", ).keys()) if params is None: return sorted(flattened_keys) flattened_params = [] for param in params: if param not in flattened_keys and param not in keys: raise ValueError( f"Parameter {param} not contained in space: {flattened_keys}") elif param not in flattened_keys and param in keys: dim = space[param] flattened_params += [ f'{dim.name}[{",".join(map(str, index))}]' for index in itertools.product(*map(range, dim.shape)) ] else: flattened_params.append(param) return flattened_params