def setUp(self): self.emergency_buffer_settings = SimpleHypergrid( name='emergency_buffer_config', dimensions=[ DiscreteDimension(name='log2_emergency_buffer_size', min=0, max=16), CategoricalDimension(name='use_colors', values=[True, False]) ]) self.emergency_buffer_color = SimpleHypergrid( name='emergency_buffer_color', dimensions=[ CategoricalDimension(name='color', values=['Maroon', 'Crimson', 'Tanager']) ]) self.emergency_buffer_settings_with_color = self.emergency_buffer_settings.join( subgrid=self.emergency_buffer_color, on_external_dimension=CategoricalDimension(name='use_colors', values=[True])) self.hierarchical_settings = SimpleHypergrid( name='communication_channel_config', dimensions=[ DiscreteDimension(name='num_readers', min=1, max=64), DiscreteDimension(name='log2_buffer_size', min=10, max=24), CategoricalDimension(name='use_emergency_buffer', values=[True, False]) ]).join(subgrid=self.emergency_buffer_settings_with_color, on_external_dimension=CategoricalDimension( name='use_emergency_buffer', values=[True]))
def setUpClass(cls) -> None: cls.simple_hypergrid = SimpleHypergrid( name='simple_adaptee', dimensions=[ CategoricalDimension(name='categorical_mixed_types', values=['red', True, False, 1]), DiscreteDimension(name='one_to_ten', min=1, max=10), ContinuousDimension(name='zero_to_one', min=0, max=1), OrdinalDimension(name='ordinal_mixed_types', ordered_values=[1, False, 'two']) ] ) cls.hierarchical_hypergrid = SimpleHypergrid( name='hierarchical_adaptee', dimensions=[ CategoricalDimension(name='categorical_mixed_types', values=['red', True, False, 3]), DiscreteDimension(name='one_to_ten', min=1, max=10), ContinuousDimension(name='zero_to_one', min=0, max=1), OrdinalDimension(name='ordinal_mixed_types', ordered_values=[3, False, 'two']) ] ).join( subgrid=SimpleHypergrid( name="nested_grid", dimensions=[ CategoricalDimension(name='categorical_mixed_types', values=['red', False, True, 3]), DiscreteDimension(name='one_to_ten', min=1, max=10), ContinuousDimension(name='zero_to_one', min=0, max=1), OrdinalDimension(name='ordinal_mixed_types', ordered_values=[3, 'two', False]) ] ), on_external_dimension=CategoricalDimension("categorical_mixed_types", values=[True]) )
def decode_discrete_dimension( serialized: OptimizerService_pb2.DiscreteDimension ) -> DiscreteDimension: assert isinstance(serialized, OptimizerService_pb2.DiscreteDimension) return DiscreteDimension(name=serialized.Name, min=serialized.Min, max=serialized.Max)
class BayesianOptimizerConfig(metaclass=DefaultConfigMeta): CONFIG_SPACE = SimpleHypergrid( name="bayesian_optimizer_config", dimensions=[ CategoricalDimension( name="surrogate_model_implementation", values=[HomogeneousRandomForestRegressionModel.__name__]), CategoricalDimension(name="experiment_designer_implementation", values=[ExperimentDesigner.__name__]), DiscreteDimension( name="min_samples_required_for_guided_design_of_experiments", min=2, max=10000) ]).join( subgrid=HomogeneousRandomForestRegressionModelConfig.CONFIG_SPACE, on_external_dimension=CategoricalDimension( name="surrogate_model_implementation", values=[HomogeneousRandomForestRegressionModel.__name__ ])).join(subgrid=ExperimentDesignerConfig.CONFIG_SPACE, on_external_dimension=CategoricalDimension( name="experiment_designer_implementation", values=[ExperimentDesigner.__name__])) _DEFAULT = Point( surrogate_model_implementation=HomogeneousRandomForestRegressionModel. __name__, experiment_designer_implementation=ExperimentDesigner.__name__, min_samples_required_for_guided_design_of_experiments=10, homogeneous_random_forest_regression_model_config= HomogeneousRandomForestRegressionModelConfig.DEFAULT, experiment_designer_config=ExperimentDesignerConfig.DEFAULT)
def test_discrete_dimension(self): discrete_dimension = DiscreteDimension(name='discrete', min=1, max=100) serialized = OptimizerServiceEncoder.encode_discrete_dimension( discrete_dimension) deserialized = OptimizerServiceDecoder.decode_discrete_dimension( serialized) assert isinstance(serialized, OptimizerService_pb2.DiscreteDimension) assert discrete_dimension == deserialized
class LruCacheConfig(metaclass=DefaultConfigMeta): CONFIG_SPACE = SimpleHypergrid( name='lru_cache_config', dimensions=[ DiscreteDimension('cache_size', min=1, max=2 ** 12) ] ) _DEFAULT = Point( cache_size=100 )
def setup_class(cls) -> None: cls.simple_hypergrid = SimpleHypergrid( name='simple_adaptee', dimensions=[ CategoricalDimension(name='categorical_mixed_types', values=['red', True, False, 5]), DiscreteDimension(name='one_to_ten', min=1, max=10), ContinuousDimension(name='z_one', min=-1, max=2), ContinuousDimension(name='z_two', min=-2, max=1), ContinuousDimension(name='z_3', min=-2, max=-1), OrdinalDimension(name='ordinal_mixed_types', ordered_values=[1, False, 'two']) ]) cls.unbalanced_hierarchical_hypergrid = SimpleHypergrid( name='hierarchical_adaptee', dimensions=[ CategoricalDimension(name='categorical_mixed_types', values=['red', True, False, 3]), DiscreteDimension(name='one_to_ten', min=1, max=10), ContinuousDimension(name='x1', min=-1, max=1), ContinuousDimension(name='x2', min=-1, max=1), OrdinalDimension(name='ordinal_mixed_types', ordered_values=[3, False, 'two']) ]).join(subgrid=SimpleHypergrid( name="nested_grid", dimensions=[ CategoricalDimension(name='categorical_mixed_types', values=['red', False, True, 3]), DiscreteDimension(name='one_to_ten', min=1, max=10), ContinuousDimension(name='x1', min=-1, max=1), ContinuousDimension(name='x2', min=-1, max=1), OrdinalDimension(name='ordinal_mixed_types', ordered_values=[3, 'two', False]) ]), on_external_dimension=CategoricalDimension( "categorical_mixed_types", values=[True])) cls.balanced_hierarchical_hypergrid = ThreeLevelQuadratic( ).parameter_space
def _add_one_hot_encoded_dimensions(self, adaptee_dimension_name, temp_df_for_fit: DataFrame) -> None: my_target_data = self._adaptee_to_target_data_dict[ adaptee_dimension_name] my_ohe_output = my_target_data.one_hot_encoder.fit_transform( temp_df_for_fit) my_target_data.num_dummy_dims = my_ohe_output.shape[1] for i in range(my_target_data.num_dummy_dims): target_dim_name = f'{adaptee_dimension_name}{self.ohe_target_column_suffix}{i}' my_target_data.target_dims.append(target_dim_name) self._target.add_dimension( DiscreteDimension(name=target_dim_name, min=0, max=1)) self._all_one_hot_encoded_target_dimension_names.append( target_dim_name)
class HomogeneousRandomForestRegressionModelConfig(RegressionModelConfig): CONFIG_SPACE = SimpleHypergrid( name="homogeneous_random_forest_regression_model_config", dimensions=[ DiscreteDimension(name="n_estimators", min=1, max=100), ContinuousDimension(name="features_fraction_per_estimator", min=0, max=1, include_min=False, include_max=True), ContinuousDimension(name="samples_fraction_per_estimator", min=0, max=1, include_min=False, include_max=True), CategoricalDimension(name="regressor_implementation", values=[DecisionTreeRegressionModel.__name__]), ] ).join( subgrid=DecisionTreeRegressionModelConfig.CONFIG_SPACE, on_external_dimension=CategoricalDimension(name="regressor_implementation", values=[DecisionTreeRegressionModel.__name__]) ) _DEFAULT = Point( n_estimators=5, features_fraction_per_estimator=1, samples_fraction_per_estimator=0.7, regressor_implementation=DecisionTreeRegressionModel.__name__, decision_tree_regression_model_config=DecisionTreeRegressionModelConfig.DEFAULT ) def __init__( self, n_estimators=_DEFAULT.n_estimators, features_fraction_per_estimator=_DEFAULT.features_fraction_per_estimator, samples_fraction_per_estimator=_DEFAULT.samples_fraction_per_estimator, regressor_implementation=_DEFAULT.regressor_implementation, decision_tree_regression_model_config: Point()=_DEFAULT.decision_tree_regression_model_config ): self.n_estimators = n_estimators self.features_fraction_per_estimator = features_fraction_per_estimator self.samples_fraction_per_estimator = samples_fraction_per_estimator self.regressor_implementation = regressor_implementation assert regressor_implementation == DecisionTreeRegressionModel.__name__ self.decision_tree_regression_model_config = DecisionTreeRegressionModelConfig.create_from_config_point(decision_tree_regression_model_config) @classmethod def contains(cls, config): # pylint: disable=unused-argument return True # TODO: see if you can remove this class entirely.
def _map_categorical_dimension(self, adaptee_dimension: CategoricalDimension) -> DiscreteDimension: """ Translates a categorical dimension into a discrete dimension and persists the mappings. :param adaptee_dimension: :return: """ forward_mapping = {} backward_mapping = {} for i, value in enumerate(adaptee_dimension): forward_mapping[value] = i backward_mapping[i] = value self._adaptee_to_target_dimension_mappings[adaptee_dimension.name] = forward_mapping self._target_to_adaptee_dimension_mappings[adaptee_dimension.name] = backward_mapping target_dimension = DiscreteDimension( name=adaptee_dimension.name, min=0, max=len(adaptee_dimension) - 1 ) return target_dimension
class RandomSearchOptimizerConfig(metaclass=DefaultConfigMeta): CONFIG_SPACE = SimpleHypergrid(name="random_search_optimizer_config", dimensions=[ DiscreteDimension( name="num_samples_per_iteration", min=1, max=1000) ]) _DEFAULT = Point(num_samples_per_iteration=1000) @classmethod def create_from_config_point(cls, config_point): config_key_value_pairs = { param_name: value for param_name, value in config_point } return cls(**config_key_value_pairs) def __init__(self, num_samples_per_iteration=_DEFAULT.num_samples_per_iteration): self.num_samples_per_iteration = num_samples_per_iteration
import json import os import sys mlos_root_path = os.environ['MLOS_ROOT'] mlos_python_root_path = os.path.join(mlos_root_path, 'Source', 'Mlos.Python') sys.path.append(mlos_python_root_path) from mlos.Spaces import SimpleHypergrid, EmptyDimension, CategoricalDimension, ContinuousDimension, DiscreteDimension, OrdinalDimension from mlos.Spaces.HypergridsJsonEncoderDecoder import HypergridJsonEncoder, HypergridJsonDecoder continuous = ContinuousDimension(name='continuous', min=1, max=10) discrete = DiscreteDimension(name='discrete', min=1, max=10) ordinal = OrdinalDimension(name='ordinal', ordered_values=[1,2,3,4,5,6,7,8,9,10]) categorical = CategoricalDimension(name='categorical', values=[1,2,3,4,5,6,7,8,9,10]) simple_hypergrid = SimpleHypergrid( name='all_kinds_of_dimensions', dimensions = [ continuous, discrete, ordinal, categorical ] ) py_simple_hypergrid_json_string = json.dumps(simple_hypergrid, cls=HypergridJsonEncoder)
class DecisionTreeRegressionModelConfig(RegressionModelConfig): """ A configuration object for a decision tree regression model. This class is responsible for validating that its objects are valid configurations. DecisionTreeRegressionModel will take an object of this class to actually create the model. """ class Criterion(Enum): """ The function to measure the quality of a split. Supported criteria are 'mse' for the mean squared error, which is equal to variance reduction as feature selection criterion and minimizes the L2 loss using the mean of each terminal node, 'friedman_mse', which uses mean squared error with Friedman’s improvement score for potential splits, and 'mae' for the mean absolute error, which minimizes the L1 loss using the median of each terminal node. Copied from scikit-learn docs. """ MSE = 'mse' FRIEDMAN_MSE = 'friedman_mse' MAE = 'mae' class Splitter(Enum): """ The strategy used to choose the split at each node. Supported strategies are “best” to choose the best split and “random” to choose the best random split. """ BEST = "best" RANDOM = "random" class MaxFeaturesFunc(Enum): AUTO = "auto" SQRT = "sqrt" LOG2 = "log2" CONFIG_SPACE = SimpleHypergrid( name="decision_tree_regression_model_config", dimensions=[ CategoricalDimension( name="criterion", values=[criterion.value for criterion in Criterion]), CategoricalDimension( name="splitter", values=[splitter.value for splitter in Splitter]), DiscreteDimension(name="max_depth", min=0, max=2**10), DiscreteDimension(name="min_samples_split", min=2, max=2**10), DiscreteDimension(name="min_samples_leaf", min=3, max=2**10), ContinuousDimension(name="min_weight_fraction_leaf", min=0.0, max=0.5), CategoricalDimension( name="max_features", values=[function.value for function in MaxFeaturesFunc]), DiscreteDimension(name="max_leaf_nodes", min=0, max=2**10), ContinuousDimension(name="min_impurity_decrease", min=0.0, max=2**10), ContinuousDimension(name="ccp_alpha", min=0.0, max=2**10), DiscreteDimension(name="min_samples_to_fit", min=1, max=2**32), DiscreteDimension(name="n_new_samples_before_refit", min=1, max=2**32) ]) _DEFAULT = Point(criterion=Criterion.MSE.value, splitter=Splitter.RANDOM.value, max_depth=0, min_samples_split=2, min_samples_leaf=3, min_weight_fraction_leaf=0.0, max_features=MaxFeaturesFunc.AUTO.value, max_leaf_nodes=0, min_impurity_decrease=0.0, ccp_alpha=0.0, min_samples_to_fit=10, n_new_samples_before_refit=10) @classmethod def contains(cls, config): return Point(criterion=config.criterion, splitter=config.splitter, max_depth=config.max_depth, min_samples_split=config.min_samples_split, min_samples_leaf=config.min_samples_leaf, min_weight_fraction_leaf=config.min_weight_fraction_leaf, max_features=config.max_features, max_leaf_nodes=config.max_leaf_nodes, min_impurity_decrease=config.min_impurity_decrease, ccp_alpha=config.ccp_alpha, min_samples_to_fit=config.min_samples_to_fit, n_new_samples_before_refit=config. n_new_samples_before_refit) in cls.CONFIG_SPACE @classmethod def create_from_config_point(cls, config_point): assert cls.contains(config_point) config_key_value_pairs = { param_name: value for param_name, value in config_point } return cls(**config_key_value_pairs) def __init__( self, criterion=Criterion.MSE.value, splitter=Splitter.RANDOM.value, max_depth=0, min_samples_split=2, # TODO: decouple the int/float interpretation min_samples_leaf=3, # Default to 3 so that there is variance. # TODO: decouple the int/float interpretation min_weight_fraction_leaf=0.0, max_features=MaxFeaturesFunc.AUTO. value, # TODO: decouple the int/float/str interpretation random_state=None, max_leaf_nodes=0, min_impurity_decrease=0.0, ccp_alpha=0.0, min_samples_to_fit=_DEFAULT.min_samples_to_fit, n_new_samples_before_refit=_DEFAULT.n_new_samples_before_refit): """ :param criterion: The function to measure the quality of a split. :param splitter: The strategy used to choose the split at each node. :param max_depth: The maximum depth of the tree. If None, then nodes are expanded until all leaves are pure or until all leaves contain less than min_samples_split samples. :param min_samples_split: The minimum number of samples required to split an internal node. :param min_samples_leaf: The minimum number of samples required to be at a leaf node. :param min_weight_fraction_leaf: The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided. :param max_features: The number of features to consider when looking for the best split. :param random_state: If int, random_state is the seed used by the random number generator; If RandomState instance, random_state is the random number generator; If None, the random number generator is the RandomState instance used by np.random. :param max_leaf_nodes: Grow a tree with max_leaf_nodes in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes. :param min_impurity_decrease: A node will be split if this split induces a decrease of the impurity greater than or equal to this value. :param ccp_alpha: complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than ccp_alpha will be chosen. By default, no pruning is performed. See Minimal Cost-Complexity Pruning for details. :param min_samples_to_fit: minimum number of samples before it makes sense to try to fit this tree :param n_new_samples_before_refit: It makes little sense to refit every model for every sample. This parameter controls how frequently we refit the decision tree. """ self.criterion = criterion self.splitter = splitter self.max_depth = max_depth self.min_samples_split = min_samples_split self.min_samples_leaf = min_samples_leaf self.min_weight_fraction_leaf = min_weight_fraction_leaf self.max_features = max_features self.random_state = random_state self.max_leaf_nodes = max_leaf_nodes self.min_impurity_decrease = min_impurity_decrease self.ccp_alpha = ccp_alpha self.min_samples_to_fit = min_samples_to_fit self.n_new_samples_before_refit = n_new_samples_before_refit @property def max_depth_value(self): if self.max_depth == 0: return None return self.max_depth @property def max_leaf_nodes_value(self): if self.max_leaf_nodes == 0 or self.max_leaf_nodes == 1: return None return self.max_leaf_nodes
# # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # from mlos.Optimizers.RegressionModels.DecisionTreeRegressionModel import DecisionTreeRegressionModel, decision_tree_config_store from mlos.Spaces import SimpleHypergrid, ContinuousDimension, DiscreteDimension, CategoricalDimension, Point from mlos.Spaces.Configs.ComponentConfigStore import ComponentConfigStore homogeneous_random_forest_config_store = ComponentConfigStore( parameter_space=SimpleHypergrid( name="homogeneous_random_forest_regression_model_config", dimensions=[ DiscreteDimension(name="n_estimators", min=1, max=10000), ContinuousDimension(name="features_fraction_per_estimator", min=0, max=1, include_min=False, include_max=True), ContinuousDimension(name="samples_fraction_per_estimator", min=0, max=1, include_min=False, include_max=True), CategoricalDimension(name="regressor_implementation", values=[DecisionTreeRegressionModel.__name__ ]), CategoricalDimension(name="bootstrap", values=[True, False]) ]).join(subgrid=decision_tree_config_store.parameter_space, on_external_dimension=CategoricalDimension( name="regressor_implementation", values=[DecisionTreeRegressionModel.__name__])),
class SklearnRidgeRegressionModelConfig(RegressionModelConfig): class Solver(Enum): """ From https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html: Solver to use in the computational routines: * ‘auto’ chooses the solver automatically based on the type of data. * ‘svd’ uses a Singular Value Decomposition of X to compute the Ridge coefficients. More stable for singular matrices than ‘cholesky’. * ‘cholesky’ uses the standard scipy.linalg.solve function to obtain a closed-form solution. * ‘sparse_cg’ uses the conjugate gradient solver as found in scipy.sparse.linalg.cg. As an iterative algorithm, this solver is more appropriate than ‘cholesky’ for large-scale data (possibility to set tol and max_iter). * ‘lsqr’ uses the dedicated regularized least-squares routine scipy.sparse.linalg.lsqr. It is the fastest and uses an iterative procedure. * ‘sag’ uses a Stochastic Average Gradient descent, and ‘saga’ uses its improved, unbiased version named SAGA. Both methods also use an iterative procedure, and are often faster than other solvers when both n_samples and n_features are large. Note that ‘sag’ and ‘saga’ fast convergence is only guaranteed on features with approximately the same scale. You can preprocess the data with a scaler from sklearn.preprocessing. All last five solvers support both dense and sparse data. However, only ‘sag’ and ‘sparse_cg’ supports sparse input when fit_intercept is True. """ AUTO = 'auto' # default SVD = 'svd' CHOLESKY = 'cholesky' LSQR = 'lsqr' SPARSE_CG = 'sparse_cg' SAG = 'sag' SAGA = 'saga' CONFIG_SPACE = SimpleHypergrid( name="sklearn_ridge_regression_model_config", dimensions=[ ContinuousDimension(name="alpha", min=0, max=2 ** 16), CategoricalDimension(name="fit_intercept", values=[False, True]), CategoricalDimension(name="normalize", values=[False, True]), CategoricalDimension(name="copy_x", values=[False, True]), DiscreteDimension(name="max_iter", min=0, max=10 ** 5), ContinuousDimension(name="tol", min=0, max=2 ** 10), CategoricalDimension(name="solver", values=[solver.value for solver in Solver]), ] ) _DEFAULT = Point( alpha=1.0, fit_intercept=True, normalize=False, copy_x=True, max_iter=1000, tol=10 ** -4, solver=Solver.AUTO.value ) @classmethod def contains(cls, config): return Point( alpha=config.alpha, fit_intercept=config.fit_intercept, normalize=config.normalize, copy_x=config.copy_x, max_iter=config.max_iter, tol=config.tol, random_state=config.random_state, solver=config.solver ) in cls.CONFIG_SPACE @classmethod def create_from_config_point(cls, config_point): assert cls.contains(config_point) config_key_value_pairs = {param_name: value for param_name, value in config_point} return cls(**config_key_value_pairs) def __init__( self, alpha=_DEFAULT.alpha, fit_intercept=_DEFAULT.fit_intercept, normalize=_DEFAULT.normalize, copy_x=_DEFAULT.copy_x, max_iter=_DEFAULT.max_iter, tol=_DEFAULT.tol, random_state=None, solver=_DEFAULT.solver ): """ Ridge parameters: :param alpha:Regularization strength; must be a positive float. Defaults to 1.0. :param fit_intercept: Whether to calculate the intercept for this model. :param normalize: This parameter is ignored when ``fit_intercept`` is set to False. If True, the regressors X will be normalized before regression by subtracting the mean and dividing by the l2-norm. :param copy_x: If ``True``, X will be copied; else, it may be overwritten. :param max_iter: The maximum number of iterations :param tol: The tolerance for the optimization: if the updates are smaller than ``tol``, the optimization code checks the dual gap for optimality and continues until it is smaller than ``tol``. :param solver: Solver to use in the computational routines: - 'auto' chooses the solver automatically based on the type of data. - 'svd' uses a Singular Value Decomposition of X to compute the Ridge coefficients. More stable for singular matrices than 'cholesky'. - 'cholesky' uses the standard scipy.linalg.solve function to obtain a closed-form solution. - 'sparse_cg' uses the conjugate gradient solver as found in scipy.sparse.linalg.cg. As an iterative algorithm, this solver is more appropriate than 'cholesky' for large-scale data (possibility to set `tol` and `max_iter`). - 'lsqr' uses the dedicated regularized least-squares routine scipy.sparse.linalg.lsqr. It is the fastest and uses an iterative procedure. - 'sag' uses a Stochastic Average Gradient descent, and 'saga' uses its improved, unbiased version named SAGA. Both methods also use an iterative procedure, and are often faster than other solvers when both n_samples and n_features are large. Note that 'sag' and 'saga' fast convergence is only guaranteed on features with approximately the same scale. You can preprocess the data with a scaler from sklearn.preprocessing. :param random_state: The seed of the pseudo random number generator that selects a random feature to update. Used when ``selection`` == 'random'. """ self.alpha = alpha self.fit_intercept = fit_intercept self.normalize = normalize self.copy_x = copy_x self.max_iter = max_iter self.tol = tol self.random_state = random_state self.solver = solver
class SklearnRandomForestRegressionModelConfig(metaclass=DefaultConfigMeta): class MaxFeatures(Enum): """ The number of features to consider when looking for the best split - If "auto", then `max_features=n_features`. - If "sqrt", then `max_features=sqrt(n_features)`. - If "log2", then `max_features=log2(n_features)`. - If None, then `max_features=n_features`. """ AUTO = "auto" SQRT = "sqrt" LOG2 = "log2" class Criterion(Enum): """ The function to measure the quality of a split. Supported criteria are "mse" for the mean squared error, which is equal to variance reduction as feature selection criterion, and "mae" for the mean absolute error. """ MSE = "mse" MAE = "mae" CONFIG_SPACE = SimpleHypergrid( name="sklearn_random_forest_regression_model_config", dimensions=[ DiscreteDimension(name="n_estimators", min=1, max=2**10), CategoricalDimension( name="criterion", values=[criterion.value for criterion in Criterion]), DiscreteDimension(name="max_depth", min=0, max=2**10), ContinuousDimension(name="min_samples_split", min=2, max=2**10), ContinuousDimension(name="min_samples_leaf", min=1, max=2**10), ContinuousDimension(name="min_weight_fraction_leaf", min=0, max=0.5), CategoricalDimension( name="max_features", values=[max_feature.value for max_feature in MaxFeatures]), DiscreteDimension(name="max_leaf_nodes", min=0, max=2**10), ContinuousDimension(name="min_impurity_decrease", min=0, max=2**10), CategoricalDimension(name="bootstrap", values=[False, True]), CategoricalDimension(name="oob_score", values=[False, True]), DiscreteDimension(name="n_jobs", min=1, max=2**10), CategoricalDimension(name="warm_start", values=[False, True]), ContinuousDimension(name="ccp_alpha", min=0, max=2**10), ContinuousDimension(name="max_samples", min=0, max=2**10) ]) _DEFAULT = Point( n_estimators=100, criterion=Criterion.MSE.value, max_depth= 0, # overloading 0 as None to deal with sklearn param type interpretation min_samples_split=2, min_samples_leaf=1, min_weight_fraction_leaf=0.0, max_features=MaxFeatures.AUTO.value, max_leaf_nodes= 0, # overloading 0 as None to deal with sklearn param type interpretation min_impurity_decrease=0, bootstrap=True, oob_score=False, n_jobs=1, warm_start=False, ccp_alpha=0, max_samples=0) @classmethod def contains(cls, config): return Point(n_estimators=config.n_estimators, criterion=config.criterion, max_depth=config.max_depth, min_samples_split=config.min_samples_split, min_samples_leaf=config.min_samples_leaf, min_weight_fraction_leaf=config.min_weight_fraction_leaf, max_features=config.max_features, max_leaf_nodes=config.max_leaf_nodes, min_impurity_decrease=config.min_impurity_decrease, bootstrap=config.bootstrap, oob_score=config.oob_score, n_jobs=config.n_jobs, warm_start=config.warm_start, ccp_alpha=config.ccp_alpha, max_samples=config.max_samples) in cls.CONFIG_SPACE @classmethod def create_from_config_point(cls, config_point): assert cls.contains(config_point) config_key_value_pairs = { param_name: value for param_name, value in config_point } return cls(**config_key_value_pairs) def __init__(self, n_estimators=_DEFAULT.n_estimators, criterion=_DEFAULT.criterion, max_depth=_DEFAULT.max_depth, min_samples_split=_DEFAULT.min_samples_split, min_samples_leaf=_DEFAULT.min_samples_leaf, min_weight_fraction_leaf=_DEFAULT.min_weight_fraction_leaf, max_features=_DEFAULT.max_features, max_leaf_nodes=_DEFAULT.max_leaf_nodes, min_impurity_decrease=_DEFAULT.min_impurity_decrease, bootstrap=_DEFAULT.bootstrap, oob_score=_DEFAULT.oob_score, n_jobs=_DEFAULT.n_jobs, warm_start=_DEFAULT.warm_start, ccp_alpha=_DEFAULT.ccp_alpha, max_samples=_DEFAULT.max_samples): """ Random Forest parameters: :param n_estimators: The number of trees in the forest. :param criterion: The function to measure the quality of a split. Supported criteria are "mse" for the mean squared error, which is equal to variance reduction as feature selection criterion, and "mae" for the mean absolute error. :param max_depth: The maximum depth of the tree. If None, then nodes are expanded until all leaves are pure or until all leaves contain less than min_samples_split samples. :param min_samples_split: The minimum number of samples required to split an internal node :param min_samples_leaf: The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least ``min_samples_leaf`` training samples in each of the left and right branches. This may have the effect of smoothing the model, especially in regression. :param min_weight_fraction_leaf: The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. :param max_features: The number of features to consider when looking for the best split - If "auto", then `max_features=n_features`. - If "sqrt", then `max_features=sqrt(n_features)`. - If "log2", then `max_features=log2(n_features)`. - If None, then `max_features=n_features`. :param max_leaf_nodes: Grow trees with ``max_leaf_nodes`` in best-first fashion. :param min_impurity_decrease: A node will be split if this split induces a decrease of the impurity greater than or equal to this value. :param bootstrap: Whether bootstrap samples are used when building trees. If False, the whole dataset is used to build each tree. :param oob_score: Whether to use out-of-bag samples to estimate the R^2 on unseen data. :param n_jobs: The number of jobs to run in parallel. :meth:`fit`, :meth:`predict`, :meth:`decision_path` and :meth:`apply` are all parallelized over the trees. :param warm_start: When set to ``True``, reuse the solution of the previous call to fit and add more estimators to the ensemble, otherwise, just fit a whole new forest. :param ccp_alpha: Complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than ``ccp_alpha`` will be chosen. .. versionadded:: 0.22 :param max_samples: If bootstrap is True, the number of samples to draw from X to train each base estimator. """ self.n_estimators = n_estimators self.criterion = criterion self.max_depth = max_depth self.min_samples_split = min_samples_split self.min_samples_leaf = min_samples_leaf self.min_weight_fraction_leaf = min_weight_fraction_leaf self.max_features = max_features self.max_leaf_nodes = max_leaf_nodes self.min_impurity_decrease = min_impurity_decrease self.bootstrap = bootstrap self.oob_score = oob_score self.n_jobs = n_jobs self.warm_start = warm_start self.ccp_alpha = ccp_alpha self.max_samples = max_samples # sklearn random forest regressor interprets max_depth = None differently than an int value # so mapping max_depth=0 to None here @property def max_depth_value(self): if self.max_depth == 0: return None return self.max_depth @property # similar mapping here as for max_depth def max_leaf_nodes_value(self): if self.max_leaf_nodes == 0 or self.max_leaf_nodes == 1: return None return self.max_leaf_nodes @property # similar mapping here as for max_depth def max_sample_value(self): if self.max_samples == 0: return None return self.max_samples
class SklearnLassoRegressionModelConfig(metaclass=DefaultConfigMeta): class Selection(Enum): """ Parameter to sklearn lasso regressor controlling how model coefficients are selected for update. From https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Lasso.html: If set to ‘random’, a random coefficient is updated every iteration rather than looping over features sequentially by default. This (setting to ‘random’) often leads to significantly faster convergence especially when tol is higher than 1e-4. """ CYCLIC = 'cyclic' RANDOM = 'random' CONFIG_SPACE = SimpleHypergrid( name="sklearn_lasso_regression_model_config", dimensions=[ ContinuousDimension(name="alpha", min=0, max=2**16), CategoricalDimension(name="fit_intercept", values=[False, True]), CategoricalDimension(name="normalize", values=[False, True]), CategoricalDimension(name="precompute", values=[False, True]), CategoricalDimension(name="copy_x", values=[False, True]), DiscreteDimension(name="max_iter", min=0, max=10**5), ContinuousDimension(name="tol", min=0, max=2**10), CategoricalDimension(name="warm_start", values=[False, True]), CategoricalDimension(name="positive", values=[False, True]), CategoricalDimension( name="selection", values=[selection.value for selection in Selection]), ]) _DEFAULT = Point( selection=Selection.CYCLIC.value, alpha=1.0, fit_intercept=False, normalize=False, # sklearn model expects precompute type str, bool, array-like, so setting to default and exclude list option precompute=False, copy_x=True, max_iter=2000, tol=10**-4, warm_start=False, positive=False) @classmethod def contains(cls, config): return Point(alpha=config.alpha, fit_intercept=config.fit_intercept, normalize=config.normalize, precompute=config.precompute, copy_x=config.copy_x, max_iter=config.max_iter, tol=config.tol, warm_start=config.warm_start, positive=config.positive, selection=config.selection) in cls.CONFIG_SPACE @classmethod def create_from_config_point(cls, config_point): assert cls.contains(config_point) config_key_value_pairs = { param_name: value for param_name, value in config_point } return cls(**config_key_value_pairs) def __init__(self, alpha=_DEFAULT.alpha, fit_intercept=_DEFAULT.fit_intercept, normalize=_DEFAULT.normalize, precompute=_DEFAULT.precompute, copy_x=_DEFAULT.copy_x, max_iter=_DEFAULT.max_iter, tol=_DEFAULT.tol, warm_start=_DEFAULT.warm_start, positive=_DEFAULT.positive, random_state=None, selection=_DEFAULT.selection): """ Lasso parameters: :param alpha: Constant that multiplies the L1 term. Defaults to 1.0. :param fit_intercept: Whether to calculate the intercept for this model. :param normalize: This parameter is ignored when ``fit_intercept`` is set to False. If True, the regressors X will be normalized before regression by subtracting the mean and dividing by the l2-norm. :param precompute: Whether to use a precomputed Gram matrix to speed up calculations. If set to ``'auto'`` let us decide. :param copy_x: If ``True``, X will be copied; else, it may be overwritten. :param max_iter: The maximum number of iterations :param tol: The tolerance for the optimization: if the updates are smaller than ``tol``, the optimization code checks the dual gap for optimality and continues until it is smaller than ``tol``. :param warm_start: When set to True, reuse the solution of the previous call to fit as initialization, otherwise, just erase the previous solution. :param positive: When set to ``True``, forces the coefficients to be positive. :param random_state: The seed of the pseudo random number generator that selects a random feature to update. Used when ``selection`` == 'random'. :param selection: {'cyclic', 'random'} If set to 'random', a random coefficient is updated every iteration rather than looping over features sequentially by default. """ self.alpha = alpha self.fit_intercept = fit_intercept self.normalize = normalize self.precompute = precompute self.copy_x = copy_x self.max_iter = max_iter self.tol = tol self.warm_start = warm_start self.positive = positive self.random_state = random_state self.selection = selection
# Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # from mlos.Optimizers.RegressionModels.LassoCrossValidatedRegressionModel import LassoCrossValidatedRegressionModel, lasso_cross_validated_config_store from mlos.Optimizers.RegressionModels.SklearnRandomForestRegressionModelConfig import SklearnRandomForestRegressionModelConfig from mlos.Spaces import SimpleHypergrid, DiscreteDimension, CategoricalDimension, Point from mlos.Spaces.Configs.ComponentConfigStore import ComponentConfigStore # TODO : Add back the RidgeRegressionModel boosting_root_model option after adding new RidgeCrossValidatedRegressionModel # TODO : Move from Sklearn random forest to HomogeneousRandomForest regression_enhanced_random_forest_config_store = ComponentConfigStore( parameter_space=SimpleHypergrid( name="regression_enhanced_random_forest_regression_model_config", dimensions=[ DiscreteDimension(name="max_basis_function_degree", min=1, max=10), CategoricalDimension( name="residual_model_name", values=[SklearnRandomForestRegressionModelConfig.__name__]), CategoricalDimension( name="boosting_root_model_name", values=[LassoCrossValidatedRegressionModel.__name__]), CategoricalDimension( name="perform_initial_random_forest_hyper_parameter_search", values=[True, False]) ]).join(subgrid=lasso_cross_validated_config_store.parameter_space, on_external_dimension=CategoricalDimension( name="boosting_root_model_name", values=[LassoCrossValidatedRegressionModel.__name__])). join(subgrid=SklearnRandomForestRegressionModelConfig.CONFIG_SPACE, on_external_dimension=CategoricalDimension(
values=[ Flower.__name__, NestedPolynomialObjective.__name__, PolynomialObjective.__name__, ThreeLevelQuadratic.__name__, ]) ]).join(subgrid=PolynomialObjective.CONFIG_SPACE, on_external_dimension=CategoricalDimension( name="implementation", values=[ PolynomialObjective.__name__ ])).join(subgrid=SimpleHypergrid( name="nested_polynomial_objective_config", dimensions=[ DiscreteDimension(name="num_nested_polynomials", min=1, max=128), CategoricalDimension( name="nested_function_implementation", values=[PolynomialObjective.__name__]) ]).join(subgrid=PolynomialObjective.CONFIG_SPACE, on_external_dimension=CategoricalDimension( name="nested_function_implementation", values=[PolynomialObjective.__name__])), on_external_dimension=CategoricalDimension( name="implementation", values=[NestedPolynomialObjective.__name__])), default=Point( implementation=PolynomialObjective.__name__, # TODO: move polynomial objective to config store polynomial_objective_config=PolynomialObjective._DEFAULT, # pylint: disable=protected-access,
class RegressionEnhancedRandomForestRegressionModelConfig( RegressionModelConfig): """A configuration object for RERF model. Class responsible for validating its objects are valid hyper parameters for the sklearn classes: Lasso (https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Lasso.html), Ridge (https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html#sklearn.linear_model.Ridge) and RandomForest () """ CONFIG_SPACE = SimpleHypergrid( name="regression_enhanced_random_forest_regression_model_config", dimensions=[ DiscreteDimension(name="max_basis_function_degree", min=1, max=10), CategoricalDimension( name="residual_model_name", values=[SklearnRandomForestRegressionModelConfig.__name__]), CategoricalDimension( name="boosting_root_model_name", values=[ SklearnLassoRegressionModelConfig.__name__, SklearnRidgeRegressionModelConfig.__name__ ]), ContinuousDimension(name="min_abs_root_model_coef", min=0, max=2**10), CategoricalDimension( name="perform_initial_root_model_hyper_parameter_search", values=[False, True]), CategoricalDimension( name="perform_initial_random_forest_hyper_parameter_search", values=[False, True]) ]).join( subgrid=SklearnLassoRegressionModelConfig.CONFIG_SPACE, on_external_dimension=CategoricalDimension( name="boosting_root_model_name", values=[SklearnLassoRegressionModelConfig.__name__]) ).join( subgrid=SklearnRidgeRegressionModelConfig.CONFIG_SPACE, on_external_dimension=CategoricalDimension( name="boosting_root_model_name", values=[SklearnRidgeRegressionModelConfig.__name__]) ).join(subgrid=SklearnRandomForestRegressionModelConfig.CONFIG_SPACE, on_external_dimension=CategoricalDimension( name="residual_model_name", values=[SklearnRandomForestRegressionModelConfig.__name__])) _DEFAULT = Point( max_basis_function_degree=2, residual_model_name=SklearnRandomForestRegressionModelConfig.__name__, boosting_root_model_name=SklearnLassoRegressionModelConfig.__name__, min_abs_root_model_coef=0.01, sklearn_lasso_regression_model_config=SklearnLassoRegressionModelConfig .DEFAULT, sklearn_ridge_regression_model_config=SklearnRidgeRegressionModelConfig .DEFAULT, sklearn_random_forest_regression_model_config= SklearnRandomForestRegressionModelConfig.DEFAULT, perform_initial_root_model_hyper_parameter_search=True, perform_initial_random_forest_hyper_parameter_search=True) @classmethod def contains(cls, config): return config in cls.CONFIG_SPACE
# import pandas as pd from mlos.Exceptions import UtilityValueUnavailableException from mlos.Optimizers.OptimizationProblem import OptimizationProblem from mlos.Optimizers.ExperimentDesigner.UtilityFunctionOptimizers.UtilityFunctionOptimizer import UtilityFunctionOptimizer from mlos.Optimizers.ExperimentDesigner.UtilityFunctions.UtilityFunction import UtilityFunction from mlos.Spaces import SimpleHypergrid, DiscreteDimension, Point from mlos.Spaces.Configs.ComponentConfigStore import ComponentConfigStore from mlos.Tracer import trace random_search_optimizer_config_store = ComponentConfigStore( parameter_space=SimpleHypergrid(name="random_search_optimizer_config", dimensions=[ DiscreteDimension( name="num_samples_per_iteration", min=1, max=100000) ]), default=Point(num_samples_per_iteration=1000)) class RandomSearchOptimizer(UtilityFunctionOptimizer): """ Performs a random search over the search space. This is the simplest optimizer to implement and a good baseline for all other optimizers to beat. """ def __init__(self, optimizer_config: Point, optimization_problem: OptimizationProblem,
from mlos.Logger import create_logger from mlos.Optimizers.ExperimentDesigner.UtilityFunctions.UtilityFunction import UtilityFunction from mlos.Optimizers.ParetoFrontier import ParetoFrontier from mlos.Optimizers.RegressionModels.MultiObjectiveRegressionModel import MultiObjectiveRegressionModel from mlos.Optimizers.RegressionModels.Prediction import Prediction from mlos.Optimizers.RegressionModels.MultiObjectivePrediction import MultiObjectivePrediction from mlos.Spaces import SimpleHypergrid, DiscreteDimension, Point from mlos.Spaces.Configs.ComponentConfigStore import ComponentConfigStore from mlos.Tracer import trace multi_objective_probability_of_improvement_utility_function_config_store = ComponentConfigStore( parameter_space=SimpleHypergrid( name="multi_objective_probability_of_improvement_config", dimensions=[ DiscreteDimension(name="num_monte_carlo_samples", min=100, max=1000) ]), default=Point(num_monte_carlo_samples=100)) class MultiObjectiveProbabilityOfImprovementUtilityFunction(UtilityFunction): """Computes the probability of improvement (POI) of a set of configurations over the existing pareto frontier. We are up against several requirements here: we need to be able to predict the probability of improvement in a multi-dimensional objective space. Our assumptions (see below) make each distribution a multi-dimensional blob that's cut in two by a nearly arbitrarily complex surface of the pareto frontier. This precludes any closed form solution to the POI question. Thus, we take a Monte Carlo approach: we generate a bunch of points from the predictive distribution, compute the proportion of these that are dominated by the existing pareto frontier, and use that proportion as an estimator for the probability of improvement.
optimizer_evaluator_config_store = ComponentConfigStore( description="Describes the configuration space for the OptimizerEvaluator.\n" "\n" "Dimensions:\n" "- num_iterations: how many optimization iterations to run.\n" "- evaluation_frequency: how often should the evaluator capture the optima and goodness of fit metrics (e.g. every 10 iterations).\n" "- include_pickled_optimizer_in_report: should the state of the optimizer be pickled and saved.\n" "- include_pickled_objective_function_in_report: should the final state of the objective function be pickled and saved.\n" "- report_regression_model_goodness_of_fit: should the goodness of fit metrics be included in the evaluation report.\n" "- report_optima_over_time: should the optima over time be included in the evaluation report.\n" "- include_execution_trace_in_report: should the execution trace produced by mlos.Tracer be included in the evaluation report.", parameter_space=SimpleHypergrid( name="optimizer_evaluator", dimensions=[ DiscreteDimension(name="num_iterations", min=1, max=2**32), DiscreteDimension(name="evaluation_frequency", min=1, max=2**10), CategoricalDimension(name="include_pickled_optimizer_in_report", values=[True, False]), CategoricalDimension(name="include_pickled_objective_function_in_report", values=[True, False]), CategoricalDimension(name="report_regression_model_goodness_of_fit", values=[True, False]), CategoricalDimension(name="report_optima_over_time", values=[True, False]), CategoricalDimension(name="report_pareto_over_time", values=[True, False]), CategoricalDimension(name="report_pareto_volume_over_time", values=[True, False]), CategoricalDimension(name="include_execution_trace_in_report", values=[True, False]), ] ), default=Point( num_iterations=100, evaluation_frequency=10, include_pickled_optimizer_in_report=True, include_pickled_objective_function_in_report=True,
# # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # from mlos.Examples.SmartCache.CacheImplementations.XruCache import XruCache from mlos.Spaces import DiscreteDimension, Point, SimpleHypergrid from mlos.Spaces.Configs.ComponentConfigStore import ComponentConfigStore lru_cache_config_store = ComponentConfigStore(parameter_space=SimpleHypergrid( name='lru_cache_config', dimensions=[DiscreteDimension('cache_size', min=1, max=2**12)]), default=Point(cache_size=100)) class LruCache(XruCache): """ An implementation of a Least Recently Used cache. We maintain a dictionary and a linked list both pointing to the same cache entry. Whenever an entry is touched (and when it is first inserted) it gets moved to the head of the linked list (in O(1) time). Whenever we try to push a new entry into a full cache, we expel the entry that's at the tail of the list (since it is the least recently used one). """ def __init__(self, max_size, logger): XruCache.__init__(self, max_size=max_size, logger=logger) def evict(self): removed_node = self._list.remove_at_tail() evicted_entry = removed_node.cache_entry del self._dict[removed_node.cache_entry.key]
def setUp(self): self.cache_param_space = SimpleHypergrid( name='cache_param_space', dimensions=[ CategoricalDimension(name='cache_implementation_name', values=['lru_cache', 'associative_cache']) ] ) self.lru_cache_param_space = SimpleHypergrid( name='lru_cache_config', dimensions=[ DiscreteDimension(name='size', min=1, max=2**20), OrdinalDimension(name='color', ordered_values=['green', 'orange', 'red']) ] ) self.associative_cache_implementation_root_param_space = SimpleHypergrid( name='associative_cache_config', dimensions=[ CategoricalDimension(name='hash_function_name', values=['mod_prime_hash_function', 'lowest_bits']), CategoricalDimension(name='bucket_implementation', values=['single_value', 'binary_search_tree', 'linked_list']) ] ) self.mod_prime_hash_function_param_space = SimpleHypergrid( name='mod_prime_hash_function', dimensions=[ OrdinalDimension(name='prime', ordered_values=[1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59]) ] ) self.lowest_bits_param_space = SimpleHypergrid( name='lowest_bits', dimensions=[ DiscreteDimension(name='num_bits', min=1, max=64) ] ) self.binary_search_tree_param_space = SimpleHypergrid( name='binary_search_tree', dimensions=[ DiscreteDimension(name='max_depth', min=1, max=2**10) ] ) self.linked_list_param_space = SimpleHypergrid( name='linked_list', dimensions=[ DiscreteDimension(name='max_length', min=1, max=2**10) ] ) self.associative_cache_implementation_param_space = self.associative_cache_implementation_root_param_space.join( subgrid=self.mod_prime_hash_function_param_space, on_external_dimension=CategoricalDimension(name='hash_function_name', values=['mod_prime_hash_function']) ).join( subgrid=self.lowest_bits_param_space, on_external_dimension=CategoricalDimension(name='hash_function_name', values='lowest_bits') ).join( subgrid=self.binary_search_tree_param_space, on_external_dimension=CategoricalDimension(name='bucket_implementation', values=['binary_search_tree']) ) self.cache_param_space = self.cache_param_space.join( subgrid=self.lru_cache_param_space, on_external_dimension=CategoricalDimension(name='cache_implementation_name', values=['lru_cache']) ).join( subgrid=self.associative_cache_implementation_param_space, on_external_dimension=CategoricalDimension(name='cache_implementation_name', values=['associative_cache']) ).join( subgrid=self.linked_list_param_space, on_external_dimension=CategoricalDimension(name='associative_cache_config.bucket_implementation', values=['linked_list']) )
Parameter to sklearn LassoCV regressor controlling how model coefficients are selected for update. From https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LassoCV.html: If set to ‘random’, a random coefficient is updated every iteration rather than looping over features sequentially by default. This (setting to ‘random’) often leads to significantly faster convergence especially when tol is higher than 1e-4. """ CYCLIC = 'cyclic' RANDOM = 'random' lasso_cross_validated_config_store = ComponentConfigStore( parameter_space=SimpleHypergrid( name="lasso_regression_model_config", dimensions=[ ContinuousDimension(name="eps", min=0, max=10.0**-3), DiscreteDimension(name="num_alphas", min=0, max=200), CategoricalDimension(name="fit_intercept", values=[False, True]), CategoricalDimension(name="normalize", values=[False, True]), CategoricalDimension(name="precompute", values=[False, True]), DiscreteDimension(name="max_iter", min=100, max=5 * 10**3), ContinuousDimension(name="tol", min=0, max=1.0), CategoricalDimension(name="copy_x", values=[False, True]), DiscreteDimension(name="num_cross_validations", min=2, max=10), CategoricalDimension(name="verbose", values=[False, True]), DiscreteDimension(name="num_jobs", min=1, max=2), CategoricalDimension(name="positive", values=[False, True]), CategoricalDimension( name="selection", values=[selection.value for selection in Selection]) ]), default=Point(
from mlos.Optimizers.RegressionModels.HomogeneousRandomForestConfigStore import homogeneous_random_forest_config_store from mlos.Optimizers.RegressionModels.HomogeneousRandomForestRegressionModel import HomogeneousRandomForestRegressionModel bayesian_optimizer_config_store = ComponentConfigStore( parameter_space=SimpleHypergrid( name="bayesian_optimizer_config", dimensions=[ CategoricalDimension( name="surrogate_model_implementation", values=[ HomogeneousRandomForestRegressionModel.__name__, ]), CategoricalDimension(name="experiment_designer_implementation", values=[ExperimentDesigner.__name__]), DiscreteDimension( name="min_samples_required_for_guided_design_of_experiments", min=2, max=10000) ]).join( subgrid=homogeneous_random_forest_config_store.parameter_space, on_external_dimension=CategoricalDimension( name="surrogate_model_implementation", values=[ HomogeneousRandomForestRegressionModel.__name__ ])).join( subgrid=experiment_designer_config_store.parameter_space, on_external_dimension=CategoricalDimension( name="experiment_designer_implementation", values=[ExperimentDesigner.__name__])), default=Point( surrogate_model_implementation=HomogeneousRandomForestRegressionModel. __name__,
# Licensed under the MIT License. # import math import pandas as pd from mlos.OptimizerEvaluationTools.ObjectiveFunctionBase import ObjectiveFunctionBase from mlos.OptimizerEvaluationTools.SyntheticFunctions.NestedPolynomialObjective import NestedPolynomialObjective, nested_polynomial_objective_config_space from mlos.Optimizers.OptimizationProblem import Objective, OptimizationProblem from mlos.Spaces import CategoricalDimension, ContinuousDimension, DiscreteDimension, Hypergrid, Point, SimpleHypergrid from mlos.Utils.KeyOrderedDict import KeyOrderedDict multi_objective_nested_polynomial_config_space = SimpleHypergrid( name="multi_objective_nested_polynomial_config", dimensions=[ DiscreteDimension(name="num_objectives", min=1, max=10), CategoricalDimension(name="objective_function_implementation", values=[NestedPolynomialObjective.__name__]) ]).join(subgrid=nested_polynomial_objective_config_space, on_external_dimension=CategoricalDimension( name="objective_function_implementation", values=[NestedPolynomialObjective.__name__])) class MultiObjectiveNestedPolynomialObjective(ObjectiveFunctionBase): """A multi-objective function where each objective is a separate NestedPolynomialObjective. """ def __init__(self, objective_function_config: Point): assert objective_function_config in multi_objective_nested_polynomial_config_space ObjectiveFunctionBase.__init__(self, objective_function_config)
# Licensed under the MIT License. # from mlos.Spaces import SimpleHypergrid, CategoricalDimension, DiscreteDimension, Point smart_cache_workload_generator_config_space = SimpleHypergrid( name='smart_cache_workload_generator_config', dimensions=[ CategoricalDimension('workload_type', values=[ 'fibonacci', 'random_key_from_range', 'cyclical_key_from_range' ]) ]).join( subgrid=SimpleHypergrid(name='fibonacci_config', dimensions=[ DiscreteDimension('min', min=0, max=2**10), DiscreteDimension('range_width', min=0, max=2**20) ]), on_external_dimension=CategoricalDimension('workload_type', values=['fibonacci']), ).join( subgrid=SimpleHypergrid(name='random_key_from_range_config', dimensions=[ DiscreteDimension('min', min=0, max=2**10), DiscreteDimension('range_width', min=0, max=2**20) ]), on_external_dimension=CategoricalDimension(
class PolynomialObjective: """ A class to enable evaluation of optimizer convergence characteristics. An instance allows one to evaluate an arbitrarily high degree (<=16) polynomial objective in up to 16 dimensions in which some number of coefficients have been eliminated (set to zero). """ CONFIG_SPACE = SimpleHypergrid( name="polynomial_objective_config", dimensions=[ DiscreteDimension(name='seed', min=1, max=2**32), DiscreteDimension(name='input_domain_dimension', min=1, max=5), DiscreteDimension(name='max_degree', min=1, max=5), CategoricalDimension(name='include_mixed_coefficients', values=[False, True]), ContinuousDimension(name='percent_coefficients_zeroed', min=0.0, max=1.0), ContinuousDimension(name='coefficient_domain_min', min=-2**32, max=2**32), ContinuousDimension(name='coefficient_domain_width', min=1, max=2**32), CategoricalDimension(name='include_noise', values=[False, True]), ContinuousDimension(name='noise_coefficient_of_variation', min=0.0, max=1.0) ]) # needs constraint coefficient_domain_min < coefficient_domain_max _DEFAULT = Point(seed=17, input_domain_dimension=2, max_degree=2, include_mixed_coefficients=True, percent_coefficients_zeroed=0.0, coefficient_domain_min=-10.0, coefficient_domain_width=9.0, include_noise=False, noise_coefficient_of_variation=0.0) """ Initialization parameters: :param coefficients If specified, will override random generation of a polynomial even if `seed` arg is specified """ def __init__( self, seed: int = 17, input_domain_dimension: int = _DEFAULT.input_domain_dimension, max_degree: int = _DEFAULT.max_degree, include_mixed_coefficients: bool = _DEFAULT. include_mixed_coefficients, percent_coefficients_zeroed: float = _DEFAULT. percent_coefficients_zeroed, coefficient_domain_min: float = _DEFAULT.coefficient_domain_min, coefficient_domain_width: float = _DEFAULT. coefficient_domain_width, include_noise: bool = _DEFAULT.include_noise, noise_coefficient_of_variation: float = _DEFAULT. noise_coefficient_of_variation, coefficients=None, logger=None): if logger is None: logger = create_logger("PolynomialObjective") self.logger = logger self.seed = seed self.input_domain_dimension = input_domain_dimension self.max_degree = max_degree self.include_mixed_coefficients = include_mixed_coefficients self.percent_coefficients_zeroed = percent_coefficients_zeroed self.coefficient_domain_min = coefficient_domain_min self.coefficient_domain_max = coefficient_domain_min + coefficient_domain_width self.coefficients = coefficients self.include_noise = include_noise self.noise_coefficient_of_variation = noise_coefficient_of_variation self.coef_ = [] # confirm min < max constraint assert coefficient_domain_min < self.coefficient_domain_max, 'Minimum coefficient range must be less than maximum' self.polynomial_features_ = PolynomialFeatures(degree=self.max_degree) discarded_x = np.array([1] * self.input_domain_dimension).reshape( 1, -1) poly_terms_x = self.polynomial_features_.fit_transform(discarded_x) self.num_expected_coefficients_ = len(poly_terms_x[0]) if coefficients is None: # generate random polynomial if coefficients not specified np.random.seed(self.seed) self.coef_ = [ r for r in np.random.uniform(self.coefficient_domain_min, self.coefficient_domain_max, self.num_expected_coefficients_) ] # temporarily a list to be convert to np.array if self.percent_coefficients_zeroed > 0.0: # reset a random subset of coefficients to 0 num_coef_to_zero = int(self.percent_coefficients_zeroed * self.num_expected_coefficients_) true_poly_term_indices_without_effects = np.random.choice( range(self.num_expected_coefficients_), size=num_coef_to_zero, replace=False) for zi in true_poly_term_indices_without_effects: self.coef_[zi] = 0.0 # eliminate mixed variable terms if requested if not self.include_mixed_coefficients: # zero term coef where input's power != max_degree for ip, p in enumerate(self.polynomial_features_.powers_): max_variable_degree = np.max(p) if max_variable_degree != self.max_degree: self.coef_[ip] = 0.0 # convert to np.array to enable matmul evaluations self.coef_ = np.array(self.coef_) else: # test if degree specified is consistent with number of coefficients passed num_specified_coefficients = len(self.coefficients) assert num_specified_coefficients == self.num_expected_coefficients_, \ 'Failed to find sufficient number of coefficients for specified polynomial degree' self.coef_ = np.array(self.coefficients) def evaluate(self, x): y = np.matmul(self.polynomial_features_.fit_transform(x), self.coef_) if self.include_noise: cv = self.noise_coefficient_of_variation y_cv = np.tile(cv, [len(y)]) y_std = np.abs(y_cv * y) y = np.random.normal(y, y_std, [len(y)]) return y