def setUp(self):
self.emergency_buffer_settings = SimpleHypergrid(
name='emergency_buffer_config',
dimensions=[
DiscreteDimension(name='log2_emergency_buffer_size',
min=0,
max=16),
CategoricalDimension(name='use_colors', values=[True, False])
])
self.emergency_buffer_color = SimpleHypergrid(
name='emergency_buffer_color',
dimensions=[
CategoricalDimension(name='color',
values=['Maroon', 'Crimson', 'Tanager'])
])
self.emergency_buffer_settings_with_color = self.emergency_buffer_settings.join(
subgrid=self.emergency_buffer_color,
on_external_dimension=CategoricalDimension(name='use_colors',
values=[True]))
self.hierarchical_settings = SimpleHypergrid(
name='communication_channel_config',
dimensions=[
DiscreteDimension(name='num_readers', min=1, max=64),
DiscreteDimension(name='log2_buffer_size', min=10, max=24),
CategoricalDimension(name='use_emergency_buffer',
values=[True, False])
]).join(subgrid=self.emergency_buffer_settings_with_color,
on_external_dimension=CategoricalDimension(
name='use_emergency_buffer', values=[True]))
def setUpClass(cls) -> None:
cls.simple_hypergrid = SimpleHypergrid(
name='simple_adaptee',
dimensions=[
CategoricalDimension(name='categorical_mixed_types', values=['red', True, False, 1]),
DiscreteDimension(name='one_to_ten', min=1, max=10),
ContinuousDimension(name='zero_to_one', min=0, max=1),
OrdinalDimension(name='ordinal_mixed_types', ordered_values=[1, False, 'two'])
]
)
cls.hierarchical_hypergrid = SimpleHypergrid(
name='hierarchical_adaptee',
dimensions=[
CategoricalDimension(name='categorical_mixed_types', values=['red', True, False, 3]),
DiscreteDimension(name='one_to_ten', min=1, max=10),
ContinuousDimension(name='zero_to_one', min=0, max=1),
OrdinalDimension(name='ordinal_mixed_types', ordered_values=[3, False, 'two'])
]
).join(
subgrid=SimpleHypergrid(
name="nested_grid",
dimensions=[
CategoricalDimension(name='categorical_mixed_types', values=['red', False, True, 3]),
DiscreteDimension(name='one_to_ten', min=1, max=10),
ContinuousDimension(name='zero_to_one', min=0, max=1),
OrdinalDimension(name='ordinal_mixed_types', ordered_values=[3, 'two', False])
]
),
on_external_dimension=CategoricalDimension("categorical_mixed_types", values=[True])
)
def decode_discrete_dimension(
serialized: OptimizerService_pb2.DiscreteDimension
) -> DiscreteDimension:
assert isinstance(serialized, OptimizerService_pb2.DiscreteDimension)
return DiscreteDimension(name=serialized.Name,
min=serialized.Min,
max=serialized.Max)
class BayesianOptimizerConfig(metaclass=DefaultConfigMeta):
CONFIG_SPACE = SimpleHypergrid(
name="bayesian_optimizer_config",
dimensions=[
CategoricalDimension(
name="surrogate_model_implementation",
values=[HomogeneousRandomForestRegressionModel.__name__]),
CategoricalDimension(name="experiment_designer_implementation",
values=[ExperimentDesigner.__name__]),
DiscreteDimension(
name="min_samples_required_for_guided_design_of_experiments",
min=2,
max=10000)
]).join(
subgrid=HomogeneousRandomForestRegressionModelConfig.CONFIG_SPACE,
on_external_dimension=CategoricalDimension(
name="surrogate_model_implementation",
values=[HomogeneousRandomForestRegressionModel.__name__
])).join(subgrid=ExperimentDesignerConfig.CONFIG_SPACE,
on_external_dimension=CategoricalDimension(
name="experiment_designer_implementation",
values=[ExperimentDesigner.__name__]))
_DEFAULT = Point(
surrogate_model_implementation=HomogeneousRandomForestRegressionModel.
__name__,
experiment_designer_implementation=ExperimentDesigner.__name__,
min_samples_required_for_guided_design_of_experiments=10,
homogeneous_random_forest_regression_model_config=
HomogeneousRandomForestRegressionModelConfig.DEFAULT,
experiment_designer_config=ExperimentDesignerConfig.DEFAULT)
def test_discrete_dimension(self):
discrete_dimension = DiscreteDimension(name='discrete', min=1, max=100)
serialized = OptimizerServiceEncoder.encode_discrete_dimension(
discrete_dimension)
deserialized = OptimizerServiceDecoder.decode_discrete_dimension(
serialized)
assert isinstance(serialized, OptimizerService_pb2.DiscreteDimension)
assert discrete_dimension == deserialized
class LruCacheConfig(metaclass=DefaultConfigMeta):
CONFIG_SPACE = SimpleHypergrid(
name='lru_cache_config',
dimensions=[
DiscreteDimension('cache_size', min=1, max=2 ** 12)
]
)
_DEFAULT = Point(
cache_size=100
)
def setup_class(cls) -> None:
cls.simple_hypergrid = SimpleHypergrid(
name='simple_adaptee',
dimensions=[
CategoricalDimension(name='categorical_mixed_types',
values=['red', True, False, 5]),
DiscreteDimension(name='one_to_ten', min=1, max=10),
ContinuousDimension(name='z_one', min=-1, max=2),
ContinuousDimension(name='z_two', min=-2, max=1),
ContinuousDimension(name='z_3', min=-2, max=-1),
OrdinalDimension(name='ordinal_mixed_types',
ordered_values=[1, False, 'two'])
])
cls.unbalanced_hierarchical_hypergrid = SimpleHypergrid(
name='hierarchical_adaptee',
dimensions=[
CategoricalDimension(name='categorical_mixed_types',
values=['red', True, False, 3]),
DiscreteDimension(name='one_to_ten', min=1, max=10),
ContinuousDimension(name='x1', min=-1, max=1),
ContinuousDimension(name='x2', min=-1, max=1),
OrdinalDimension(name='ordinal_mixed_types',
ordered_values=[3, False, 'two'])
]).join(subgrid=SimpleHypergrid(
name="nested_grid",
dimensions=[
CategoricalDimension(name='categorical_mixed_types',
values=['red', False, True, 3]),
DiscreteDimension(name='one_to_ten', min=1, max=10),
ContinuousDimension(name='x1', min=-1, max=1),
ContinuousDimension(name='x2', min=-1, max=1),
OrdinalDimension(name='ordinal_mixed_types',
ordered_values=[3, 'two', False])
]),
on_external_dimension=CategoricalDimension(
"categorical_mixed_types", values=[True]))
cls.balanced_hierarchical_hypergrid = ThreeLevelQuadratic(
).parameter_space
def _add_one_hot_encoded_dimensions(self, adaptee_dimension_name,
temp_df_for_fit: DataFrame) -> None:
my_target_data = self._adaptee_to_target_data_dict[
adaptee_dimension_name]
my_ohe_output = my_target_data.one_hot_encoder.fit_transform(
temp_df_for_fit)
my_target_data.num_dummy_dims = my_ohe_output.shape[1]
for i in range(my_target_data.num_dummy_dims):
target_dim_name = f'{adaptee_dimension_name}{self.ohe_target_column_suffix}{i}'
my_target_data.target_dims.append(target_dim_name)
self._target.add_dimension(
DiscreteDimension(name=target_dim_name, min=0, max=1))
self._all_one_hot_encoded_target_dimension_names.append(
target_dim_name)
class HomogeneousRandomForestRegressionModelConfig(RegressionModelConfig):
CONFIG_SPACE = SimpleHypergrid(
name="homogeneous_random_forest_regression_model_config",
dimensions=[
DiscreteDimension(name="n_estimators", min=1, max=100),
ContinuousDimension(name="features_fraction_per_estimator", min=0, max=1, include_min=False, include_max=True),
ContinuousDimension(name="samples_fraction_per_estimator", min=0, max=1, include_min=False, include_max=True),
CategoricalDimension(name="regressor_implementation", values=[DecisionTreeRegressionModel.__name__]),
]
).join(
subgrid=DecisionTreeRegressionModelConfig.CONFIG_SPACE,
on_external_dimension=CategoricalDimension(name="regressor_implementation", values=[DecisionTreeRegressionModel.__name__])
)
_DEFAULT = Point(
n_estimators=5,
features_fraction_per_estimator=1,
samples_fraction_per_estimator=0.7,
regressor_implementation=DecisionTreeRegressionModel.__name__,
decision_tree_regression_model_config=DecisionTreeRegressionModelConfig.DEFAULT
)
def __init__(
self,
n_estimators=_DEFAULT.n_estimators,
features_fraction_per_estimator=_DEFAULT.features_fraction_per_estimator,
samples_fraction_per_estimator=_DEFAULT.samples_fraction_per_estimator,
regressor_implementation=_DEFAULT.regressor_implementation,
decision_tree_regression_model_config: Point()=_DEFAULT.decision_tree_regression_model_config
):
self.n_estimators = n_estimators
self.features_fraction_per_estimator = features_fraction_per_estimator
self.samples_fraction_per_estimator = samples_fraction_per_estimator
self.regressor_implementation = regressor_implementation
assert regressor_implementation == DecisionTreeRegressionModel.__name__
self.decision_tree_regression_model_config = DecisionTreeRegressionModelConfig.create_from_config_point(decision_tree_regression_model_config)
@classmethod
def contains(cls, config): # pylint: disable=unused-argument
return True # TODO: see if you can remove this class entirely.
def _map_categorical_dimension(self, adaptee_dimension: CategoricalDimension) -> DiscreteDimension:
""" Translates a categorical dimension into a discrete dimension and persists the mappings.
:param adaptee_dimension:
:return:
"""
forward_mapping = {}
backward_mapping = {}
for i, value in enumerate(adaptee_dimension):
forward_mapping[value] = i
backward_mapping[i] = value
self._adaptee_to_target_dimension_mappings[adaptee_dimension.name] = forward_mapping
self._target_to_adaptee_dimension_mappings[adaptee_dimension.name] = backward_mapping
target_dimension = DiscreteDimension(
name=adaptee_dimension.name,
min=0,
max=len(adaptee_dimension) - 1
)
return target_dimension
class RandomSearchOptimizerConfig(metaclass=DefaultConfigMeta):
CONFIG_SPACE = SimpleHypergrid(name="random_search_optimizer_config",
dimensions=[
DiscreteDimension(
name="num_samples_per_iteration",
min=1,
max=1000)
])
_DEFAULT = Point(num_samples_per_iteration=1000)
@classmethod
def create_from_config_point(cls, config_point):
config_key_value_pairs = {
param_name: value
for param_name, value in config_point
}
return cls(**config_key_value_pairs)
def __init__(self,
num_samples_per_iteration=_DEFAULT.num_samples_per_iteration):
self.num_samples_per_iteration = num_samples_per_iteration
import json
import os
import sys
mlos_root_path = os.environ['MLOS_ROOT']
mlos_python_root_path = os.path.join(mlos_root_path, 'Source', 'Mlos.Python')
sys.path.append(mlos_python_root_path)
from mlos.Spaces import SimpleHypergrid, EmptyDimension, CategoricalDimension, ContinuousDimension, DiscreteDimension, OrdinalDimension
from mlos.Spaces.HypergridsJsonEncoderDecoder import HypergridJsonEncoder, HypergridJsonDecoder
continuous = ContinuousDimension(name='continuous', min=1, max=10)
discrete = DiscreteDimension(name='discrete', min=1, max=10)
ordinal = OrdinalDimension(name='ordinal', ordered_values=[1,2,3,4,5,6,7,8,9,10])
categorical = CategoricalDimension(name='categorical', values=[1,2,3,4,5,6,7,8,9,10])
simple_hypergrid = SimpleHypergrid(
name='all_kinds_of_dimensions',
dimensions = [
continuous,
discrete,
ordinal,
categorical
]
)
py_simple_hypergrid_json_string = json.dumps(simple_hypergrid, cls=HypergridJsonEncoder)
class DecisionTreeRegressionModelConfig(RegressionModelConfig):
""" A configuration object for a decision tree regression model.
This class is responsible for validating that its objects are valid configurations.
DecisionTreeRegressionModel will take an object of this class to actually create
the model.
"""
class Criterion(Enum):
""" The function to measure the quality of a split.
Supported criteria are 'mse' for the mean squared error, which is equal to variance reduction as feature
selection criterion and minimizes the L2 loss using the mean of each terminal node, 'friedman_mse',
which uses mean squared error with Friedman’s improvement score for potential splits, and 'mae' for the
mean absolute error, which minimizes the L1 loss using the median of each terminal node.
Copied from scikit-learn docs.
"""
MSE = 'mse'
FRIEDMAN_MSE = 'friedman_mse'
MAE = 'mae'
class Splitter(Enum):
""" The strategy used to choose the split at each node.
Supported strategies are “best” to choose the best split and “random” to choose the best random split.
"""
BEST = "best"
RANDOM = "random"
class MaxFeaturesFunc(Enum):
AUTO = "auto"
SQRT = "sqrt"
LOG2 = "log2"
CONFIG_SPACE = SimpleHypergrid(
name="decision_tree_regression_model_config",
dimensions=[
CategoricalDimension(
name="criterion",
values=[criterion.value for criterion in Criterion]),
CategoricalDimension(
name="splitter",
values=[splitter.value for splitter in Splitter]),
DiscreteDimension(name="max_depth", min=0, max=2**10),
DiscreteDimension(name="min_samples_split", min=2, max=2**10),
DiscreteDimension(name="min_samples_leaf", min=3, max=2**10),
ContinuousDimension(name="min_weight_fraction_leaf",
min=0.0,
max=0.5),
CategoricalDimension(
name="max_features",
values=[function.value for function in MaxFeaturesFunc]),
DiscreteDimension(name="max_leaf_nodes", min=0, max=2**10),
ContinuousDimension(name="min_impurity_decrease",
min=0.0,
max=2**10),
ContinuousDimension(name="ccp_alpha", min=0.0, max=2**10),
DiscreteDimension(name="min_samples_to_fit", min=1, max=2**32),
DiscreteDimension(name="n_new_samples_before_refit",
min=1,
max=2**32)
])
_DEFAULT = Point(criterion=Criterion.MSE.value,
splitter=Splitter.RANDOM.value,
max_depth=0,
min_samples_split=2,
min_samples_leaf=3,
min_weight_fraction_leaf=0.0,
max_features=MaxFeaturesFunc.AUTO.value,
max_leaf_nodes=0,
min_impurity_decrease=0.0,
ccp_alpha=0.0,
min_samples_to_fit=10,
n_new_samples_before_refit=10)
@classmethod
def contains(cls, config):
return Point(criterion=config.criterion,
splitter=config.splitter,
max_depth=config.max_depth,
min_samples_split=config.min_samples_split,
min_samples_leaf=config.min_samples_leaf,
min_weight_fraction_leaf=config.min_weight_fraction_leaf,
max_features=config.max_features,
max_leaf_nodes=config.max_leaf_nodes,
min_impurity_decrease=config.min_impurity_decrease,
ccp_alpha=config.ccp_alpha,
min_samples_to_fit=config.min_samples_to_fit,
n_new_samples_before_refit=config.
n_new_samples_before_refit) in cls.CONFIG_SPACE
@classmethod
def create_from_config_point(cls, config_point):
assert cls.contains(config_point)
config_key_value_pairs = {
param_name: value
for param_name, value in config_point
}
return cls(**config_key_value_pairs)
def __init__(
self,
criterion=Criterion.MSE.value,
splitter=Splitter.RANDOM.value,
max_depth=0,
min_samples_split=2, # TODO: decouple the int/float interpretation
min_samples_leaf=3, # Default to 3 so that there is variance. # TODO: decouple the int/float interpretation
min_weight_fraction_leaf=0.0,
max_features=MaxFeaturesFunc.AUTO.
value, # TODO: decouple the int/float/str interpretation
random_state=None,
max_leaf_nodes=0,
min_impurity_decrease=0.0,
ccp_alpha=0.0,
min_samples_to_fit=_DEFAULT.min_samples_to_fit,
n_new_samples_before_refit=_DEFAULT.n_new_samples_before_refit):
"""
:param criterion: The function to measure the quality of a split.
:param splitter: The strategy used to choose the split at each node.
:param max_depth: The maximum depth of the tree. If None, then nodes are expanded until all leaves are pure or until all leaves contain less than
min_samples_split samples.
:param min_samples_split: The minimum number of samples required to split an internal node.
:param min_samples_leaf: The minimum number of samples required to be at a leaf node.
:param min_weight_fraction_leaf: The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node.
Samples have equal weight when sample_weight is not provided.
:param max_features: The number of features to consider when looking for the best split.
:param random_state: If int, random_state is the seed used by the random number generator; If RandomState instance, random_state is the random number
generator; If None, the random number generator is the RandomState instance used by np.random.
:param max_leaf_nodes: Grow a tree with max_leaf_nodes in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then
unlimited number of leaf nodes.
:param min_impurity_decrease: A node will be split if this split induces a decrease of the impurity greater than or equal to this value.
:param ccp_alpha: complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than
ccp_alpha will be chosen. By default, no pruning is performed. See Minimal Cost-Complexity Pruning for details.
:param min_samples_to_fit: minimum number of samples before it makes sense to try to fit this tree
:param n_new_samples_before_refit: It makes little sense to refit every model for every sample. This parameter controls
how frequently we refit the decision tree.
"""
self.criterion = criterion
self.splitter = splitter
self.max_depth = max_depth
self.min_samples_split = min_samples_split
self.min_samples_leaf = min_samples_leaf
self.min_weight_fraction_leaf = min_weight_fraction_leaf
self.max_features = max_features
self.random_state = random_state
self.max_leaf_nodes = max_leaf_nodes
self.min_impurity_decrease = min_impurity_decrease
self.ccp_alpha = ccp_alpha
self.min_samples_to_fit = min_samples_to_fit
self.n_new_samples_before_refit = n_new_samples_before_refit
@property
def max_depth_value(self):
if self.max_depth == 0:
return None
return self.max_depth
@property
def max_leaf_nodes_value(self):
if self.max_leaf_nodes == 0 or self.max_leaf_nodes == 1:
return None
return self.max_leaf_nodes
#
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
#
from mlos.Optimizers.RegressionModels.DecisionTreeRegressionModel import DecisionTreeRegressionModel, decision_tree_config_store
from mlos.Spaces import SimpleHypergrid, ContinuousDimension, DiscreteDimension, CategoricalDimension, Point
from mlos.Spaces.Configs.ComponentConfigStore import ComponentConfigStore
homogeneous_random_forest_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name="homogeneous_random_forest_regression_model_config",
dimensions=[
DiscreteDimension(name="n_estimators", min=1, max=10000),
ContinuousDimension(name="features_fraction_per_estimator",
min=0,
max=1,
include_min=False,
include_max=True),
ContinuousDimension(name="samples_fraction_per_estimator",
min=0,
max=1,
include_min=False,
include_max=True),
CategoricalDimension(name="regressor_implementation",
values=[DecisionTreeRegressionModel.__name__
]),
CategoricalDimension(name="bootstrap", values=[True, False])
]).join(subgrid=decision_tree_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
name="regressor_implementation",
values=[DecisionTreeRegressionModel.__name__])),
class SklearnRidgeRegressionModelConfig(RegressionModelConfig):
class Solver(Enum):
"""
From https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html:
Solver to use in the computational routines:
* ‘auto’ chooses the solver automatically based on the type of data.
* ‘svd’ uses a Singular Value Decomposition of X to compute the Ridge coefficients. More stable for
singular matrices than ‘cholesky’.
* ‘cholesky’ uses the standard scipy.linalg.solve function to obtain a closed-form solution.
* ‘sparse_cg’ uses the conjugate gradient solver as found in scipy.sparse.linalg.cg.
As an iterative algorithm, this solver is more appropriate than ‘cholesky’ for
large-scale data (possibility to set tol and max_iter).
* ‘lsqr’ uses the dedicated regularized least-squares routine scipy.sparse.linalg.lsqr.
It is the fastest and uses an iterative procedure.
* ‘sag’ uses a Stochastic Average Gradient descent, and ‘saga’ uses its improved,
unbiased version named SAGA. Both methods also use an iterative procedure, and are
often faster than other solvers when both n_samples and n_features are large.
Note that ‘sag’ and ‘saga’ fast convergence is only guaranteed on features with
approximately the same scale. You can preprocess the data with a scaler from sklearn.preprocessing.
All last five solvers support both dense and sparse data. However, only ‘sag’ and ‘sparse_cg’ supports
sparse input when fit_intercept is True.
"""
AUTO = 'auto' # default
SVD = 'svd'
CHOLESKY = 'cholesky'
LSQR = 'lsqr'
SPARSE_CG = 'sparse_cg'
SAG = 'sag'
SAGA = 'saga'
CONFIG_SPACE = SimpleHypergrid(
name="sklearn_ridge_regression_model_config",
dimensions=[
ContinuousDimension(name="alpha", min=0, max=2 ** 16),
CategoricalDimension(name="fit_intercept", values=[False, True]),
CategoricalDimension(name="normalize", values=[False, True]),
CategoricalDimension(name="copy_x", values=[False, True]),
DiscreteDimension(name="max_iter", min=0, max=10 ** 5),
ContinuousDimension(name="tol", min=0, max=2 ** 10),
CategoricalDimension(name="solver", values=[solver.value for solver in Solver]),
]
)
_DEFAULT = Point(
alpha=1.0,
fit_intercept=True,
normalize=False,
copy_x=True,
max_iter=1000,
tol=10 ** -4,
solver=Solver.AUTO.value
)
@classmethod
def contains(cls, config):
return Point(
alpha=config.alpha,
fit_intercept=config.fit_intercept,
normalize=config.normalize,
copy_x=config.copy_x,
max_iter=config.max_iter,
tol=config.tol,
random_state=config.random_state,
solver=config.solver
) in cls.CONFIG_SPACE
@classmethod
def create_from_config_point(cls, config_point):
assert cls.contains(config_point)
config_key_value_pairs = {param_name: value for param_name, value in config_point}
return cls(**config_key_value_pairs)
def __init__(
self,
alpha=_DEFAULT.alpha,
fit_intercept=_DEFAULT.fit_intercept,
normalize=_DEFAULT.normalize,
copy_x=_DEFAULT.copy_x,
max_iter=_DEFAULT.max_iter,
tol=_DEFAULT.tol,
random_state=None,
solver=_DEFAULT.solver
):
"""
Ridge parameters:
:param alpha:Regularization strength; must be a positive float. Defaults to 1.0.
:param fit_intercept: Whether to calculate the intercept for this model.
:param normalize: This parameter is ignored when ``fit_intercept`` is set to False.
If True, the regressors X will be normalized before regression by
subtracting the mean and dividing by the l2-norm.
:param copy_x: If ``True``, X will be copied; else, it may be overwritten.
:param max_iter: The maximum number of iterations
:param tol: The tolerance for the optimization: if the updates are
smaller than ``tol``, the optimization code checks the
dual gap for optimality and continues until it is smaller
than ``tol``.
:param solver: Solver to use in the computational routines:
- 'auto' chooses the solver automatically based on the type of data.
- 'svd' uses a Singular Value Decomposition of X to compute the Ridge
coefficients. More stable for singular matrices than 'cholesky'.
- 'cholesky' uses the standard scipy.linalg.solve function to
obtain a closed-form solution.
- 'sparse_cg' uses the conjugate gradient solver as found in
scipy.sparse.linalg.cg. As an iterative algorithm, this solver is
more appropriate than 'cholesky' for large-scale data
(possibility to set `tol` and `max_iter`).
- 'lsqr' uses the dedicated regularized least-squares routine
scipy.sparse.linalg.lsqr. It is the fastest and uses an iterative
procedure.
- 'sag' uses a Stochastic Average Gradient descent, and 'saga' uses
its improved, unbiased version named SAGA. Both methods also use an
iterative procedure, and are often faster than other solvers when
both n_samples and n_features are large. Note that 'sag' and
'saga' fast convergence is only guaranteed on features with
approximately the same scale. You can preprocess the data with a
scaler from sklearn.preprocessing.
:param random_state: The seed of the pseudo random number generator that selects a random
feature to update. Used when ``selection`` == 'random'.
"""
self.alpha = alpha
self.fit_intercept = fit_intercept
self.normalize = normalize
self.copy_x = copy_x
self.max_iter = max_iter
self.tol = tol
self.random_state = random_state
self.solver = solver
class SklearnRandomForestRegressionModelConfig(metaclass=DefaultConfigMeta):
class MaxFeatures(Enum):
"""
The number of features to consider when looking for the best split
- If "auto", then `max_features=n_features`.
- If "sqrt", then `max_features=sqrt(n_features)`.
- If "log2", then `max_features=log2(n_features)`.
- If None, then `max_features=n_features`.
"""
AUTO = "auto"
SQRT = "sqrt"
LOG2 = "log2"
class Criterion(Enum):
"""
The function to measure the quality of a split. Supported criteria
are "mse" for the mean squared error, which is equal to variance
reduction as feature selection criterion, and "mae" for the mean
absolute error.
"""
MSE = "mse"
MAE = "mae"
CONFIG_SPACE = SimpleHypergrid(
name="sklearn_random_forest_regression_model_config",
dimensions=[
DiscreteDimension(name="n_estimators", min=1, max=2**10),
CategoricalDimension(
name="criterion",
values=[criterion.value for criterion in Criterion]),
DiscreteDimension(name="max_depth", min=0, max=2**10),
ContinuousDimension(name="min_samples_split", min=2, max=2**10),
ContinuousDimension(name="min_samples_leaf", min=1, max=2**10),
ContinuousDimension(name="min_weight_fraction_leaf",
min=0,
max=0.5),
CategoricalDimension(
name="max_features",
values=[max_feature.value for max_feature in MaxFeatures]),
DiscreteDimension(name="max_leaf_nodes", min=0, max=2**10),
ContinuousDimension(name="min_impurity_decrease", min=0,
max=2**10),
CategoricalDimension(name="bootstrap", values=[False, True]),
CategoricalDimension(name="oob_score", values=[False, True]),
DiscreteDimension(name="n_jobs", min=1, max=2**10),
CategoricalDimension(name="warm_start", values=[False, True]),
ContinuousDimension(name="ccp_alpha", min=0, max=2**10),
ContinuousDimension(name="max_samples", min=0, max=2**10)
])
_DEFAULT = Point(
n_estimators=100,
criterion=Criterion.MSE.value,
max_depth=
0, # overloading 0 as None to deal with sklearn param type interpretation
min_samples_split=2,
min_samples_leaf=1,
min_weight_fraction_leaf=0.0,
max_features=MaxFeatures.AUTO.value,
max_leaf_nodes=
0, # overloading 0 as None to deal with sklearn param type interpretation
min_impurity_decrease=0,
bootstrap=True,
oob_score=False,
n_jobs=1,
warm_start=False,
ccp_alpha=0,
max_samples=0)
@classmethod
def contains(cls, config):
return Point(n_estimators=config.n_estimators,
criterion=config.criterion,
max_depth=config.max_depth,
min_samples_split=config.min_samples_split,
min_samples_leaf=config.min_samples_leaf,
min_weight_fraction_leaf=config.min_weight_fraction_leaf,
max_features=config.max_features,
max_leaf_nodes=config.max_leaf_nodes,
min_impurity_decrease=config.min_impurity_decrease,
bootstrap=config.bootstrap,
oob_score=config.oob_score,
n_jobs=config.n_jobs,
warm_start=config.warm_start,
ccp_alpha=config.ccp_alpha,
max_samples=config.max_samples) in cls.CONFIG_SPACE
@classmethod
def create_from_config_point(cls, config_point):
assert cls.contains(config_point)
config_key_value_pairs = {
param_name: value
for param_name, value in config_point
}
return cls(**config_key_value_pairs)
def __init__(self,
n_estimators=_DEFAULT.n_estimators,
criterion=_DEFAULT.criterion,
max_depth=_DEFAULT.max_depth,
min_samples_split=_DEFAULT.min_samples_split,
min_samples_leaf=_DEFAULT.min_samples_leaf,
min_weight_fraction_leaf=_DEFAULT.min_weight_fraction_leaf,
max_features=_DEFAULT.max_features,
max_leaf_nodes=_DEFAULT.max_leaf_nodes,
min_impurity_decrease=_DEFAULT.min_impurity_decrease,
bootstrap=_DEFAULT.bootstrap,
oob_score=_DEFAULT.oob_score,
n_jobs=_DEFAULT.n_jobs,
warm_start=_DEFAULT.warm_start,
ccp_alpha=_DEFAULT.ccp_alpha,
max_samples=_DEFAULT.max_samples):
"""
Random Forest parameters:
:param n_estimators: The number of trees in the forest.
:param criterion: The function to measure the quality of a split. Supported criteria
are "mse" for the mean squared error, which is equal to variance
reduction as feature selection criterion, and "mae" for the mean
absolute error.
:param max_depth: The maximum depth of the tree. If None, then nodes are expanded until
all leaves are pure or until all leaves contain less than min_samples_split samples.
:param min_samples_split: The minimum number of samples required to split an internal node
:param min_samples_leaf: The minimum number of samples required to be at a leaf node.
A split point at any depth will only be considered if it leaves at
least ``min_samples_leaf`` training samples in each of the left and
right branches. This may have the effect of smoothing the model,
especially in regression.
:param min_weight_fraction_leaf: The minimum weighted fraction of the sum total of weights (of all
the input samples) required to be at a leaf node.
:param max_features: The number of features to consider when looking for the best split
- If "auto", then `max_features=n_features`.
- If "sqrt", then `max_features=sqrt(n_features)`.
- If "log2", then `max_features=log2(n_features)`.
- If None, then `max_features=n_features`.
:param max_leaf_nodes: Grow trees with ``max_leaf_nodes`` in best-first fashion.
:param min_impurity_decrease: A node will be split if this split induces a decrease of the impurity
greater than or equal to this value.
:param bootstrap: Whether bootstrap samples are used when building trees. If False, the
whole dataset is used to build each tree.
:param oob_score: Whether to use out-of-bag samples to estimate the R^2 on unseen data.
:param n_jobs: The number of jobs to run in parallel. :meth:`fit`, :meth:`predict`,
:meth:`decision_path` and :meth:`apply` are all parallelized over the
trees.
:param warm_start: When set to ``True``, reuse the solution of the previous call to fit
and add more estimators to the ensemble, otherwise, just fit a whole
new forest.
:param ccp_alpha: Complexity parameter used for Minimal Cost-Complexity Pruning. The
subtree with the largest cost complexity that is smaller than
``ccp_alpha`` will be chosen.
.. versionadded:: 0.22
:param max_samples: If bootstrap is True, the number of samples to draw from X
to train each base estimator.
"""
self.n_estimators = n_estimators
self.criterion = criterion
self.max_depth = max_depth
self.min_samples_split = min_samples_split
self.min_samples_leaf = min_samples_leaf
self.min_weight_fraction_leaf = min_weight_fraction_leaf
self.max_features = max_features
self.max_leaf_nodes = max_leaf_nodes
self.min_impurity_decrease = min_impurity_decrease
self.bootstrap = bootstrap
self.oob_score = oob_score
self.n_jobs = n_jobs
self.warm_start = warm_start
self.ccp_alpha = ccp_alpha
self.max_samples = max_samples
# sklearn random forest regressor interprets max_depth = None differently than an int value
# so mapping max_depth=0 to None here
@property
def max_depth_value(self):
if self.max_depth == 0:
return None
return self.max_depth
@property
# similar mapping here as for max_depth
def max_leaf_nodes_value(self):
if self.max_leaf_nodes == 0 or self.max_leaf_nodes == 1:
return None
return self.max_leaf_nodes
@property
# similar mapping here as for max_depth
def max_sample_value(self):
if self.max_samples == 0:
return None
return self.max_samples
class SklearnLassoRegressionModelConfig(metaclass=DefaultConfigMeta):
class Selection(Enum):
"""
Parameter to sklearn lasso regressor controlling how model coefficients are selected for update.
From https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Lasso.html:
If set to ‘random’, a random coefficient is updated every iteration rather than looping
over features sequentially by default. This (setting to ‘random’) often leads to significantly
faster convergence especially when tol is higher than 1e-4.
"""
CYCLIC = 'cyclic'
RANDOM = 'random'
CONFIG_SPACE = SimpleHypergrid(
name="sklearn_lasso_regression_model_config",
dimensions=[
ContinuousDimension(name="alpha", min=0, max=2**16),
CategoricalDimension(name="fit_intercept", values=[False, True]),
CategoricalDimension(name="normalize", values=[False, True]),
CategoricalDimension(name="precompute", values=[False, True]),
CategoricalDimension(name="copy_x", values=[False, True]),
DiscreteDimension(name="max_iter", min=0, max=10**5),
ContinuousDimension(name="tol", min=0, max=2**10),
CategoricalDimension(name="warm_start", values=[False, True]),
CategoricalDimension(name="positive", values=[False, True]),
CategoricalDimension(
name="selection",
values=[selection.value for selection in Selection]),
])
_DEFAULT = Point(
selection=Selection.CYCLIC.value,
alpha=1.0,
fit_intercept=False,
normalize=False,
# sklearn model expects precompute type str, bool, array-like, so setting to default and exclude list option
precompute=False,
copy_x=True,
max_iter=2000,
tol=10**-4,
warm_start=False,
positive=False)
@classmethod
def contains(cls, config):
return Point(alpha=config.alpha,
fit_intercept=config.fit_intercept,
normalize=config.normalize,
precompute=config.precompute,
copy_x=config.copy_x,
max_iter=config.max_iter,
tol=config.tol,
warm_start=config.warm_start,
positive=config.positive,
selection=config.selection) in cls.CONFIG_SPACE
@classmethod
def create_from_config_point(cls, config_point):
assert cls.contains(config_point)
config_key_value_pairs = {
param_name: value
for param_name, value in config_point
}
return cls(**config_key_value_pairs)
def __init__(self,
alpha=_DEFAULT.alpha,
fit_intercept=_DEFAULT.fit_intercept,
normalize=_DEFAULT.normalize,
precompute=_DEFAULT.precompute,
copy_x=_DEFAULT.copy_x,
max_iter=_DEFAULT.max_iter,
tol=_DEFAULT.tol,
warm_start=_DEFAULT.warm_start,
positive=_DEFAULT.positive,
random_state=None,
selection=_DEFAULT.selection):
"""
Lasso parameters:
:param alpha: Constant that multiplies the L1 term. Defaults to 1.0.
:param fit_intercept: Whether to calculate the intercept for this model.
:param normalize: This parameter is ignored when ``fit_intercept`` is set to False.
If True, the regressors X will be normalized before regression by
subtracting the mean and dividing by the l2-norm.
:param precompute: Whether to use a precomputed Gram matrix to speed up
calculations. If set to ``'auto'`` let us decide.
:param copy_x: If ``True``, X will be copied; else, it may be overwritten.
:param max_iter: The maximum number of iterations
:param tol: The tolerance for the optimization: if the updates are
smaller than ``tol``, the optimization code checks the
dual gap for optimality and continues until it is smaller
than ``tol``.
:param warm_start: When set to True, reuse the solution of the previous call to fit as
initialization, otherwise, just erase the previous solution.
:param positive: When set to ``True``, forces the coefficients to be positive.
:param random_state: The seed of the pseudo random number generator that selects a random
feature to update. Used when ``selection`` == 'random'.
:param selection: {'cyclic', 'random'} If set to 'random', a random coefficient is updated every iteration
rather than looping over features sequentially by default.
"""
self.alpha = alpha
self.fit_intercept = fit_intercept
self.normalize = normalize
self.precompute = precompute
self.copy_x = copy_x
self.max_iter = max_iter
self.tol = tol
self.warm_start = warm_start
self.positive = positive
self.random_state = random_state
self.selection = selection
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
#
from mlos.Optimizers.RegressionModels.LassoCrossValidatedRegressionModel import LassoCrossValidatedRegressionModel, lasso_cross_validated_config_store
from mlos.Optimizers.RegressionModels.SklearnRandomForestRegressionModelConfig import SklearnRandomForestRegressionModelConfig
from mlos.Spaces import SimpleHypergrid, DiscreteDimension, CategoricalDimension, Point
from mlos.Spaces.Configs.ComponentConfigStore import ComponentConfigStore
# TODO : Add back the RidgeRegressionModel boosting_root_model option after adding new RidgeCrossValidatedRegressionModel
# TODO : Move from Sklearn random forest to HomogeneousRandomForest
regression_enhanced_random_forest_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name="regression_enhanced_random_forest_regression_model_config",
dimensions=[
DiscreteDimension(name="max_basis_function_degree", min=1, max=10),
CategoricalDimension(
name="residual_model_name",
values=[SklearnRandomForestRegressionModelConfig.__name__]),
CategoricalDimension(
name="boosting_root_model_name",
values=[LassoCrossValidatedRegressionModel.__name__]),
CategoricalDimension(
name="perform_initial_random_forest_hyper_parameter_search",
values=[True, False])
]).join(subgrid=lasso_cross_validated_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
name="boosting_root_model_name",
values=[LassoCrossValidatedRegressionModel.__name__])).
join(subgrid=SklearnRandomForestRegressionModelConfig.CONFIG_SPACE,
on_external_dimension=CategoricalDimension(
values=[
Flower.__name__,
NestedPolynomialObjective.__name__,
PolynomialObjective.__name__,
ThreeLevelQuadratic.__name__,
])
]).join(subgrid=PolynomialObjective.CONFIG_SPACE,
on_external_dimension=CategoricalDimension(
name="implementation",
values=[
PolynomialObjective.__name__
])).join(subgrid=SimpleHypergrid(
name="nested_polynomial_objective_config",
dimensions=[
DiscreteDimension(name="num_nested_polynomials",
min=1,
max=128),
CategoricalDimension(
name="nested_function_implementation",
values=[PolynomialObjective.__name__])
]).join(subgrid=PolynomialObjective.CONFIG_SPACE,
on_external_dimension=CategoricalDimension(
name="nested_function_implementation",
values=[PolynomialObjective.__name__])),
on_external_dimension=CategoricalDimension(
name="implementation",
values=[NestedPolynomialObjective.__name__])),
default=Point(
implementation=PolynomialObjective.__name__,
# TODO: move polynomial objective to config store
polynomial_objective_config=PolynomialObjective._DEFAULT, # pylint: disable=protected-access,
class RegressionEnhancedRandomForestRegressionModelConfig(
RegressionModelConfig):
"""A configuration object for RERF model.
Class responsible for validating its objects are valid hyper parameters for the sklearn classes:
Lasso (https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Lasso.html),
Ridge
(https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html#sklearn.linear_model.Ridge)
and
RandomForest ()
"""
CONFIG_SPACE = SimpleHypergrid(
name="regression_enhanced_random_forest_regression_model_config",
dimensions=[
DiscreteDimension(name="max_basis_function_degree", min=1, max=10),
CategoricalDimension(
name="residual_model_name",
values=[SklearnRandomForestRegressionModelConfig.__name__]),
CategoricalDimension(
name="boosting_root_model_name",
values=[
SklearnLassoRegressionModelConfig.__name__,
SklearnRidgeRegressionModelConfig.__name__
]),
ContinuousDimension(name="min_abs_root_model_coef",
min=0,
max=2**10),
CategoricalDimension(
name="perform_initial_root_model_hyper_parameter_search",
values=[False, True]),
CategoricalDimension(
name="perform_initial_random_forest_hyper_parameter_search",
values=[False, True])
]).join(
subgrid=SklearnLassoRegressionModelConfig.CONFIG_SPACE,
on_external_dimension=CategoricalDimension(
name="boosting_root_model_name",
values=[SklearnLassoRegressionModelConfig.__name__])
).join(
subgrid=SklearnRidgeRegressionModelConfig.CONFIG_SPACE,
on_external_dimension=CategoricalDimension(
name="boosting_root_model_name",
values=[SklearnRidgeRegressionModelConfig.__name__])
).join(subgrid=SklearnRandomForestRegressionModelConfig.CONFIG_SPACE,
on_external_dimension=CategoricalDimension(
name="residual_model_name",
values=[SklearnRandomForestRegressionModelConfig.__name__]))
_DEFAULT = Point(
max_basis_function_degree=2,
residual_model_name=SklearnRandomForestRegressionModelConfig.__name__,
boosting_root_model_name=SklearnLassoRegressionModelConfig.__name__,
min_abs_root_model_coef=0.01,
sklearn_lasso_regression_model_config=SklearnLassoRegressionModelConfig
.DEFAULT,
sklearn_ridge_regression_model_config=SklearnRidgeRegressionModelConfig
.DEFAULT,
sklearn_random_forest_regression_model_config=
SklearnRandomForestRegressionModelConfig.DEFAULT,
perform_initial_root_model_hyper_parameter_search=True,
perform_initial_random_forest_hyper_parameter_search=True)
@classmethod
def contains(cls, config):
return config in cls.CONFIG_SPACE
#
import pandas as pd
from mlos.Exceptions import UtilityValueUnavailableException
from mlos.Optimizers.OptimizationProblem import OptimizationProblem
from mlos.Optimizers.ExperimentDesigner.UtilityFunctionOptimizers.UtilityFunctionOptimizer import UtilityFunctionOptimizer
from mlos.Optimizers.ExperimentDesigner.UtilityFunctions.UtilityFunction import UtilityFunction
from mlos.Spaces import SimpleHypergrid, DiscreteDimension, Point
from mlos.Spaces.Configs.ComponentConfigStore import ComponentConfigStore
from mlos.Tracer import trace
random_search_optimizer_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(name="random_search_optimizer_config",
dimensions=[
DiscreteDimension(
name="num_samples_per_iteration",
min=1,
max=100000)
]),
default=Point(num_samples_per_iteration=1000))
class RandomSearchOptimizer(UtilityFunctionOptimizer):
""" Performs a random search over the search space.
This is the simplest optimizer to implement and a good baseline for all other optimizers
to beat.
"""
def __init__(self,
optimizer_config: Point,
optimization_problem: OptimizationProblem,
from mlos.Logger import create_logger
from mlos.Optimizers.ExperimentDesigner.UtilityFunctions.UtilityFunction import UtilityFunction
from mlos.Optimizers.ParetoFrontier import ParetoFrontier
from mlos.Optimizers.RegressionModels.MultiObjectiveRegressionModel import MultiObjectiveRegressionModel
from mlos.Optimizers.RegressionModels.Prediction import Prediction
from mlos.Optimizers.RegressionModels.MultiObjectivePrediction import MultiObjectivePrediction
from mlos.Spaces import SimpleHypergrid, DiscreteDimension, Point
from mlos.Spaces.Configs.ComponentConfigStore import ComponentConfigStore
from mlos.Tracer import trace
multi_objective_probability_of_improvement_utility_function_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name="multi_objective_probability_of_improvement_config",
dimensions=[
DiscreteDimension(name="num_monte_carlo_samples",
min=100,
max=1000)
]),
default=Point(num_monte_carlo_samples=100))
class MultiObjectiveProbabilityOfImprovementUtilityFunction(UtilityFunction):
"""Computes the probability of improvement (POI) of a set of configurations over the existing pareto frontier.
We are up against several requirements here: we need to be able to predict the probability of improvement in a multi-dimensional
objective space. Our assumptions (see below) make each distribution a multi-dimensional blob that's cut in two by a nearly
arbitrarily complex surface of the pareto frontier. This precludes any closed form solution to the POI question.
Thus, we take a Monte Carlo approach: we generate a bunch of points from the predictive distribution, compute the proportion of
these that are dominated by the existing pareto frontier, and use that proportion as an estimator for the probability of
improvement.
optimizer_evaluator_config_store = ComponentConfigStore(
description="Describes the configuration space for the OptimizerEvaluator.\n"
"\n"
"Dimensions:\n"
"- num_iterations: how many optimization iterations to run.\n"
"- evaluation_frequency: how often should the evaluator capture the optima and goodness of fit metrics (e.g. every 10 iterations).\n"
"- include_pickled_optimizer_in_report: should the state of the optimizer be pickled and saved.\n"
"- include_pickled_objective_function_in_report: should the final state of the objective function be pickled and saved.\n"
"- report_regression_model_goodness_of_fit: should the goodness of fit metrics be included in the evaluation report.\n"
"- report_optima_over_time: should the optima over time be included in the evaluation report.\n"
"- include_execution_trace_in_report: should the execution trace produced by mlos.Tracer be included in the evaluation report.",
parameter_space=SimpleHypergrid(
name="optimizer_evaluator",
dimensions=[
DiscreteDimension(name="num_iterations", min=1, max=2**32),
DiscreteDimension(name="evaluation_frequency", min=1, max=2**10),
CategoricalDimension(name="include_pickled_optimizer_in_report", values=[True, False]),
CategoricalDimension(name="include_pickled_objective_function_in_report", values=[True, False]),
CategoricalDimension(name="report_regression_model_goodness_of_fit", values=[True, False]),
CategoricalDimension(name="report_optima_over_time", values=[True, False]),
CategoricalDimension(name="report_pareto_over_time", values=[True, False]),
CategoricalDimension(name="report_pareto_volume_over_time", values=[True, False]),
CategoricalDimension(name="include_execution_trace_in_report", values=[True, False]),
]
),
default=Point(
num_iterations=100,
evaluation_frequency=10,
include_pickled_optimizer_in_report=True,
include_pickled_objective_function_in_report=True,
#
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
#
from mlos.Examples.SmartCache.CacheImplementations.XruCache import XruCache
from mlos.Spaces import DiscreteDimension, Point, SimpleHypergrid
from mlos.Spaces.Configs.ComponentConfigStore import ComponentConfigStore
lru_cache_config_store = ComponentConfigStore(parameter_space=SimpleHypergrid(
name='lru_cache_config',
dimensions=[DiscreteDimension('cache_size', min=1, max=2**12)]),
default=Point(cache_size=100))
class LruCache(XruCache):
""" An implementation of a Least Recently Used cache.
We maintain a dictionary and a linked list both pointing to the same cache entry.
Whenever an entry is touched (and when it is first inserted) it gets moved to the
head of the linked list (in O(1) time). Whenever we try to push a new entry into
a full cache, we expel the entry that's at the tail of the list (since it is the
least recently used one).
"""
def __init__(self, max_size, logger):
XruCache.__init__(self, max_size=max_size, logger=logger)
def evict(self):
removed_node = self._list.remove_at_tail()
evicted_entry = removed_node.cache_entry
del self._dict[removed_node.cache_entry.key]
def setUp(self):
self.cache_param_space = SimpleHypergrid(
name='cache_param_space',
dimensions=[
CategoricalDimension(name='cache_implementation_name', values=['lru_cache', 'associative_cache'])
]
)
self.lru_cache_param_space = SimpleHypergrid(
name='lru_cache_config',
dimensions=[
DiscreteDimension(name='size', min=1, max=2**20),
OrdinalDimension(name='color', ordered_values=['green', 'orange', 'red'])
]
)
self.associative_cache_implementation_root_param_space = SimpleHypergrid(
name='associative_cache_config',
dimensions=[
CategoricalDimension(name='hash_function_name', values=['mod_prime_hash_function', 'lowest_bits']),
CategoricalDimension(name='bucket_implementation', values=['single_value', 'binary_search_tree', 'linked_list'])
]
)
self.mod_prime_hash_function_param_space = SimpleHypergrid(
name='mod_prime_hash_function',
dimensions=[
OrdinalDimension(name='prime', ordered_values=[1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59])
]
)
self.lowest_bits_param_space = SimpleHypergrid(
name='lowest_bits',
dimensions=[
DiscreteDimension(name='num_bits', min=1, max=64)
]
)
self.binary_search_tree_param_space = SimpleHypergrid(
name='binary_search_tree',
dimensions=[
DiscreteDimension(name='max_depth', min=1, max=2**10)
]
)
self.linked_list_param_space = SimpleHypergrid(
name='linked_list',
dimensions=[
DiscreteDimension(name='max_length', min=1, max=2**10)
]
)
self.associative_cache_implementation_param_space = self.associative_cache_implementation_root_param_space.join(
subgrid=self.mod_prime_hash_function_param_space,
on_external_dimension=CategoricalDimension(name='hash_function_name', values=['mod_prime_hash_function'])
).join(
subgrid=self.lowest_bits_param_space,
on_external_dimension=CategoricalDimension(name='hash_function_name', values='lowest_bits')
).join(
subgrid=self.binary_search_tree_param_space,
on_external_dimension=CategoricalDimension(name='bucket_implementation', values=['binary_search_tree'])
)
self.cache_param_space = self.cache_param_space.join(
subgrid=self.lru_cache_param_space,
on_external_dimension=CategoricalDimension(name='cache_implementation_name', values=['lru_cache'])
).join(
subgrid=self.associative_cache_implementation_param_space,
on_external_dimension=CategoricalDimension(name='cache_implementation_name', values=['associative_cache'])
).join(
subgrid=self.linked_list_param_space,
on_external_dimension=CategoricalDimension(name='associative_cache_config.bucket_implementation', values=['linked_list'])
)
Parameter to sklearn LassoCV regressor controlling how model coefficients are selected for update.
From https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LassoCV.html:
If set to ‘random’, a random coefficient is updated every iteration rather than looping
over features sequentially by default. This (setting to ‘random’) often leads to significantly
faster convergence especially when tol is higher than 1e-4.
"""
CYCLIC = 'cyclic'
RANDOM = 'random'
lasso_cross_validated_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name="lasso_regression_model_config",
dimensions=[
ContinuousDimension(name="eps", min=0, max=10.0**-3),
DiscreteDimension(name="num_alphas", min=0, max=200),
CategoricalDimension(name="fit_intercept", values=[False, True]),
CategoricalDimension(name="normalize", values=[False, True]),
CategoricalDimension(name="precompute", values=[False, True]),
DiscreteDimension(name="max_iter", min=100, max=5 * 10**3),
ContinuousDimension(name="tol", min=0, max=1.0),
CategoricalDimension(name="copy_x", values=[False, True]),
DiscreteDimension(name="num_cross_validations", min=2, max=10),
CategoricalDimension(name="verbose", values=[False, True]),
DiscreteDimension(name="num_jobs", min=1, max=2),
CategoricalDimension(name="positive", values=[False, True]),
CategoricalDimension(
name="selection",
values=[selection.value for selection in Selection])
]),
default=Point(
from mlos.Optimizers.RegressionModels.HomogeneousRandomForestConfigStore import homogeneous_random_forest_config_store
from mlos.Optimizers.RegressionModels.HomogeneousRandomForestRegressionModel import HomogeneousRandomForestRegressionModel
bayesian_optimizer_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name="bayesian_optimizer_config",
dimensions=[
CategoricalDimension(
name="surrogate_model_implementation",
values=[
HomogeneousRandomForestRegressionModel.__name__,
]),
CategoricalDimension(name="experiment_designer_implementation",
values=[ExperimentDesigner.__name__]),
DiscreteDimension(
name="min_samples_required_for_guided_design_of_experiments",
min=2,
max=10000)
]).join(
subgrid=homogeneous_random_forest_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
name="surrogate_model_implementation",
values=[
HomogeneousRandomForestRegressionModel.__name__
])).join(
subgrid=experiment_designer_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
name="experiment_designer_implementation",
values=[ExperimentDesigner.__name__])),
default=Point(
surrogate_model_implementation=HomogeneousRandomForestRegressionModel.
__name__,
# Licensed under the MIT License.
#
import math
import pandas as pd
from mlos.OptimizerEvaluationTools.ObjectiveFunctionBase import ObjectiveFunctionBase
from mlos.OptimizerEvaluationTools.SyntheticFunctions.NestedPolynomialObjective import NestedPolynomialObjective, nested_polynomial_objective_config_space
from mlos.Optimizers.OptimizationProblem import Objective, OptimizationProblem
from mlos.Spaces import CategoricalDimension, ContinuousDimension, DiscreteDimension, Hypergrid, Point, SimpleHypergrid
from mlos.Utils.KeyOrderedDict import KeyOrderedDict
multi_objective_nested_polynomial_config_space = SimpleHypergrid(
name="multi_objective_nested_polynomial_config",
dimensions=[
DiscreteDimension(name="num_objectives", min=1, max=10),
CategoricalDimension(name="objective_function_implementation",
values=[NestedPolynomialObjective.__name__])
]).join(subgrid=nested_polynomial_objective_config_space,
on_external_dimension=CategoricalDimension(
name="objective_function_implementation",
values=[NestedPolynomialObjective.__name__]))
class MultiObjectiveNestedPolynomialObjective(ObjectiveFunctionBase):
"""A multi-objective function where each objective is a separate NestedPolynomialObjective.
"""
def __init__(self, objective_function_config: Point):
assert objective_function_config in multi_objective_nested_polynomial_config_space
ObjectiveFunctionBase.__init__(self, objective_function_config)
# Licensed under the MIT License.
#
from mlos.Spaces import SimpleHypergrid, CategoricalDimension, DiscreteDimension, Point
smart_cache_workload_generator_config_space = SimpleHypergrid(
name='smart_cache_workload_generator_config',
dimensions=[
CategoricalDimension('workload_type',
values=[
'fibonacci', 'random_key_from_range',
'cyclical_key_from_range'
])
]).join(
subgrid=SimpleHypergrid(name='fibonacci_config',
dimensions=[
DiscreteDimension('min', min=0, max=2**10),
DiscreteDimension('range_width',
min=0,
max=2**20)
]),
on_external_dimension=CategoricalDimension('workload_type',
values=['fibonacci']),
).join(
subgrid=SimpleHypergrid(name='random_key_from_range_config',
dimensions=[
DiscreteDimension('min', min=0, max=2**10),
DiscreteDimension('range_width',
min=0,
max=2**20)
]),
on_external_dimension=CategoricalDimension(
class PolynomialObjective:
""" A class to enable evaluation of optimizer convergence characteristics.
An instance allows one to evaluate an arbitrarily high degree (<=16) polynomial objective
in up to 16 dimensions in which some number of coefficients have been eliminated (set to zero).
"""
CONFIG_SPACE = SimpleHypergrid(
name="polynomial_objective_config",
dimensions=[
DiscreteDimension(name='seed', min=1, max=2**32),
DiscreteDimension(name='input_domain_dimension', min=1, max=5),
DiscreteDimension(name='max_degree', min=1, max=5),
CategoricalDimension(name='include_mixed_coefficients',
values=[False, True]),
ContinuousDimension(name='percent_coefficients_zeroed',
min=0.0,
max=1.0),
ContinuousDimension(name='coefficient_domain_min',
min=-2**32,
max=2**32),
ContinuousDimension(name='coefficient_domain_width',
min=1,
max=2**32),
CategoricalDimension(name='include_noise', values=[False, True]),
ContinuousDimension(name='noise_coefficient_of_variation',
min=0.0,
max=1.0)
])
# needs constraint coefficient_domain_min < coefficient_domain_max
_DEFAULT = Point(seed=17,
input_domain_dimension=2,
max_degree=2,
include_mixed_coefficients=True,
percent_coefficients_zeroed=0.0,
coefficient_domain_min=-10.0,
coefficient_domain_width=9.0,
include_noise=False,
noise_coefficient_of_variation=0.0)
"""
Initialization parameters:
:param coefficients If specified, will override random generation of a polynomial even if `seed` arg is specified
"""
def __init__(
self,
seed: int = 17,
input_domain_dimension: int = _DEFAULT.input_domain_dimension,
max_degree: int = _DEFAULT.max_degree,
include_mixed_coefficients: bool = _DEFAULT.
include_mixed_coefficients,
percent_coefficients_zeroed: float = _DEFAULT.
percent_coefficients_zeroed,
coefficient_domain_min: float = _DEFAULT.coefficient_domain_min,
coefficient_domain_width: float = _DEFAULT.
coefficient_domain_width,
include_noise: bool = _DEFAULT.include_noise,
noise_coefficient_of_variation: float = _DEFAULT.
noise_coefficient_of_variation,
coefficients=None,
logger=None):
if logger is None:
logger = create_logger("PolynomialObjective")
self.logger = logger
self.seed = seed
self.input_domain_dimension = input_domain_dimension
self.max_degree = max_degree
self.include_mixed_coefficients = include_mixed_coefficients
self.percent_coefficients_zeroed = percent_coefficients_zeroed
self.coefficient_domain_min = coefficient_domain_min
self.coefficient_domain_max = coefficient_domain_min + coefficient_domain_width
self.coefficients = coefficients
self.include_noise = include_noise
self.noise_coefficient_of_variation = noise_coefficient_of_variation
self.coef_ = []
# confirm min < max constraint
assert coefficient_domain_min < self.coefficient_domain_max, 'Minimum coefficient range must be less than maximum'
self.polynomial_features_ = PolynomialFeatures(degree=self.max_degree)
discarded_x = np.array([1] * self.input_domain_dimension).reshape(
1, -1)
poly_terms_x = self.polynomial_features_.fit_transform(discarded_x)
self.num_expected_coefficients_ = len(poly_terms_x[0])
if coefficients is None:
# generate random polynomial if coefficients not specified
np.random.seed(self.seed)
self.coef_ = [
r for r in np.random.uniform(self.coefficient_domain_min,
self.coefficient_domain_max,
self.num_expected_coefficients_)
] # temporarily a list to be convert to np.array
if self.percent_coefficients_zeroed > 0.0:
# reset a random subset of coefficients to 0
num_coef_to_zero = int(self.percent_coefficients_zeroed *
self.num_expected_coefficients_)
true_poly_term_indices_without_effects = np.random.choice(
range(self.num_expected_coefficients_),
size=num_coef_to_zero,
replace=False)
for zi in true_poly_term_indices_without_effects:
self.coef_[zi] = 0.0
# eliminate mixed variable terms if requested
if not self.include_mixed_coefficients:
# zero term coef where input's power != max_degree
for ip, p in enumerate(self.polynomial_features_.powers_):
max_variable_degree = np.max(p)
if max_variable_degree != self.max_degree:
self.coef_[ip] = 0.0
# convert to np.array to enable matmul evaluations
self.coef_ = np.array(self.coef_)
else:
# test if degree specified is consistent with number of coefficients passed
num_specified_coefficients = len(self.coefficients)
assert num_specified_coefficients == self.num_expected_coefficients_, \
'Failed to find sufficient number of coefficients for specified polynomial degree'
self.coef_ = np.array(self.coefficients)
def evaluate(self, x):
y = np.matmul(self.polynomial_features_.fit_transform(x), self.coef_)
if self.include_noise:
cv = self.noise_coefficient_of_variation
y_cv = np.tile(cv, [len(y)])
y_std = np.abs(y_cv * y)
y = np.random.normal(y, y_std, [len(y)])
return y
