experiment_designer_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name='experiment_designer_config',
dimensions=[
CategoricalDimension(
'utility_function_implementation',
values=[
ConfidenceBoundUtilityFunction.__name__,
MultiObjectiveProbabilityOfImprovementUtilityFunction.
__name__
]),
CategoricalDimension('numeric_optimizer_implementation',
values=[
RandomSearchOptimizer.__name__,
GlowWormSwarmOptimizer.__name__
]),
ContinuousDimension('fraction_random_suggestions', min=0, max=1)
]).join(subgrid=confidence_bound_utility_function_config_store.
parameter_space,
on_external_dimension=CategoricalDimension(
'utility_function_implementation',
values=[ConfidenceBoundUtilityFunction.__name__])).
join(
subgrid=
multi_objective_probability_of_improvement_utility_function_config_store
.parameter_space,
on_external_dimension=CategoricalDimension(
'utility_function_implementation',
values=[
MultiObjectiveProbabilityOfImprovementUtilityFunction.__name__
])).join(
subgrid=random_search_optimizer_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
'numeric_optimizer_implementation',
values=[RandomSearchOptimizer.__name__])).
join(subgrid=glow_worm_swarm_optimizer_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
'numeric_optimizer_implementation',
values=[GlowWormSwarmOptimizer.__name__])),
default=Point(
utility_function_implementation=ConfidenceBoundUtilityFunction.
__name__,
numeric_optimizer_implementation=RandomSearchOptimizer.__name__,
confidence_bound_utility_function_config=
confidence_bound_utility_function_config_store.default,
random_search_optimizer_config=random_search_optimizer_config_store.
default,
fraction_random_suggestions=0.5))
bayesian_optimizer_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name="bayesian_optimizer_config",
dimensions=[
CategoricalDimension(
name="surrogate_model_implementation",
values=[
HomogeneousRandomForestRegressionModel.__name__,
MultiObjectiveHomogeneousRandomForest.__name__,
MultiObjectiveLassoCrossValidated.__name__,
MultiObjectiveRegressionEnhancedRandomForest.__name__
]),
CategoricalDimension(name="experiment_designer_implementation",
values=[ExperimentDesigner.__name__]),
DiscreteDimension(
name="min_samples_required_for_guided_design_of_experiments",
min=2,
max=100)
]).join(
subgrid=homogeneous_random_forest_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
name="surrogate_model_implementation",
values=[
HomogeneousRandomForestRegressionModel.__name__,
MultiObjectiveHomogeneousRandomForest.__name__
])).join(
subgrid=lasso_cross_validated_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
name="surrogate_model_implementation",
values=[MultiObjectiveLassoCrossValidated.__name__])).
join(
subgrid=regression_enhanced_random_forest_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
name="surrogate_model_implementation",
values=[
MultiObjectiveRegressionEnhancedRandomForest.__name__
])).join(subgrid=experiment_designer_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
name="experiment_designer_implementation",
values=[ExperimentDesigner.__name__])),
default=Point(
surrogate_model_implementation=HomogeneousRandomForestRegressionModel.
__name__,
experiment_designer_implementation=ExperimentDesigner.__name__,
min_samples_required_for_guided_design_of_experiments=10,
homogeneous_random_forest_regression_model_config=
homogeneous_random_forest_config_store.default,
experiment_designer_config=experiment_designer_config_store.default),
description="TODO")
bayesian_optimizer_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name="bayesian_optimizer_config",
dimensions=[
CategoricalDimension(
name="surrogate_model_implementation",
values=[
HomogeneousRandomForestRegressionModel.__name__,
]),
CategoricalDimension(name="experiment_designer_implementation",
values=[ExperimentDesigner.__name__]),
DiscreteDimension(
name="min_samples_required_for_guided_design_of_experiments",
min=2,
max=10000)
]).join(
subgrid=homogeneous_random_forest_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
name="surrogate_model_implementation",
values=[
HomogeneousRandomForestRegressionModel.__name__
])).join(
subgrid=experiment_designer_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
name="experiment_designer_implementation",
values=[ExperimentDesigner.__name__])),
default=Point(
surrogate_model_implementation=HomogeneousRandomForestRegressionModel.
__name__,
experiment_designer_implementation=ExperimentDesigner.__name__,
min_samples_required_for_guided_design_of_experiments=10,
homogeneous_random_forest_regression_model_config=
homogeneous_random_forest_config_store.default,
experiment_designer_config=experiment_designer_config_store.default),
description="TODO")
lasso_cross_validated_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name="lasso_regression_model_config",
dimensions=[
ContinuousDimension(name="eps", min=0, max=10.0**-3),
DiscreteDimension(name="num_alphas", min=0, max=200),
CategoricalDimension(name="fit_intercept", values=[False, True]),
CategoricalDimension(name="normalize", values=[False, True]),
CategoricalDimension(name="precompute", values=[False, True]),
DiscreteDimension(name="max_iter", min=100, max=5 * 10**3),
ContinuousDimension(name="tol", min=0, max=1.0),
CategoricalDimension(name="copy_x", values=[False, True]),
DiscreteDimension(name="num_cross_validations", min=2, max=10),
CategoricalDimension(name="verbose", values=[False, True]),
DiscreteDimension(name="num_jobs", min=1, max=2),
CategoricalDimension(name="positive", values=[False, True]),
CategoricalDimension(
name="selection",
values=[selection.value for selection in Selection])
]),
default=Point(
eps=10**-6,
num_alphas=100,
fit_intercept=False,
normalize=False,
# sklearn model expects precompute type str, bool, array-like, so setting to sklearn's default and excluding their list option
precompute=False,
max_iter=2000,
tol=10**-4,
copy_x=True,
num_cross_validations=5,
verbose=False,
num_jobs=1,
positive=False,
selection=Selection.CYCLIC.value),
description="Wrapper for sklearn.linear_model.Lasso model."
"This wrapper includes optional CV grid search to tune Lasso hyper parameters within each fit."
)
homogeneous_random_forest_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name="homogeneous_random_forest_regression_model_config",
dimensions=[
DiscreteDimension(name="n_estimators", min=1, max=10000),
ContinuousDimension(name="features_fraction_per_estimator",
min=0,
max=1,
include_min=False,
include_max=True),
ContinuousDimension(name="samples_fraction_per_estimator",
min=0,
max=1,
include_min=False,
include_max=True),
CategoricalDimension(name="regressor_implementation",
values=[DecisionTreeRegressionModel.__name__
]),
CategoricalDimension(name="bootstrap", values=[True, False])
]).join(subgrid=decision_tree_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
name="regressor_implementation",
values=[DecisionTreeRegressionModel.__name__])),
default=Point(
n_estimators=10,
features_fraction_per_estimator=1,
samples_fraction_per_estimator=1,
regressor_implementation=DecisionTreeRegressionModel.__name__,
decision_tree_regression_model_config=decision_tree_config_store.
default,
bootstrap=True),
description="TODO")
regression_enhanced_random_forest_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name="regression_enhanced_random_forest_regression_model_config",
dimensions=[
DiscreteDimension(name="max_basis_function_degree", min=1, max=10),
CategoricalDimension(
name="residual_model_name",
values=[SklearnRandomForestRegressionModelConfig.__name__]),
CategoricalDimension(
name="boosting_root_model_name",
values=[LassoCrossValidatedRegressionModel.__name__]),
CategoricalDimension(
name="perform_initial_random_forest_hyper_parameter_search",
values=[True, False])
]).join(subgrid=lasso_cross_validated_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
name="boosting_root_model_name",
values=[LassoCrossValidatedRegressionModel.__name__])).
join(subgrid=SklearnRandomForestRegressionModelConfig.CONFIG_SPACE,
on_external_dimension=CategoricalDimension(
name="residual_model_name",
values=[SklearnRandomForestRegressionModelConfig.__name__])),
default=Point(
max_basis_function_degree=2,
residual_model_name=SklearnRandomForestRegressionModelConfig.__name__,
boosting_root_model_name=LassoCrossValidatedRegressionModel.__name__,
lasso_regression_model_config=lasso_cross_validated_config_store.
default,
sklearn_random_forest_regression_model_config=
SklearnRandomForestRegressionModelConfig.DEFAULT,
perform_initial_random_forest_hyper_parameter_search=False),
description="Regression-enhanced random forest model hyper-parameters. "
"Model inspired by : https://arxiv.org/pdf/1904.10416.pdf")
#
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
#
from mlos.Examples.SmartCache.CacheImplementations.XruCache import XruCache
from mlos.Spaces import DiscreteDimension, Point, SimpleHypergrid
from mlos.Spaces.Configs.ComponentConfigStore import ComponentConfigStore
lru_cache_config_store = ComponentConfigStore(parameter_space=SimpleHypergrid(
name='lru_cache_config',
dimensions=[DiscreteDimension('cache_size', min=1, max=2**12)]),
default=Point(cache_size=100))
class LruCache(XruCache):
""" An implementation of a Least Recently Used cache.
We maintain a dictionary and a linked list both pointing to the same cache entry.
Whenever an entry is touched (and when it is first inserted) it gets moved to the
head of the linked list (in O(1) time). Whenever we try to push a new entry into
a full cache, we expel the entry that's at the tail of the list (since it is the
least recently used one).
"""
def __init__(self, max_size, logger):
XruCache.__init__(self, max_size=max_size, logger=logger)
def evict(self):
removed_node = self._list.remove_at_tail()
evicted_entry = removed_node.cache_entry
del self._dict[removed_node.cache_entry.key]
#
import pandas as pd
from mlos.Exceptions import UtilityValueUnavailableException
from mlos.Optimizers.OptimizationProblem import OptimizationProblem
from mlos.Optimizers.ExperimentDesigner.UtilityFunctionOptimizers.UtilityFunctionOptimizer import UtilityFunctionOptimizer
from mlos.Optimizers.ExperimentDesigner.UtilityFunctions.UtilityFunction import UtilityFunction
from mlos.Spaces import SimpleHypergrid, DiscreteDimension, Point
from mlos.Spaces.Configs.ComponentConfigStore import ComponentConfigStore
from mlos.Tracer import trace
random_search_optimizer_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(name="random_search_optimizer_config",
dimensions=[
DiscreteDimension(
name="num_samples_per_iteration",
min=1,
max=100000)
]),
default=Point(num_samples_per_iteration=1000))
class RandomSearchOptimizer(UtilityFunctionOptimizer):
""" Performs a random search over the search space.
This is the simplest optimizer to implement and a good baseline for all other optimizers
to beat.
"""
def __init__(self,
optimizer_config: Point,
from mlos.Logger import create_logger
from mlos.Optimizers.ExperimentDesigner.UtilityFunctions.UtilityFunction import UtilityFunction
from mlos.Optimizers.RegressionModels.MultiObjectiveRegressionModel import MultiObjectiveRegressionModel
from mlos.Optimizers.RegressionModels.Prediction import Prediction
from mlos.Spaces import SimpleHypergrid, ContinuousDimension, CategoricalDimension, Point
from mlos.Spaces.Configs.ComponentConfigStore import ComponentConfigStore
from mlos.Tracer import trace
confidence_bound_utility_function_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name="confidence_bound_utility_function_config",
dimensions=[
CategoricalDimension(name="utility_function_name", values=["lower_confidence_bound_on_improvement", "upper_confidence_bound_on_improvement"]),
ContinuousDimension(name="alpha", min=0.01, max=0.5)
]
),
default=Point(
utility_function_name="upper_confidence_bound_on_improvement",
alpha=0.01
)
)
class ConfidenceBoundUtilityFunction(UtilityFunction):
def __init__(self, function_config: Point, surrogate_model: MultiObjectiveRegressionModel, minimize: bool, logger=None):
if logger is None:
logger = create_logger(self.__class__.__name__)
self.logger = logger
self.config = function_config
from .UtilityFunctions.ConfidenceBoundUtilityFunction import ConfidenceBoundUtilityFunction, confidence_bound_utility_function_config_store
experiment_designer_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name='experiment_designer_config',
dimensions=[
CategoricalDimension('utility_function_implementation', values=[ConfidenceBoundUtilityFunction.__name__]),
CategoricalDimension('numeric_optimizer_implementation', values=[RandomSearchOptimizer.__name__, GlowWormSwarmOptimizer.__name__]),
ContinuousDimension('fraction_random_suggestions', min=0, max=1)
]
).join(
subgrid=confidence_bound_utility_function_config_store.parameter_space,
on_external_dimension=CategoricalDimension('utility_function_implementation', values=[ConfidenceBoundUtilityFunction.__name__])
).join(
subgrid=random_search_optimizer_config_store.parameter_space,
on_external_dimension=CategoricalDimension('numeric_optimizer_implementation', values=[RandomSearchOptimizer.__name__])
).join(
subgrid=glow_worm_swarm_optimizer_config_store.parameter_space,
on_external_dimension=CategoricalDimension('numeric_optimizer_implementation', values=[GlowWormSwarmOptimizer.__name__])
),
default=Point(
utility_function_implementation=ConfidenceBoundUtilityFunction.__name__,
numeric_optimizer_implementation=RandomSearchOptimizer.__name__,
confidence_bound_utility_function_config=confidence_bound_utility_function_config_store.default,
random_search_optimizer_config=random_search_optimizer_config_store.default,
fraction_random_suggestions=0.5
)
)
class ExperimentDesigner:
""" Portion of a BayesianOptimizer concerned with Design of Experiments.
import pandas as pd
from mlos.Logger import create_logger
from mlos.Optimizers.ExperimentDesigner.UtilityFunctions.UtilityFunction import UtilityFunction
from mlos.Optimizers.ParetoFrontier import ParetoFrontier
from mlos.Optimizers.RegressionModels.MultiObjectiveRegressionModel import MultiObjectiveRegressionModel
from mlos.Optimizers.RegressionModels.Prediction import Prediction
from mlos.Optimizers.RegressionModels.MultiObjectivePrediction import MultiObjectivePrediction
from mlos.Spaces import SimpleHypergrid, DiscreteDimension, Point
from mlos.Spaces.Configs.ComponentConfigStore import ComponentConfigStore
from mlos.Tracer import trace
multi_objective_probability_of_improvement_utility_function_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name="multi_objective_probability_of_improvement_config",
dimensions=[
DiscreteDimension(name="num_monte_carlo_samples",
min=100,
max=1000)
]),
default=Point(num_monte_carlo_samples=100))
class MultiObjectiveProbabilityOfImprovementUtilityFunction(UtilityFunction):
"""Computes the probability of improvement (POI) of a set of configurations over the existing pareto frontier.
We are up against several requirements here: we need to be able to predict the probability of improvement in a multi-dimensional
objective space. Our assumptions (see below) make each distribution a multi-dimensional blob that's cut in two by a nearly
arbitrarily complex surface of the pareto frontier. This precludes any closed form solution to the POI question.
Thus, we take a Monte Carlo approach: we generate a bunch of points from the predictive distribution, compute the proportion of
these that are dominated by the existing pareto frontier, and use that proportion as an estimator for the probability of
glow_worm_swarm_optimizer_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name="glow_worm_swarm_optimizer_config",
dimensions=[
DiscreteDimension(name="num_initial_points_multiplier",
min=1,
max=10),
DiscreteDimension(name="num_worms", min=10, max=1000),
DiscreteDimension(
name="num_iterations", min=1,
max=20), # TODO: consider other stopping criteria too
ContinuousDimension(name="luciferin_decay_constant", min=0, max=1),
ContinuousDimension(name="luciferin_enhancement_constant",
min=0,
max=1),
ContinuousDimension(name="step_size", min=0,
max=1), # TODO: make this adaptive
ContinuousDimension(name="initial_decision_radius",
min=0,
max=1,
include_min=False),
ContinuousDimension(name="max_sensory_radius", min=0.5,
max=10), # TODO: add constraints
DiscreteDimension(
name="desired_num_neighbors", min=1, max=100
), # TODO: add constraint to make it smaller than num_worms
ContinuousDimension(name="decision_radius_adjustment_constant",
min=0,
max=1)
]),
default=Point(num_initial_points_multiplier=5,
num_worms=100,
num_iterations=10,
luciferin_decay_constant=0.2,
luciferin_enhancement_constant=0.2,
step_size=0.01,
initial_decision_radius=0.2,
max_sensory_radius=2,
desired_num_neighbors=10,
decision_radius_adjustment_constant=0.05))
decision_tree_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name="decision_tree_regression_model_config",
dimensions=[
CategoricalDimension(name="criterion", values=[criterion.value for criterion in Criterion]),
CategoricalDimension(name="splitter", values=[splitter.value for splitter in Splitter]),
DiscreteDimension(name="max_depth", min=0, max=2**10),
DiscreteDimension(name="min_samples_split", min=2, max=2**10),
DiscreteDimension(name="min_samples_leaf", min=3, max=2**10),
ContinuousDimension(name="min_weight_fraction_leaf", min=0.0, max=0.5),
CategoricalDimension(name="max_features", values=[function.value for function in MaxFeaturesFunc]),
DiscreteDimension(name="max_leaf_nodes", min=0, max=2**10),
ContinuousDimension(name="min_impurity_decrease", min=0.0, max=2**10),
ContinuousDimension(name="ccp_alpha", min=0.0, max=2**10),
DiscreteDimension(name="min_samples_to_fit", min=1, max=2 ** 32),
DiscreteDimension(name="n_new_samples_before_refit", min=1, max=2**32)
]
),
default=Point(
criterion=Criterion.MSE.value,
splitter=Splitter.BEST.value,
max_depth=0,
min_samples_split=2,
min_samples_leaf=3,
min_weight_fraction_leaf=0.0,
max_features=MaxFeaturesFunc.AUTO.value,
max_leaf_nodes=0,
min_impurity_decrease=0.0,
ccp_alpha=0.0,
min_samples_to_fit=10,
n_new_samples_before_refit=10
),
description="Governs the construction of an instance of a decision tree regressor. Most of the parameters are passed directly"
"to the DecisionTreeRegressor constructor. Two exceptions: "
"min_samples_to_fit determines the minimum number of samples required for the tree to be fitted."
"n_new_samples_before_refit determines the number of new samples before a tree will be refitted."
"Copied from scikit-learn docs:"
"criterion: The function to measure the quality of a split."
"splitter: The strategy used to choose the split at each node."
"max_depth: The maximum depth of the tree. If None, then nodes are expanded until all leaves are pure or until all leaves contain less than"
" min_samples_split samples."
"min_samples_split: The minimum number of samples required to split an internal node."
"min_samples_leaf: The minimum number of samples required to be at a leaf node."
"min_weight_fraction_leaf: The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node."
" Samples have equal weight when sample_weight is not provided."
"max_features: The number of features to consider when looking for the best split."
"random_state: If int, random_state is the seed used by the random number generator; If RandomState instance, random_state is the random number"
" generator; If None, the random number generator is the RandomState instance used by np.random."
"max_leaf_nodes: Grow a tree with max_leaf_nodes in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then"
" unlimited number of leaf nodes."
"min_impurity_decrease: A node will be split if this split induces a decrease of the impurity greater than or equal to this value."
"ccp_alpha: complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than"
" ccp_alpha will be chosen. By default, no pruning is performed. See Minimal Cost-Complexity Pruning for details."
"min_samples_to_fit: minimum number of samples before it makes sense to try to fit this tree"
"n_new_samples_before_refit: It makes little sense to refit every model for every sample. This parameter controls"
" how frequently we refit the decision tree."
)
from mlos.Examples.SmartCache.MlosInterface import PushRuntimeDecisionContext, ReconfigurationRuntimeDecisionContext
from mlos.Examples.SmartCache.MlosInterface.MlosTelemetryMessages import SmartCacheGet, SmartCachePush, SmartCacheEvict
from mlos.Mlos.Infrastructure.ConfigurationManager import Configuration
from mlos.Mlos.SDK import MlosObject, MlosSmartComponentRuntimeAttributes
from mlos.Spaces import CategoricalDimension, Point, SimpleHypergrid
from mlos.Spaces.Configs.ComponentConfigStore import ComponentConfigStore
smart_cache_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name='smart_cache_config',
dimensions=[
CategoricalDimension(name='implementation', values=['LRU', 'MRU'])
]).join(subgrid=lru_cache_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
name='implementation', values=['LRU'])).join(
subgrid=mru_cache_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
name='implementation', values=['MRU'])),
default=Point(implementation='LRU',
lru_cache_config=lru_cache_config_store.default))
class SmartCache:
""" A tunable and observable cache that takes advantage of Mlos.
The goal here is to provide a bunch of cache implementations that are parameterizable.
Parameters
----------
multi_objective_pass_through_model_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name="multi_objective_pass_through_model_config",
dimensions=[
CategoricalDimension(name="uncertainty_type", values=["constant", "coefficient_of_variation"]),
CategoricalDimension(name="use_objective_function", values=[True]),
DiscreteDimension(name="predicted_value_degrees_of_freedom", min=3, max=10000)
]
).join(
on_external_dimension=CategoricalDimension(name="uncertainty_type", values=["constant"]),
subgrid=SimpleHypergrid(
name="constant_uncertainty_config",
dimensions=[ContinuousDimension(name="value", min=0, max=2 ** 10)]
)
).join(
on_external_dimension=CategoricalDimension(name="uncertainty_type", values=["coefficient_of_variation"]),
subgrid=SimpleHypergrid(
name="coefficient_of_variation_config",
dimensions=[ContinuousDimension(name="value", min=0, max=1)]
)
).join(
on_external_dimension=CategoricalDimension(name="use_objective_function", values=[True]),
subgrid=objective_function_config_store.parameter_space
),
default=Point(
uncertainty_type="constant",
use_objective_function=True,
predicted_value_degrees_of_freedom=10,
constant_uncertainty_config=Point(value=1),
objective_function_config=objective_function_config_store.get_config_by_name("three_level_quadratic")
),
description=""
)
random_near_incumbent_optimizer_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name="random_near_incumbent_optimizer_config",
dimensions=[
DiscreteDimension(name="num_starting_configs", min=1, max=1000),
ContinuousDimension(name="initial_velocity", min=0.01, max=1),
ContinuousDimension(name="velocity_update_constant", min=0, max=1),
ContinuousDimension(name="velocity_convergence_threshold",
min=0,
max=1),
DiscreteDimension(name="max_num_iterations", min=1, max=1000),
DiscreteDimension(name="num_neighbors", min=1, max=1000),
DiscreteDimension(name="num_cached_good_params", min=0, max=2**16),
ContinuousDimension(name="initial_points_pareto_weight",
min=0,
max=1),
ContinuousDimension(
name="initial_points_cached_good_params_weight", min=0, max=1),
ContinuousDimension(name="initial_points_random_params_weight",
min=0,
max=1),
]),
default=Point(num_starting_configs=10,
initial_velocity=0.3,
velocity_update_constant=0.5,
velocity_convergence_threshold=0.01,
max_num_iterations=50,
num_neighbors=20,
num_cached_good_params=2**10,
initial_points_pareto_weight=0.5,
initial_points_cached_good_params_weight=0.3,
initial_points_random_params_weight=0.2),
description="""
* num_starting_configs - how many points to start the search from?
* initial_velocity - how far from the incumbent should the random neighbors be generated?
* velocity_update_constant - how quickly to change the velocity (0 - don't change it at all, 1 - change it as fast as possible)?
* velocity_convergence_threshold - when an incumbent's velocity drops below this threshold, it is assumed to have converged.
* max_num_iterations - cap on the number of iterations. A failsafe - should be higher than what the algorithm needs to converge on average.
* num_neighbors - how many random neighbors to generate for each incumbent?
* num_cached_good_params - how many good configurations should this optimizer cache for future use?
* initial_points_pareto_weight - what proportion of initial points should come from the pareto frontier?
* initial_points_cached_good_params_weight - what proportion of initial points should come from the good params cache?
* initial_points_random_params_weight - what proportion of initial points should be randomly generated?
""")
