def test_filtering_out_invalid_rows(self):
spaces = [
bayesian_optimizer_config_store.parameter_space,
glow_worm_swarm_optimizer_config_store.parameter_space
]
# Just to make sure we are testing both hierarchical and flat code paths.
#
assert any(space.is_hierarchical() for space in spaces)
assert any(not space.is_hierarchical() for space in spaces)
num_samples = 1000
for space in spaces:
random_dataframe_with_invalid_rows = space.random_dataframe(
num_samples=num_samples)
for dimension in space.dimensions:
if isinstance(dimension,
(ContinuousDimension, DiscreteDimension)):
# This makes about half of the rows invalid.
#
random_dataframe_with_invalid_rows.loc[:, [dimension.
name]] *= 2
break
with traced(scope_name="slow_filtering"):
# Let's filter out invalid rows the slow way.
#
valid_indices = []
for idx in random_dataframe_with_invalid_rows.index:
row_as_df = random_dataframe_with_invalid_rows.loc[[idx]]
row_as_point = Point.from_dataframe(row_as_df)
if row_as_point in space:
valid_indices.append(idx)
expected_valid_rows_index = pd.Index(valid_indices)
print(
f"{len(expected_valid_rows_index)}/{len(random_dataframe_with_invalid_rows.index)} rows are valid."
)
assert 0 < len(expected_valid_rows_index)
assert len(expected_valid_rows_index) < num_samples
# Let's filter out invalid rows the fast way.
#
actual_valid_rows_index = space.filter_out_invalid_rows(
original_dataframe=random_dataframe_with_invalid_rows,
exclude_extra_columns=True).index
assert expected_valid_rows_index.equals(actual_valid_rows_index)
if not space.is_hierarchical():
# For flat spaces we can choose between the column-wise operators and the row-wise validation. This is to get the tracing data to see the
# perf difference, but also to validate correctness by computing the desired index in yet another way.
#
with traced(scope_name="faster_filtering"):
expected_valid_rows_index_2 = random_dataframe_with_invalid_rows[
random_dataframe_with_invalid_rows.apply(
lambda row: Point(
**{
dim_name: row[i]
for i, dim_name in enumerate(
space.dimension_names)
}) in space,
axis=1)].index
assert expected_valid_rows_index_2.equals(
actual_valid_rows_index)
class PolynomialObjective:
""" A class to enable evaluation of optimizer convergence characteristics.
An instance allows one to evaluate an arbitrarily high degree (<=16) polynomial objective
in up to 16 dimensions in which some number of coefficients have been eliminated (set to zero).
"""
CONFIG_SPACE = SimpleHypergrid(
name="polynomial_objective_config",
dimensions=[
DiscreteDimension(name='seed', min=1, max=2**32),
DiscreteDimension(name='input_domain_dimension', min=1, max=5),
ContinuousDimension(name='input_domain_min', min=-2**20,
max=2**20),
ContinuousDimension(name='input_domain_width', min=1, max=2**21),
DiscreteDimension(name='max_degree', min=1, max=5),
CategoricalDimension(name='include_mixed_coefficients',
values=[False, True]),
ContinuousDimension(name='percent_coefficients_zeroed',
min=0.0,
max=1.0),
ContinuousDimension(name='coefficient_domain_min',
min=-2**32,
max=2**32),
ContinuousDimension(name='coefficient_domain_width',
min=1,
max=2**32),
CategoricalDimension(name='include_noise', values=[False, True]),
ContinuousDimension(name='noise_coefficient_of_variation',
min=0.0,
max=1.0)
])
# needs constraint coefficient_domain_min < coefficient_domain_max
_DEFAULT = Point(seed=17,
input_domain_dimension=2,
input_domain_min=-2**10,
input_domain_width=2**11,
max_degree=2,
include_mixed_coefficients=True,
percent_coefficients_zeroed=0.0,
coefficient_domain_min=-10.0,
coefficient_domain_width=9.0,
include_noise=False,
noise_coefficient_of_variation=0.0)
"""
Initialization parameters:
:param coefficients If specified, will override random generation of a polynomial even if `seed` arg is specified
"""
def __init__(
self,
seed: int = 17,
input_domain_dimension: int = _DEFAULT.input_domain_dimension,
max_degree: int = _DEFAULT.max_degree,
include_mixed_coefficients: bool = _DEFAULT.
include_mixed_coefficients,
percent_coefficients_zeroed: float = _DEFAULT.
percent_coefficients_zeroed,
coefficient_domain_min: float = _DEFAULT.coefficient_domain_min,
coefficient_domain_width: float = _DEFAULT.
coefficient_domain_width,
include_noise: bool = _DEFAULT.include_noise,
noise_coefficient_of_variation: float = _DEFAULT.
noise_coefficient_of_variation,
coefficients=None,
logger=None):
if logger is None:
logger = create_logger("PolynomialObjective")
self.logger = logger
self.seed = seed
self.input_domain_dimension = input_domain_dimension
self.max_degree = max_degree
self.include_mixed_coefficients = include_mixed_coefficients
self.percent_coefficients_zeroed = percent_coefficients_zeroed
self.coefficient_domain_min = coefficient_domain_min
self.coefficient_domain_max = coefficient_domain_min + coefficient_domain_width
self.coefficients = coefficients
self.include_noise = include_noise
self.noise_coefficient_of_variation = noise_coefficient_of_variation
self.coef_ = []
# confirm min < max constraint
assert coefficient_domain_min < self.coefficient_domain_max, 'Minimum coefficient range must be less than maximum'
self.polynomial_features_ = PolynomialFeatures(degree=self.max_degree)
discarded_x = np.array([1] * self.input_domain_dimension).reshape(
1, -1)
poly_terms_x = self.polynomial_features_.fit_transform(discarded_x)
self.num_expected_coefficients_ = len(poly_terms_x[0])
if coefficients is None:
# generate random polynomial if coefficients not specified
np.random.seed(self.seed)
self.coef_ = [
r for r in np.random.uniform(self.coefficient_domain_min,
self.coefficient_domain_max,
self.num_expected_coefficients_)
] # temporarily a list to be convert to np.array
if self.percent_coefficients_zeroed > 0.0:
# reset a random subset of coefficients to 0
num_coef_to_zero = int(self.percent_coefficients_zeroed *
self.num_expected_coefficients_)
true_poly_term_indices_without_effects = np.random.choice(
range(self.num_expected_coefficients_),
size=num_coef_to_zero,
replace=False)
for zi in true_poly_term_indices_without_effects:
self.coef_[zi] = 0.0
# eliminate mixed variable terms if requested
if not self.include_mixed_coefficients:
# zero term coef where input's power != max_degree
for ip, p in enumerate(self.polynomial_features_.powers_):
max_variable_degree = np.max(p)
if max_variable_degree != self.max_degree:
self.coef_[ip] = 0.0
# convert to np.array to enable matmul evaluations
self.coef_ = np.array(self.coef_)
else:
# test if degree specified is consistent with number of coefficients passed
num_specified_coefficients = len(self.coefficients)
assert num_specified_coefficients == self.num_expected_coefficients_, \
'Failed to find sufficient number of coefficients for specified polynomial degree'
self.coef_ = np.array(self.coefficients)
def evaluate(self, x):
y = np.matmul(self.polynomial_features_.fit_transform(x), self.coef_)
if self.include_noise:
cv = self.noise_coefficient_of_variation
y_cv = np.tile(cv, [len(y)])
y_std = np.abs(y_cv * y)
y = np.random.normal(y, y_std, [len(y)])
return y
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
#
from mlos.Spaces import CategoricalDimension, ContinuousDimension, DiscreteDimension, Point, SimpleHypergrid
from mlos.Spaces.Configs import ComponentConfigStore
hypersphere_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name="hypersphere_config",
dimensions=[
DiscreteDimension(name="num_objectives", min=1, max=100),
CategoricalDimension(name="minimize",
values=["all", "none", "some"]),
ContinuousDimension(name="radius",
min=0,
max=100,
include_min=False)
]),
default=Point(num_objectives=3, minimize="all", radius=10))
for num_objectives in [2, 10]:
for minimize in ["all", "none", "some"]:
hypersphere_config_store.add_config_by_name(
config_name=f"{num_objectives}d_hypersphere_minimize_{minimize}",
config_point=Point(num_objectives=num_objectives,
minimize=minimize,
radius=10),
description=
f"An objective function with {num_objectives + 1} parameters and {num_objectives} objectives to maximize."
)
on_external_dimension=CategoricalDimension(
'numeric_optimizer_implementation',
values=[RandomSearchOptimizer.__name__])).
join(subgrid=glow_worm_swarm_optimizer_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
'numeric_optimizer_implementation',
values=[GlowWormSwarmOptimizer.__name__])).
join(subgrid=random_near_incumbent_optimizer_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
'numeric_optimizer_implementation',
values=[RandomNearIncumbentOptimizer.__name__])),
default=Point(
utility_function_implementation=ConfidenceBoundUtilityFunction.
__name__,
numeric_optimizer_implementation=RandomSearchOptimizer.__name__,
confidence_bound_utility_function_config=
confidence_bound_utility_function_config_store.default,
random_search_optimizer_config=random_search_optimizer_config_store.
default,
fraction_random_suggestions=0.5))
experiment_designer_config_store.add_config_by_name(
config_name="default_random_near_incumbent_config",
config_point=Point(
utility_function_implementation=
MultiObjectiveProbabilityOfImprovementUtilityFunction.__name__,
numeric_optimizer_implementation=RandomNearIncumbentOptimizer.__name__,
multi_objective_probability_of_improvement_config=
multi_objective_probability_of_improvement_utility_function_config_store
.default,
random_near_incumbent_optimizer_config=
def test_optimization_with_context(self):
# Gaussian blob in x with position dependent on context variable y.
def f(parameters, context):
if isinstance(parameters, pd.DataFrame):
index = parameters.index
else:
index = [0]
return pd.DataFrame(
{
'function_value':
-np.exp(-50 * (parameters.x - 0.5 * context.y - 0.5)**2)
},
index=index)
input_space = SimpleHypergrid(
name="input",
dimensions=[ContinuousDimension(name="x", min=0, max=1)])
output_space = SimpleHypergrid(name="objective",
dimensions=[
ContinuousDimension(
name="function_value",
min=-10,
max=10)
])
context_space = SimpleHypergrid(
name="context",
dimensions=[ContinuousDimension(name="y", min=-1, max=1)])
optimization_problem = OptimizationProblem(
parameter_space=input_space,
objective_space=output_space,
# we want to minimize the function
objectives=[Objective(name="function_value", minimize=True)],
context_space=context_space)
# create some data points to eval
n_samples = 100
parameter_df = input_space.random_dataframe(n_samples)
context_df = context_space.random_dataframe(n_samples)
target_df = f(parameter_df, context_df)
local_optimizer = self.bayesian_optimizer_factory.create_local_optimizer(
optimization_problem=optimization_problem, )
with pytest.raises(ValueError, match="Context required"):
local_optimizer.register(
parameter_values_pandas_frame=parameter_df,
target_values_pandas_frame=target_df)
with pytest.raises(
ValueError,
match="Incompatible shape of parameters and context"):
local_optimizer.register(
parameter_values_pandas_frame=parameter_df,
target_values_pandas_frame=target_df,
context_values_pandas_frame=context_df.iloc[:-1])
local_optimizer.register(parameter_values_pandas_frame=parameter_df,
target_values_pandas_frame=target_df,
context_values_pandas_frame=context_df)
with pytest.raises(ValueError, match="Context required"):
local_optimizer.suggest()
with pytest.raises(ValueError, match="Context required"):
local_optimizer.predict(parameter_values_pandas_frame=parameter_df)
suggestion = local_optimizer.suggest(context=context_space.random())
assert isinstance(suggestion, Point)
assert suggestion in input_space
with pytest.raises(
ValueError,
match="Incompatible shape of parameters and context"):
# unaligned parameters and context
local_optimizer.predict(
parameter_values_pandas_frame=parameter_df,
context_values_pandas_frame=context_df.iloc[:-1])
predictions = local_optimizer.predict(
parameter_values_pandas_frame=parameter_df,
context_values_pandas_frame=context_df)
predictions_df = predictions.get_dataframe()
assert len(predictions_df) == len(parameter_df)
remote_optimizer = self.bayesian_optimizer_factory.create_remote_optimizer(
optimization_problem=optimization_problem, )
with pytest.raises(ValueError,
match="not supported if context is provided"):
local_optimizer.optimum(
optimum_definition=OptimumDefinition.BEST_OBSERVATION,
context=Point(y=0).to_dataframe())
with pytest.raises(ValueError,
match="not supported if context is provided"):
local_optimizer.optimum(
optimum_definition=OptimumDefinition.BEST_OBSERVATION)
with pytest.raises(ValueError,
match="requires context to be not None"):
local_optimizer.optimum(optimum_definition=OptimumDefinition.
BEST_SPECULATIVE_WITHIN_CONTEXT)
# can't register, predict, suggest with context on remote optimizer
with pytest.raises(NotImplementedError,
match="Context not currently supported"):
remote_optimizer.register(
parameter_values_pandas_frame=parameter_df,
target_values_pandas_frame=target_df,
context_values_pandas_frame=context_df)
with pytest.raises(NotImplementedError,
match="Context not currently supported"):
remote_optimizer.predict(
parameter_values_pandas_frame=parameter_df,
context_values_pandas_frame=context_df)
with pytest.raises(NotImplementedError,
match="Context not currently supported"):
remote_optimizer.suggest(context=context_df)
# context is missing but required by problem, should give error
with pytest.raises(grpc.RpcError):
remote_optimizer.register(
parameter_values_pandas_frame=parameter_df,
target_values_pandas_frame=target_df)
# run some iterations on local optimizer to see we do something sensible
for _ in range(100):
# pick context at random
context = context_space.random()
suggested_config = local_optimizer.suggest(context=context)
target_values = f(suggested_config, context)
local_optimizer.register(
parameter_values_pandas_frame=suggested_config.to_dataframe(),
target_values_pandas_frame=target_values,
context_values_pandas_frame=context.to_dataframe())
optimum_y_1 = local_optimizer.optimum(
optimum_definition=OptimumDefinition.
BEST_SPECULATIVE_WITHIN_CONTEXT,
context=Point(y=-1).to_dataframe())
optimum_y1 = local_optimizer.optimum(
optimum_definition=OptimumDefinition.
BEST_SPECULATIVE_WITHIN_CONTEXT,
context=Point(y=1).to_dataframe())
assert optimum_y1.x > .6
assert optimum_y_1.x < .4
dimensions=[ContinuousDimension(name="value", min=0, max=2 ** 10)]
)
).join(
on_external_dimension=CategoricalDimension(name="uncertainty_type", values=["coefficient_of_variation"]),
subgrid=SimpleHypergrid(
name="coefficient_of_variation_config",
dimensions=[ContinuousDimension(name="value", min=0, max=1)]
)
).join(
on_external_dimension=CategoricalDimension(name="use_objective_function", values=[True]),
subgrid=objective_function_config_store.parameter_space
),
default=Point(
uncertainty_type="constant",
use_objective_function=True,
predicted_value_degrees_of_freedom=10,
constant_uncertainty_config=Point(value=1),
objective_function_config=objective_function_config_store.get_config_by_name("three_level_quadratic")
),
description=""
)
multi_objective_pass_through_model_config_store.add_config_by_name(
config_name="three_level_quadratic",
config_point=Point(
uncertainty_type="constant",
use_objective_function=True,
predicted_value_degrees_of_freedom=10,
constant_uncertainty_config=Point(value=1),
objective_function_config=objective_function_config_store.get_config_by_name("three_level_quadratic")
)
def _untranslate_point(self, point: Point) -> Point:
unflattened_dict = {
self._backward_name_mapping[dim_name]: value
for dim_name, value in point
}
return Point(**unflattened_dict)
def suggest(self, context_values_dataframe=None): # pylint: disable=unused-argument
""" Returns the next best configuration to try.
The idea is pretty simple:
1. We start with a random population of glowworms, whose luciferin levels are equal to their utility function value.
2. Each glowworm looks around for all other glowworms in its neighborhood and finds ones that are brighter.
3. Each glowworm randomly selects from its brighter neighbors the one to walk towards (with probability proportional to the diff in brightness).
4. Everybody takes a step.
5. Everybody updates step size to have the desired number of neighbors.
5. Update luciferin levels.
"""
assert context_values_dataframe is None or len(context_values_dataframe.index) == 1
self.logger.info(f"Suggesting config for context: {context_values_dataframe}")
# TODO: consider remembering great features from previous invocations of the suggest() method.
parameters_df = self.optimization_problem.parameter_space.random_dataframe(
num_samples=self.optimizer_config.num_worms * self.optimizer_config.num_initial_points_multiplier
)
features_df = self.optimization_problem.construct_feature_dataframe(
parameters_df=parameters_df.copy(deep=False),
context_df=context_values_dataframe,
product=True
)
self.logger.info(f"Computing utility values for the initial {len(features_df.index)} configurations.")
utility_function_values = self.utility_function(feature_values_pandas_frame=features_df.copy(deep=False))
num_utility_function_values = len(utility_function_values.index)
self.logger.info(f"Obtained {num_utility_function_values} utility function values.")
if num_utility_function_values == 0:
error_message = f"Utility function {self.utility_function.__class__.__name__} produced no values."
self.logger.info(error_message)
raise UtilityValueUnavailableException(f"Utility function {self.utility_function.__class__.__name__} produced no values.")
# TODO: keep getting configs until we have enough utility values to get started. Or assign 0 to missing ones,
# and let them climb out of their infeasible holes.
top_utility_values = utility_function_values.nlargest(n=self.optimizer_config.num_worms, columns=['utility'])
self.logger.info(f"Selected {len(top_utility_values.index)} initial glow worm positions.")
self.logger.info("Initializing glow worm parameters.")
# TODO: could it be in place?
params_for_top_utility = self.parameter_adapter.project_dataframe(parameters_df.loc[top_utility_values.index], in_place=False)
worms = pd.concat([params_for_top_utility, top_utility_values], axis=1)
# Let's reset the index to make keeping track down the road easier.
#
worms.index = pd.Index(range(len(worms.index)))
# Initialize luciferin to the value of the utility function
#
worms['decision_radius'] = self.optimizer_config.initial_decision_radius
worms['luciferin'] = worms['utility']
self.logger.info(f"Starting {self.optimizer_config.num_iterations} iterations.")
for i in range(self.optimizer_config.num_iterations):
self.logger.info(f"[{i+1}/{self.optimizer_config.num_iterations}] Updating glow-worm positions.")
worms = self.run_iteration(worms=worms)
# TODO: keep track of the max configs over iterations
self.logger.info(f"[{i+1}/{self.optimizer_config.num_iterations}] Computing utility.")
worms = self.compute_utility(worms, context_values_dataframe)
self.logger.info(f"[{i+1}/{self.optimizer_config.num_iterations}] Updating luciferin levels.")
worms['luciferin'] = (1 - self.optimizer_config.luciferin_decay_constant) * worms['luciferin'] + \
self.optimizer_config.luciferin_enhancement_constant * worms['utility']
# TODO: return the max of all seen configs - not just the configs that the glowworms occupied in this iteration.
idx_of_max = worms['utility'].idxmax()
best_config = worms.loc[[idx_of_max], self.dimension_names]
config_to_suggest = Point.from_dataframe(best_config)
self.logger.info(f"Suggesting: {str(config_to_suggest)}.")
# TODO: we might have to go for second or nth best if the projection won't work out. But then again if we were
# TODO: able to compute the utility function then the projection has worked out once before...
return self.parameter_adapter.unproject_point(config_to_suggest)
name="max_features",
values=[function.value for function in MaxFeaturesFunc]),
DiscreteDimension(name="max_leaf_nodes", min=0, max=2**10),
ContinuousDimension(name="min_impurity_decrease",
min=0.0,
max=2**10),
ContinuousDimension(name="ccp_alpha", min=0.0, max=2**10),
DiscreteDimension(name="min_samples_to_fit", min=1, max=32),
DiscreteDimension(name="n_new_samples_before_refit", min=1, max=32)
]),
default=Point(criterion=Criterion.MSE.value,
splitter=Splitter.BEST.value,
max_depth=0,
min_samples_split=2,
min_samples_leaf=3,
min_weight_fraction_leaf=0.0,
max_features=MaxFeaturesFunc.AUTO.value,
max_leaf_nodes=0,
min_impurity_decrease=0.0,
ccp_alpha=0.0,
min_samples_to_fit=10,
n_new_samples_before_refit=10),
description=
"Governs the construction of an instance of a decision tree regressor. Most of the parameters are passed directly"
"to the DecisionTreeRegressor constructor. Two exceptions: "
"min_samples_to_fit determines the minimum number of samples required for the tree to be fitted."
"n_new_samples_before_refit determines the number of new samples before a tree will be refitted."
"Copied from scikit-learn docs:"
"criterion: The function to measure the quality of a split."
"splitter: The strategy used to choose the split at each node."
"max_depth: The maximum depth of the tree. If None, then nodes are expanded until all leaves are pure or until all leaves contain less than"
" min_samples_split samples."
from mlos.Spaces.Configs.ComponentConfigStore import ComponentConfigStore
from mlos.Tracer import trace
confidence_bound_utility_function_config_store = ComponentConfigStore(
parameter_space=SimpleHypergrid(
name="confidence_bound_utility_function_config",
dimensions=[
CategoricalDimension(name="utility_function_name",
values=[
"lower_confidence_bound_on_improvement",
"upper_confidence_bound_on_improvement"
]),
ContinuousDimension(name="alpha", min=0.01, max=0.5)
]),
default=Point(
utility_function_name="upper_confidence_bound_on_improvement",
alpha=0.01))
class ConfidenceBoundUtilityFunction(UtilityFunction):
def __init__(self,
function_config: Point,
surrogate_model,
minimize,
logger=None):
if logger is None:
logger = create_logger(self.__class__.__name__)
self.logger = logger
self.config = function_config
self.minimize = minimize
DiscreteDimension(name="num_iterations", min=1, max=20), # TODO: consider other stopping criteria too
ContinuousDimension(name="luciferin_decay_constant", min=0, max=1),
ContinuousDimension(name="luciferin_enhancement_constant", min=0, max=1),
ContinuousDimension(name="step_size", min=0, max=1), # TODO: make this adaptive
ContinuousDimension(name="initial_decision_radius", min=0, max=1, include_min=False),
ContinuousDimension(name="max_sensory_radius", min=0.5, max=10), # TODO: add constraints
DiscreteDimension(name="desired_num_neighbors", min=1, max=100), # TODO: add constraint to make it smaller than num_worms
ContinuousDimension(name="decision_radius_adjustment_constant", min=0, max=1)
]
),
default=Point(
num_initial_points_multiplier=5,
num_worms=100,
num_iterations=10,
luciferin_decay_constant=0.2,
luciferin_enhancement_constant=0.2,
step_size=0.01,
initial_decision_radius=0.2,
max_sensory_radius=2,
desired_num_neighbors=10,
decision_radius_adjustment_constant=0.05
)
)
class GlowWormSwarmOptimizer(UtilityFunctionOptimizer):
""" Searches the utility function for maxima using glowworms.
The first part of this has a good description:
https://www.hindawi.com/journals/mpe/2016/5481602/
The main benefits are:
def read_from_disk(target_folder):
"""Mirrors write_to_disk by reading into memory the contents of an OptimizerEvaluationReport from disk."""
optimizer_evaluation_report = OptimizerEvaluationReport()
optimizer_config_file = os.path.join(target_folder,
"optimizer_config.json")
with open(optimizer_config_file, 'r') as in_file:
optimizer_evaluation_report.optimizer_configuration = Point.from_json(
in_file.read())
objective_function_config_file = os.path.join(
target_folder, "objective_function_config.json")
with open(objective_function_config_file, 'r') as in_file:
optimizer_evaluation_report.objective_function_configuration = Point.from_json(
in_file.read())
pickled_optimizers_dir = os.path.join(target_folder,
"pickled_optimizers")
if os.path.exists(pickled_optimizers_dir):
for file_name in os.listdir(pickled_optimizers_dir):
iteration_number, file_extension = file_name.split(".")
assert file_extension == "pickle"
iteration_number = int(iteration_number)
with open(os.path.join(pickled_optimizers_dir, file_name),
'rb') as in_file:
optimizer_evaluation_report.pickled_optimizers_over_time[
iteration_number] = in_file.read()
objective_function_initial_state_file_path = os.path.join(
target_folder, "objective_function_initial_state.pickle")
if os.path.exists(objective_function_initial_state_file_path):
with open(objective_function_initial_state_file_path,
'rb') as in_file:
optimizer_evaluation_report.pickled_objective_function_initial_state = in_file.read(
)
objective_function_final_state_file_path = os.path.join(
target_folder, "objective_function_final_state.pickle")
if os.path.exists(objective_function_final_state_file_path):
with open(objective_function_final_state_file_path,
'rb') as in_file:
optimizer_evaluation_report.pickled_objective_function_final_state = in_file.read(
)
gof_file_path = os.path.join(
target_folder, "regression_model_goodness_of_fit_state.pickle")
if os.path.exists(gof_file_path):
with open(gof_file_path, 'rb') as in_file:
optimizer_evaluation_report.regression_model_fit_state = pickle.load(
in_file)
optima_over_time_file_path = os.path.join(target_folder,
"optima_over_time.pickle")
if os.path.exists(optima_over_time_file_path):
with open(optima_over_time_file_path, 'rb') as in_file:
optimizer_evaluation_report.optima_over_time = pickle.load(
in_file)
pareto_over_time_file_path = os.path.join(target_folder,
"pareto_over_time.pickle")
if os.path.exists(pareto_over_time_file_path):
with open(pareto_over_time_file_path, "rb") as in_file:
optimizer_evaluation_report.pareto_over_time = pickle.load(
in_file)
pareto_volume_over_time_file_path = os.path.join(
target_folder, "pareto_volume_over_time.json")
if os.path.exists(pareto_volume_over_time_file_path):
with open(pareto_volume_over_time_file_path, 'r') as in_file:
optimizer_evaluation_report.pareto_volume_over_time = json.load(
in_file)
execution_info_file_path = os.path.join(target_folder,
"execution_info.json")
if os.path.exists(execution_info_file_path):
with open(execution_info_file_path, 'r') as in_file:
execution_info_dict = json.load(in_file)
optimizer_evaluation_report.success = execution_info_dict[
'success']
optimizer_evaluation_report.num_optimization_iterations = execution_info_dict[
'num_optimization_iterations']
optimizer_evaluation_report.evaluation_frequency = execution_info_dict[
'evaluation_frequency']
optimizer_evaluation_report.exception = execution_info_dict[
'exception']
optimizer_evaluation_report.exception_traceback = execution_info_dict[
'exception_stack_trace']
return optimizer_evaluation_report
def suggest(self, context_values_dataframe: pd.DataFrame = None):
""" Returns the next best configuration to try.
The idea is pretty simple:
1. We start with some configs on the pareto frontier, plus some good points from previous calls to suggest plus some random configs.
2. For each point we generate random neighbors and optionally adjust them using our velocity.
3. We compute utility for all neighbors and select a new incumbent.
4. We update the velocity.
5. We repeat until we run out of iterations or until velocity falls below some threshold.
"""
self.logger.info(
f"Suggesting config for context: {context_values_dataframe}")
assert context_values_dataframe is None or len(
context_values_dataframe.index) == 1
incumbent_params_df = self._prepare_initial_params_df()
incumbent_utility_df = self._compute_utility_for_params(
params_df=incumbent_params_df, context_df=context_values_dataframe)
if len(incumbent_utility_df.index) == 0:
error_message = f"Utility function {self.utility_function.__class__.__name__} produced no values."
self.logger.info(error_message)
raise UtilityValueUnavailableException(error_message)
# Before we can create random neighbors, we need to normalize all parameter values by projecting them into unit hypercube.
#
projected_incumbent_params_df = self.parameter_adapter.project_dataframe(
df=incumbent_params_df, in_place=False)
# Now, let's put together our incumbents_df which contains the projected params, the accompanying utiliy, as well as the velocity
# component along each dimension.
incumbents_df = projected_incumbent_params_df
incumbents_df['utility'] = incumbent_utility_df['utility']
incumbents_df['speed'] = 0
for dimension_name in self.parameter_dimension_names:
incumbents_df[
f'{dimension_name}_velocity'] = self.optimizer_config.initial_velocity
incumbents_df['speed'] += self.optimizer_config.initial_velocity**2
incumbents_df['speed'] = np.sqrt(incumbents_df['speed'])
incumbents_df['active'] = incumbents_df[
'speed'] > self.optimizer_config.velocity_convergence_threshold
# Let's disable all incumbents for which we couldn't compute utility.
#
null_utility_index = incumbents_df[
incumbents_df['utility'].isna()].index
incumbents_df.loc[null_utility_index, 'active'] = False
num_iterations = 0
while num_iterations < self.optimizer_config.max_num_iterations and incumbents_df[
'active'].any():
num_iterations += 1
incumbents_df = self._run_iteration(
incumbents_df,
context_df=context_values_dataframe,
iteration_number=num_iterations)
if incumbents_df['utility'].isna().all():
error_message = "Utility values were not available for the incumbent."
self.logger.info(error_message)
raise UtilityValueUnavailableException(error_message)
if num_iterations == 0:
error_message = f"{self.__class__.__name__} performed 0 iterations."
self.logger.info(error_message)
raise UnableToProduceGuidedSuggestionException(error_message)
if incumbents_df.dtypes['utility'] != float:
self.logger.info(
f"The type of incumbents_df['utility'] is {incumbents_df.dtypes['utility']}. Utility function: {self.utility_function.__class__.__name__}, "
f"incumbents_df length: {len(incumbents_df.index)}")
incumbents_df['utility'] = pd.to_numeric(
arg=incumbents_df['utility'], errors='raise')
self._cache_good_incumbents(incumbents_df)
idx_of_max = incumbents_df['utility'].idxmax()
best_config_df = incumbents_df.loc[[idx_of_max],
self.parameter_dimension_names]
config_to_suggest = Point.from_dataframe(best_config_df)
unprojected_config_to_suggest = self.parameter_adapter.unproject_point(
config_to_suggest)
self.logger.info(
f"After {num_iterations} iterations suggesting: {unprojected_config_to_suggest.to_json(indent=2)}"
)
return unprojected_config_to_suggest
DiscreteDimension(name="num_neighbors", min=1, max=1000),
DiscreteDimension(name="num_cached_good_params", min=0, max=2**16),
ContinuousDimension(name="initial_points_pareto_weight",
min=0,
max=1),
ContinuousDimension(
name="initial_points_cached_good_params_weight", min=0, max=1),
ContinuousDimension(name="initial_points_random_params_weight",
min=0,
max=1),
]),
default=Point(num_starting_configs=10,
initial_velocity=0.3,
velocity_update_constant=0.5,
velocity_convergence_threshold=0.01,
max_num_iterations=50,
num_neighbors=20,
num_cached_good_params=2**10,
initial_points_pareto_weight=0.5,
initial_points_cached_good_params_weight=0.3,
initial_points_random_params_weight=0.2),
description="""
* num_starting_configs - how many points to start the search from?
* initial_velocity - how far from the incumbent should the random neighbors be generated?
* velocity_update_constant - how quickly to change the velocity (0 - don't change it at all, 1 - change it as fast as possible)?
* velocity_convergence_threshold - when an incumbent's velocity drops below this threshold, it is assumed to have converged.
* max_num_iterations - cap on the number of iterations. A failsafe - should be higher than what the algorithm needs to converge on average.
* num_neighbors - how many random neighbors to generate for each incumbent?
* num_cached_good_params - how many good configurations should this optimizer cache for future use?
* initial_points_pareto_weight - what proportion of initial points should come from the pareto frontier?
* initial_points_cached_good_params_weight - what proportion of initial points should come from the good params cache?
* initial_points_random_params_weight - what proportion of initial points should be randomly generated?
]),
on_external_dimension=CategoricalDimension('workload_type',
values=['fibonacci']),
).join(
subgrid=SimpleHypergrid(name='random_key_from_range_config',
dimensions=[
DiscreteDimension('min', min=0, max=2**10),
DiscreteDimension('range_width',
min=0,
max=2**20)
]),
on_external_dimension=CategoricalDimension(
'workload_type', values=['random_key_from_range']),
).join(
subgrid=SimpleHypergrid(name='sequential_key_from_range_config',
dimensions=[
DiscreteDimension('min', min=0, max=2**10),
DiscreteDimension('range_width',
min=0,
max=2**20)
]),
on_external_dimension=CategoricalDimension(
'workload_type', values=['sequential_key_from_range']),
)
smart_cache_workload_generator_default_config = Point(
workload_type='fibonacci',
reconfiguration_interval=10,
fibonacci_config=Point(min=2**10, range_width=2**10))
assert smart_cache_workload_generator_default_config in smart_cache_workload_generator_config_space
name="optimizer_evaluator",
dimensions=[
DiscreteDimension(name="num_iterations", min=1, max=2**32),
DiscreteDimension(name="evaluation_frequency", min=1, max=2**10),
CategoricalDimension(name="include_pickled_optimizer_in_report", values=[True, False]),
CategoricalDimension(name="include_pickled_objective_function_in_report", values=[True, False]),
CategoricalDimension(name="report_regression_model_goodness_of_fit", values=[True, False]),
CategoricalDimension(name="report_optima_over_time", values=[True, False]),
CategoricalDimension(name="include_execution_trace_in_report", values=[True, False]),
]
),
default=Point(
num_iterations=100,
evaluation_frequency=10,
include_pickled_optimizer_in_report=True,
include_pickled_objective_function_in_report=True,
report_regression_model_goodness_of_fit=True,
report_optima_over_time=True,
include_execution_trace_in_report=True,
)
)
# Parallel unit tests config.
#
parallel_unit_tests_config = optimizer_evaluator_config_store.default
parallel_unit_tests_config.num_iterations = 50
parallel_unit_tests_config.evaluation_frequency = 10
optimizer_evaluator_config_store.add_config_by_name(
config_name="parallel_unit_tests_config",
description="This config is to be used in our parallel unit tests.",
class SmartCache:
""" A tunable and observable cache that takes advantage of Mlos.
The goal here is to provide a bunch of cache implementations that are parameterizable.
Parameters
----------
logger : Logger
Logger to use.
Attributes
----------
RuntimeAttributes : MlosSmartComponentRuntimeAttributes
parameter_search_space : SimpleHypergrid
default_config : Point
telemetry_message_types : list
runtime_decision_contexts : list
"""
# Used during registration
RuntimeAttributes = MlosSmartComponentRuntimeAttributes(
smart_component_name="SmartCache", attribute_names=[])
parameter_search_space = SimpleHypergrid(
name='smart_cache_config',
dimensions=[
CategoricalDimension(name='implementation', values=['LRU', 'MRU'])
]).join(subgrid=LruCacheConfig.CONFIG_SPACE,
on_external_dimension=CategoricalDimension(
name='implementation', values=['LRU'])).join(
subgrid=MruCacheConfig.CONFIG_SPACE,
on_external_dimension=CategoricalDimension(
name='implementation', values=['MRU']))
# Used if no intelligence is hooked up
default_config = Point(implementation='LRU',
lru_cache_config=LruCacheConfig.DEFAULT)
# Used to inform the Mlos Global Context about all types of telemetry messages that this component can emit
telemetry_message_types = [
(SmartCachePush, 0b1),
(SmartCacheGet, 0b10),
(SmartCacheEvict, 0b100),
]
# Used to inform the Mlos Global Context about all types of runtime decisions that can be expected
runtime_decision_contexts = [
PushRuntimeDecisionContext,
ReconfigurationRuntimeDecisionContext,
]
def __init__(self, logger):
self.logger = logger
self.mlos_object = MlosObject(
smart_component_type=type(self),
smart_component_runtime_attributes=self.RuntimeAttributes(
component_id=id(self)))
self.current_config = Configuration(component_type=SmartCache,
values=self.default_config,
id=-1)
self.cache_implementation = LruCache(
max_size=self.current_config.values.lru_cache_config.cache_size,
logger=self.logger)
self.mlos_object.register()
self.reconfigure()
def __del__(self):
self.mlos_object.unregister()
def __iter__(self):
return self.cache_implementation.__iter__()
def __len__(self):
return len(self.cache_implementation)
def __contains__(self, item):
return item in self.cache_implementation
def push(self, key, value):
self.reconfigure() # TODO: make this less frequent
if key in self:
return
should_push = self.mlos_object.make_runtime_decision(
PushRuntimeDecisionContext(mlos_object=self.mlos_object,
current_config=self.current_config))
if not should_push:
return
if self.mlos_object.is_message_type_enabled(SmartCachePush):
# Note that we hide this behind an 'is_enabled' check. This is for the cases
# when assembling the message itself can be expensive.
self.mlos_object.send_telemetry_message(SmartCachePush(key=key))
cache_entry = CacheEntry(key, value)
evicted_cache_entry = self.cache_implementation.push(cache_entry)
if evicted_cache_entry is not None:
# Note that here we skip the 'is message type enabled check' since assembling the message is cheap and
# the check can be done by mlos_object
self.mlos_object.send_telemetry_message(
SmartCacheEvict(key=evicted_cache_entry.key))
def get(self, key):
if key not in self.cache_implementation:
self.mlos_object.send_telemetry_message(
SmartCacheGet(key=key, was_hit=False))
return None
self.mlos_object.send_telemetry_message(
SmartCacheGet(key=key, was_hit=True))
return self.cache_implementation.get(key)
def reconfigure(self):
""" Reconfigures the cache according to the configuration present in self.mlos_object
:return:
"""
smart_cache_reconfiguration_decision_runtime_context = ReconfigurationRuntimeDecisionContext(
self.mlos_object)
should_reconfigure = self.mlos_object.make_runtime_decision(
smart_cache_reconfiguration_decision_runtime_context)
if not should_reconfigure or self.current_config == self.mlos_object.config or self.mlos_object.config is None:
return
self.current_config = self.mlos_object.config
self.logger.info(
f"Reconfiguring. New config values: {self.current_config.values.to_json()}"
)
if self.current_config.values.implementation == 'LRU':
self.cache_implementation = LruCache(
max_size=self.current_config.values.lru_cache_config.
cache_size,
logger=self.logger)
elif self.current_config.values.implementation == 'MRU':
self.cache_implementation = MruCache(
max_size=self.current_config.values.mru_cache_config.
cache_size,
logger=self.logger)
else:
raise RuntimeError("Invalid config")
class SklearnRidgeRegressionModelConfig(metaclass=DefaultConfigMeta):
class Solver(Enum):
"""
From https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html:
Solver to use in the computational routines:
* ‘auto’ chooses the solver automatically based on the type of data.
* ‘svd’ uses a Singular Value Decomposition of X to compute the Ridge coefficients. More stable for
singular matrices than ‘cholesky’.
* ‘cholesky’ uses the standard scipy.linalg.solve function to obtain a closed-form solution.
* ‘sparse_cg’ uses the conjugate gradient solver as found in scipy.sparse.linalg.cg.
As an iterative algorithm, this solver is more appropriate than ‘cholesky’ for
large-scale data (possibility to set tol and max_iter).
* ‘lsqr’ uses the dedicated regularized least-squares routine scipy.sparse.linalg.lsqr.
It is the fastest and uses an iterative procedure.
* ‘sag’ uses a Stochastic Average Gradient descent, and ‘saga’ uses its improved,
unbiased version named SAGA. Both methods also use an iterative procedure, and are
often faster than other solvers when both n_samples and n_features are large.
Note that ‘sag’ and ‘saga’ fast convergence is only guaranteed on features with
approximately the same scale. You can preprocess the data with a scaler from sklearn.preprocessing.
All last five solvers support both dense and sparse data. However, only ‘sag’ and ‘sparse_cg’ supports
sparse input when fit_intercept is True.
"""
AUTO = 'auto' # default
SVD = 'svd'
CHOLESKY = 'cholesky'
LSQR = 'lsqr'
SPARSE_CG = 'sparse_cg'
SAG = 'sag'
SAGA = 'saga'
CONFIG_SPACE = SimpleHypergrid(
name="sklearn_ridge_regression_model_config",
dimensions=[
ContinuousDimension(name="alpha", min=0, max=2**16),
CategoricalDimension(name="fit_intercept", values=[False, True]),
CategoricalDimension(name="normalize", values=[False, True]),
CategoricalDimension(name="copy_x", values=[False, True]),
DiscreteDimension(name="max_iter", min=0, max=10**5),
ContinuousDimension(name="tol", min=0, max=2**10),
CategoricalDimension(name="solver",
values=[solver.value for solver in Solver]),
])
_DEFAULT = Point(alpha=1.0,
fit_intercept=False,
normalize=False,
copy_x=True,
max_iter=1000,
tol=10**-4,
solver=Solver.AUTO.value)
@classmethod
def contains(cls, config):
return Point(alpha=config.alpha,
fit_intercept=config.fit_intercept,
normalize=config.normalize,
copy_x=config.copy_x,
max_iter=config.max_iter,
tol=config.tol,
random_state=config.random_state,
solver=config.solver) in cls.CONFIG_SPACE
@classmethod
def create_from_config_point(cls, config_point):
assert cls.contains(config_point)
config_key_value_pairs = {
param_name: value
for param_name, value in config_point
}
return cls(**config_key_value_pairs)
def __init__(self,
alpha=_DEFAULT.alpha,
fit_intercept=_DEFAULT.fit_intercept,
normalize=_DEFAULT.normalize,
copy_x=_DEFAULT.copy_x,
max_iter=_DEFAULT.max_iter,
tol=_DEFAULT.tol,
random_state=None,
solver=_DEFAULT.solver):
"""
Ridge parameters:
:param alpha:Regularization strength; must be a positive float. Defaults to 1.0.
:param fit_intercept: Whether to calculate the intercept for this model.
:param normalize: This parameter is ignored when ``fit_intercept`` is set to False.
If True, the regressors X will be normalized before regression by
subtracting the mean and dividing by the l2-norm.
:param copy_x: If ``True``, X will be copied; else, it may be overwritten.
:param max_iter: The maximum number of iterations
:param tol: The tolerance for the optimization: if the updates are
smaller than ``tol``, the optimization code checks the
dual gap for optimality and continues until it is smaller
than ``tol``.
:param solver: Solver to use in the computational routines:
- 'auto' chooses the solver automatically based on the type of data.
- 'svd' uses a Singular Value Decomposition of X to compute the Ridge
coefficients. More stable for singular matrices than 'cholesky'.
- 'cholesky' uses the standard scipy.linalg.solve function to
obtain a closed-form solution.
- 'sparse_cg' uses the conjugate gradient solver as found in
scipy.sparse.linalg.cg. As an iterative algorithm, this solver is
more appropriate than 'cholesky' for large-scale data
(possibility to set `tol` and `max_iter`).
- 'lsqr' uses the dedicated regularized least-squares routine
scipy.sparse.linalg.lsqr. It is the fastest and uses an iterative
procedure.
- 'sag' uses a Stochastic Average Gradient descent, and 'saga' uses
its improved, unbiased version named SAGA. Both methods also use an
iterative procedure, and are often faster than other solvers when
both n_samples and n_features are large. Note that 'sag' and
'saga' fast convergence is only guaranteed on features with
approximately the same scale. You can preprocess the data with a
scaler from sklearn.preprocessing.
:param random_state: The seed of the pseudo random number generator that selects a random
feature to update. Used when ``selection`` == 'random'.
"""
self.alpha = alpha
self.fit_intercept = fit_intercept
self.normalize = normalize
self.copy_x = copy_x
self.max_iter = max_iter
self.tol = tol
self.random_state = random_state
self.solver = solver
def _translate_point(self, point: Point) -> Point:
return point.flat_copy()
max=100)
]).join(
subgrid=homogeneous_random_forest_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
name="surrogate_model_implementation",
values=[
HomogeneousRandomForestRegressionModel.__name__
])).join(
subgrid=experiment_designer_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
name="experiment_designer_implementation",
values=[ExperimentDesigner.__name__])),
default=Point(
surrogate_model_implementation=HomogeneousRandomForestRegressionModel.
__name__,
experiment_designer_implementation=ExperimentDesigner.__name__,
min_samples_required_for_guided_design_of_experiments=10,
homogeneous_random_forest_regression_model_config=
homogeneous_random_forest_config_store.default,
experiment_designer_config=experiment_designer_config_store.default),
description="TODO")
# Add a config with homogeneous random forest where the decision trees refit for every new observation.
#
optimizer_config = bayesian_optimizer_config_store.default
optimizer_config.homogeneous_random_forest_regression_model_config.decision_tree_regression_model_config.n_new_samples_before_refit = 1
optimizer_config.homogeneous_random_forest_regression_model_config.n_estimators = 50
bayesian_optimizer_config_store.add_config_by_name(
config_name='default_refit_tree_every_time', config_point=optimizer_config)
# Add a default config with glowworm swarm optimizer
#
class RegressionEnhancedRandomForestRegressionModelConfig(RegressionModelConfig):
"""A configuration object for RERF model.
Class responsible for validating its objects are valid hyper parameters for the sklearn classes:
Lasso (https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Lasso.html),
Ridge
(https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html#sklearn.linear_model.Ridge)
and
RandomForest ()
"""
CONFIG_SPACE = SimpleHypergrid(
name="regression_enhanced_random_forest_regression_model_config",
dimensions=[
DiscreteDimension(name="max_basis_function_degree", min=1, max=10),
CategoricalDimension(name="residual_model_name",
values=[SklearnRandomForestRegressionModelConfig.__name__]),
CategoricalDimension(name="boosting_root_model_name",
values=[SklearnLassoRegressionModelConfig.__name__,
SklearnRidgeRegressionModelConfig.__name__]),
ContinuousDimension(name="min_abs_root_model_coef", min=0, max=2 ** 10),
CategoricalDimension(name="perform_initial_root_model_hyper_parameter_search", values=[False, True]),
CategoricalDimension(name="perform_initial_random_forest_hyper_parameter_search", values=[False, True])
]
).join(
subgrid=SklearnLassoRegressionModelConfig.CONFIG_SPACE,
on_external_dimension=CategoricalDimension(name="boosting_root_model_name",
values=[SklearnLassoRegressionModelConfig.__name__])
).join(
subgrid=SklearnRidgeRegressionModelConfig.CONFIG_SPACE,
on_external_dimension=CategoricalDimension(name="boosting_root_model_name",
values=[SklearnRidgeRegressionModelConfig.__name__])
).join(
subgrid=SklearnRandomForestRegressionModelConfig.CONFIG_SPACE,
on_external_dimension=CategoricalDimension(name="residual_model_name",
values=[SklearnRandomForestRegressionModelConfig.__name__])
)
_DEFAULT = Point(
max_basis_function_degree=2,
residual_model_name=SklearnRandomForestRegressionModelConfig.__name__,
boosting_root_model_name=SklearnLassoRegressionModelConfig.__name__,
min_abs_root_model_coef=0.01,
sklearn_lasso_regression_model_config=SklearnLassoRegressionModelConfig.DEFAULT,
sklearn_ridge_regression_model_config=SklearnRidgeRegressionModelConfig.DEFAULT,
sklearn_random_forest_regression_model_config=SklearnRandomForestRegressionModelConfig.DEFAULT,
perform_initial_root_model_hyper_parameter_search=True,
perform_initial_random_forest_hyper_parameter_search=False
)
@classmethod
def contains(cls, config):
# following example set in HomogeneousRandomForestRegressionModelConfig.contains()
return True
@classmethod
def create_from_config_point(cls, config_point):
assert cls.contains(config_point)
config_key_value_pairs = {param_name: value for param_name, value in config_point}
return cls(**config_key_value_pairs)
def __init__(
self,
max_basis_function_degree=_DEFAULT.max_basis_function_degree,
boosting_root_model_name=_DEFAULT.boosting_root_model_name,
min_abs_root_model_coef=_DEFAULT.min_abs_root_model_coef,
boosting_root_model_config: Point()=_DEFAULT.sklearn_lasso_regression_model_config,
random_forest_model_config: Point()=_DEFAULT.sklearn_random_forest_regression_model_config,
residual_model_name=_DEFAULT.residual_model_name,
perform_initial_root_model_hyper_parameter_search=_DEFAULT.perform_initial_root_model_hyper_parameter_search,
perform_initial_random_forest_hyper_parameter_search=_DEFAULT.perform_initial_random_forest_hyper_parameter_search
):
self.max_basis_function_degree = max_basis_function_degree
self.residual_model_name = residual_model_name
self.min_abs_root_model_coef = min_abs_root_model_coef
self.perform_initial_root_model_hyper_parameter_search = perform_initial_root_model_hyper_parameter_search
self.perform_initial_random_forest_hyper_parameter_search = perform_initial_random_forest_hyper_parameter_search
self.boosting_root_model_name = boosting_root_model_name
self.boosting_root_model_config = None
if self.boosting_root_model_name == SklearnLassoRegressionModelConfig.__name__:
self.boosting_root_model_config = SklearnLassoRegressionModelConfig \
.create_from_config_point(boosting_root_model_config)
elif self.boosting_root_model_name == SklearnRidgeRegressionModelConfig.__name__:
self.boosting_root_model_config = SklearnRidgeRegressionModelConfig \
.create_from_config_point(boosting_root_model_config)
else:
print('Unrecognized boosting_root_model_name "{}"'.format(self.boosting_root_model_name))
self.random_forest_model_config = SklearnRandomForestRegressionModelConfig \
.create_from_config_point(random_forest_model_config)
name="homogeneous_random_forest_regression_model_config",
dimensions=[
DiscreteDimension(name="n_estimators", min=1, max=256),
ContinuousDimension(name="features_fraction_per_estimator",
min=0,
max=1,
include_min=False,
include_max=True),
ContinuousDimension(name="samples_fraction_per_estimator",
min=0.2,
max=1,
include_min=False,
include_max=True),
CategoricalDimension(name="regressor_implementation",
values=[DecisionTreeRegressionModel.__name__
]),
CategoricalDimension(name="bootstrap", values=[True, False])
]).join(subgrid=decision_tree_config_store.parameter_space,
on_external_dimension=CategoricalDimension(
name="regressor_implementation",
values=[DecisionTreeRegressionModel.__name__])),
default=Point(
n_estimators=10,
features_fraction_per_estimator=1,
samples_fraction_per_estimator=0.7,
regressor_implementation=DecisionTreeRegressionModel.__name__,
decision_tree_regression_model_config=decision_tree_config_store.
default,
bootstrap=True),
description="TODO")
def test_multi_objective_optimization(self,
objective_function_implementation,
minimize, num_output_dimensions,
num_points):
if objective_function_implementation == Hypersphere:
hypersphere_radius = 10
objective_function_config = Point(
implementation=Hypersphere.__name__,
hypersphere_config=Point(num_objectives=num_output_dimensions,
minimize=minimize,
radius=hypersphere_radius))
else:
objective_function_config = Point(
implementation=MultiObjectiveNestedPolynomialObjective.
__name__,
multi_objective_nested_polynomial_config=Point(
num_objectives=num_output_dimensions,
objective_function_implementation=NestedPolynomialObjective
.__name__,
nested_polynomial_objective_config=Point(
num_nested_polynomials=2,
nested_function_implementation=PolynomialObjective.
__name__,
polynomial_objective_config=Point(
seed=17,
input_domain_dimension=2,
input_domain_min=-2**10,
input_domain_width=2**11,
max_degree=2,
include_mixed_coefficients=True,
percent_coefficients_zeroed=0.0,
coefficient_domain_min=-10.0,
coefficient_domain_width=9.0,
include_noise=False,
noise_coefficient_of_variation=0.0))))
objective_function = ObjectiveFunctionFactory.create_objective_function(
objective_function_config)
optimization_problem = objective_function.default_optimization_problem
if objective_function_implementation == MultiObjectiveNestedPolynomialObjective:
# We need to modify the default optimization problem to respect the "minimize" argument.
#
objectives = []
for i, default_objective in enumerate(
optimization_problem.objectives):
if minimize == "all":
minimize = True
elif minimize == "some":
minimize = ((i % 2) == 0)
else:
minimize = False
new_objective = Objective(name=default_objective.name,
minimize=minimize)
objectives.append(new_objective)
optimization_problem.objectives = objectives
optimizer_config = bayesian_optimizer_config_store.get_config_by_name(
"default_multi_objective_optimizer_config")
self.logger.info(optimizer_config)
optimizer = self.bayesian_optimizer_factory.create_local_optimizer(
optimization_problem=optimization_problem,
optimizer_config=optimizer_config)
assert optimizer.optimizer_config.surrogate_model_implementation == MultiObjectiveHomogeneousRandomForest.__name__
# We can now go through the optimization loop, at each point validating that:
# 1) The suggested point is valid.
# 2) The volume of the pareto frontier is monotonically increasing.
lower_bounds_on_pareto_volume = []
upper_bounds_on_pareto_volume = []
for i in range(num_points):
suggestion = optimizer.suggest()
assert suggestion in optimization_problem.parameter_space
objectives = objective_function.evaluate_point(suggestion)
optimizer.register(
parameter_values_pandas_frame=suggestion.to_dataframe(),
target_values_pandas_frame=objectives.to_dataframe())
if i > 10:
pareto_volume_estimator = optimizer.pareto_frontier.approximate_pareto_volume(
num_samples=1000000)
lower_bound, upper_bound = pareto_volume_estimator.get_two_sided_confidence_interval_on_pareto_volume(
alpha=0.95)
lower_bounds_on_pareto_volume.append(lower_bound)
upper_bounds_on_pareto_volume.append(upper_bound)
pareto_volumes_over_time_df = pd.DataFrame({
'lower_bounds':
lower_bounds_on_pareto_volume,
'upper_bounds':
upper_bounds_on_pareto_volume
})
# If we had precise volume measurements, we would want to ascertain that the volume of the pareto frontier is monotonically increasing.
# However, we only have estimates so we cannot assert that they are monotonic. But we can assert that they are approximately monotonic:
# we can make sure that any dip between consecutive volumes is smaller than some small number. Actually we can make sure that there
# is no drift, by looking over larger windows too.
#
threshold = -0.1
for periods in [1, 10, 20]:
min_pct_increase_in_lower_bound = pareto_volumes_over_time_df[
'lower_bounds'].pct_change(periods=periods).fillna(0).min()
if not (min_pct_increase_in_lower_bound > threshold):
print(pareto_volumes_over_time_df)
assert min_pct_increase_in_lower_bound > threshold
min_pct_increase_in_upper_bound = pareto_volumes_over_time_df[
'upper_bounds'].pct_change(periods=periods).fillna(0).min()
if not (min_pct_increase_in_upper_bound > threshold):
print(pareto_volumes_over_time_df)
assert min_pct_increase_in_upper_bound > threshold
def test_registering_multiple_objectives(self):
input_space = SimpleHypergrid(name='input',
dimensions=[
ContinuousDimension(name="x_1",
min=0,
max=10),
ContinuousDimension(name="x_2",
min=0,
max=10)
])
output_space = SimpleHypergrid(name='output',
dimensions=[
ContinuousDimension(name="y_1",
min=0,
max=10),
ContinuousDimension(name="y_2",
min=0,
max=10)
])
optimization_problem = OptimizationProblem(
parameter_space=input_space,
objective_space=output_space,
objectives=[Objective(name='y_1', minimize=True)])
optimizer = self.bayesian_optimizer_factory.create_local_optimizer(
optimization_problem=optimization_problem)
for _ in range(100):
input = optimizer.suggest()
output = Point(y_1=input.x_1, y_2=input.x_2)
optimizer.register(input.to_dataframe(), output.to_dataframe())
num_predictions = 100
prediction = optimizer.predict(
parameter_values_pandas_frame=optimization_problem.parameter_space.
random_dataframe(num_predictions))
prediction_df = prediction.get_dataframe()
assert len(prediction_df.index) == num_predictions
# Let's test invalid observations.
#
input = input_space.random()
input_df = input.to_dataframe()
# We should only remember the valid dimensions.
#
output_with_extra_dimension = Point(y_1=input.x_1,
y_2=input.x_2,
invalid_dimension=42)
output_with_extra_dimension_df = output_with_extra_dimension.to_dataframe(
)
optimizer.register(input_df, output_with_extra_dimension_df)
# Let's make sure that the invalid_dimension was not remembered.
#
all_inputs_df, all_outputs_df, _ = optimizer.get_all_observations()
assert all(column in {'y_1', 'y_2'}
for column in all_outputs_df.columns)
# We should accept inputs with missing output dimensions, as long as at least one is specified.
#
output_with_missing_dimension = Point(y_1=input.x_1)
output_with_missing_dimension_df = output_with_missing_dimension.to_dataframe(
)
optimizer.register(input_df, output_with_missing_dimension_df)
all_inputs_df, all_outputs_df, _ = optimizer.get_all_observations()
# Let's make sure the missing dimension ends up being a null.
#
last_observation = all_outputs_df.iloc[[-1]]
assert last_observation['y_2'].isnull().values.all()
# Inserting an observation with no valid dimensions should fail.
#
empty_output = Point()
empty_output_df = empty_output.to_dataframe()
with pytest.raises(ValueError):
optimizer.register(input_df, empty_output_df)
only_invalid_outputs = Point(invalid_col1=0, invalid_col2=2)
only_invalid_outputs_df = only_invalid_outputs.to_dataframe()
with pytest.raises(ValueError):
optimizer.register(input_df, only_invalid_outputs_df)
