def test_lower_confidence_bound(self):
"""Tests if the lower confidence bound utility function is behaving properly."""
utility_function_config = Point(
utility_function_name="lower_confidence_bound_on_improvement",
alpha=0.01)
utility_function = ConfidenceBoundUtilityFunction(
function_config=utility_function_config,
surrogate_model=self.model,
minimize=False)
predicted_value_col = Prediction.LegalColumnNames.PREDICTED_VALUE.value
predicted_value_var_col = Prediction.LegalColumnNames.PREDICTED_VALUE_VARIANCE.value
dof_col = Prediction.LegalColumnNames.PREDICTED_VALUE_DEGREES_OF_FREEDOM.value
prediction_df = self.sample_predictions.get_dataframe()
t_values = t.ppf(1 - utility_function_config.alpha / 2.0,
prediction_df[dof_col])
confidence_interval_radii = t_values * prediction_df[
predicted_value_var_col].apply('sqrt')
expected_utility_function_values = prediction_df[
predicted_value_col] - confidence_interval_radii
utility_function_values = utility_function(
self.sample_inputs_pandas_dataframe)['utility']
for expected, actual in zip(expected_utility_function_values,
utility_function_values):
assert (expected == actual) or (np.isnan(expected)
and np.isnan(actual))
def test_random_function_configs(self):
for i in range(100):
minimize = [True, False][i % 2]
utility_function_config = confidence_bound_utility_function_config_store.parameter_space.random(
)
utility_function = ConfidenceBoundUtilityFunction(
function_config=utility_function_config,
surrogate_model=self.model,
minimize=minimize)
predicted_value_col = Prediction.LegalColumnNames.PREDICTED_VALUE.value
predicted_value_var_col = Prediction.LegalColumnNames.PREDICTED_VALUE_VARIANCE.value
dof_col = Prediction.LegalColumnNames.PREDICTED_VALUE_DEGREES_OF_FREEDOM.value
sign = -1 if minimize else 1
prediction_df = self.sample_predictions.get_dataframe()
t_values = t.ppf(1 - utility_function_config.alpha / 2.0,
prediction_df[dof_col])
confidence_interval_radii = t_values * prediction_df[
predicted_value_var_col].apply('sqrt')
if utility_function_config.utility_function_name == 'lower_confidence_bound_on_improvement':
expected_utility_function_values = sign * prediction_df[
predicted_value_col] - confidence_interval_radii
else:
expected_utility_function_values = sign * prediction_df[
predicted_value_col] + confidence_interval_radii
utility_function_values = utility_function(
self.sample_inputs_pandas_dataframe)['utility']
for expected, actual in zip(expected_utility_function_values,
utility_function_values):
assert (expected == actual) or (np.isnan(expected)
and np.isnan(actual))
def setup_class(cls):
""" Set's up all the objects needed to test the RandomSearchOptimizer
To test the RandomSearchOptimizer we need to first construct:
* an optimization problem
* a utility function
To construct a utility function we need the same set up as in the TestConfidenceBoundUtilityFunction
test.
:return:
"""
global_values.declare_singletons()
global_values.tracer = Tracer(actor_id=cls.__name__, thread_id=0)
objective_function_config = objective_function_config_store.get_config_by_name(
'2d_quadratic_concave_up')
objective_function = ObjectiveFunctionFactory.create_objective_function(
objective_function_config=objective_function_config)
cls.input_space = objective_function.parameter_space
cls.output_space = objective_function.output_space
cls.input_values_dataframe = objective_function.parameter_space.random_dataframe(
num_samples=2500)
cls.output_values_dataframe = objective_function.evaluate_dataframe(
cls.input_values_dataframe)
cls.model_config = homogeneous_random_forest_config_store.default
print(cls.model_config)
cls.model = MultiObjectiveHomogeneousRandomForest(
model_config=cls.model_config,
input_space=cls.input_space,
output_space=cls.output_space)
cls.model.fit(cls.input_values_dataframe,
cls.output_values_dataframe,
iteration_number=len(cls.input_values_dataframe.index))
cls.utility_function_config = Point(
utility_function_name="upper_confidence_bound_on_improvement",
alpha=0.05)
cls.optimization_problem = OptimizationProblem(
parameter_space=cls.input_space,
objective_space=cls.output_space,
objectives=[Objective(name='y', minimize=True)])
cls.utility_function = ConfidenceBoundUtilityFunction(
function_config=cls.utility_function_config,
surrogate_model=cls.model,
minimize=cls.optimization_problem.objectives[0].minimize)
def setup_class(cls):
""" Set's up all the objects needed to test the UtilityFunctionOptimizers
To test the UtilityFunctionOptimizers we need to first construct:
* an objective function for the model to approximate and its corresponding parameter and output spaces
* an optimization problem
* a regression model, then train it on some random parameters to the objective function
* a utility function that utilizes the model
* a pareto frontier over the random parameters
And only then do we get to test our utility function optimizers. This is a lot of work and a somewhat cleaner approach
would be to simply create an instance of the BayesianOptimizer to do all of the above for us, but then we might not be able
to test the utility function optimizers as thoroughly as we need to.
:return:
"""
global_values.declare_singletons()
global_values.tracer = Tracer(actor_id=cls.__name__, thread_id=0)
cls.logger = create_logger("TestUtilityFunctionOptimizers")
cls.model_config = multi_objective_pass_through_model_config_store.default
cls.model = MultiObjectivePassThroughModelForTesting(
model_config=cls.model_config,
logger=cls.logger
)
cls.objective_function = cls.model.objective_function
cls.parameter_space = cls.objective_function.parameter_space
cls.objective_space = cls.objective_function.output_space
cls.optimization_problem = cls.objective_function.default_optimization_problem
cls.utility_function_config = Point(utility_function_name="upper_confidence_bound_on_improvement", alpha=0.05)
cls.utility_function = ConfidenceBoundUtilityFunction(
function_config=cls.utility_function_config,
surrogate_model=cls.model,
minimize=cls.optimization_problem.objectives[0].minimize,
logger=cls.logger
)
# To make the pareto frontier we have to generate some random points.
#
cls.parameters_df = cls.objective_function.parameter_space.random_dataframe(1000)
cls.objectives_df = cls.objective_function.evaluate_dataframe(cls.parameters_df)
cls.pareto_frontier = ParetoFrontier(
optimization_problem=cls.optimization_problem,
objectives_df=cls.objectives_df,
parameters_df=cls.parameters_df
)
def test_glow_worm_on_three_level_quadratic(self):
output_space = SimpleHypergrid(name="output",
dimensions=[
ContinuousDimension(name='y',
min=-math.inf,
max=math.inf)
])
objective_function_config = objective_function_config_store.get_config_by_name(
'three_level_quadratic')
objective_function = ObjectiveFunctionFactory.create_objective_function(
objective_function_config=objective_function_config)
# Let's warm up the model a bit
#
num_warmup_samples = 1000
random_params_df = objective_function.parameter_space.random_dataframe(
num_samples=num_warmup_samples)
y = objective_function.evaluate_dataframe(random_params_df)
model = HomogeneousRandomForestRegressionModel(
model_config=self.model_config,
input_space=objective_function.parameter_space,
output_space=output_space)
model.fit(feature_values_pandas_frame=random_params_df,
target_values_pandas_frame=y,
iteration_number=num_warmup_samples)
optimization_problem = OptimizationProblem(
parameter_space=objective_function.parameter_space,
objective_space=output_space,
objectives=[Objective(name='y', minimize=True)])
utility_function = ConfidenceBoundUtilityFunction(
function_config=self.utility_function_config,
surrogate_model=model,
minimize=optimization_problem.objectives[0].minimize)
glow_worm_swarm_optimizer = GlowWormSwarmOptimizer(
optimization_problem=optimization_problem,
utility_function=utility_function,
optimizer_config=glow_worm_swarm_optimizer_config_store.default)
num_iterations = 5
for i in range(num_iterations):
suggested_params = glow_worm_swarm_optimizer.suggest()
print(f"[{i+1}/{num_iterations}] {suggested_params.to_json()}")
self.assertTrue(
suggested_params in objective_function.parameter_space)
def _prepare_dummy_model_based_test_artifacts(self, dummy_model_config, logger):
"""Prepares all the artifacts we need to create and run a utility function optimizer.
I chose to create them here rather than in setup class, to avoid unnecessarily creating all possible combinations for all
possible tests. It's easier and cheaper to produce this artifacts just in time, rather than upfront.
I suspect that pytest has a functionality to accomplish just this, but haven't found it yet.
We need to produce:
* an optimization problem
* a model
* a utility function
* pareto frontier
"""
model = MultiObjectivePassThroughModelForTesting(model_config=dummy_model_config, logger=logger)
objective_function = model.objective_function
optimization_problem = objective_function.default_optimization_problem
# Let's create the pareto frontier.
#
params_df = objective_function.parameter_space.random_dataframe(1000)
objectives_df = objective_function.evaluate_dataframe(params_df)
pareto_frontier = ParetoFrontier(
optimization_problem=optimization_problem,
objectives_df=objectives_df,
parameters_df=params_df
)
if len(optimization_problem.objectives) == 1:
utility_function_config = Point(utility_function_name="upper_confidence_bound_on_improvement", alpha=0.05)
utility_function=ConfidenceBoundUtilityFunction(
function_config=utility_function_config,
surrogate_model=model,
minimize=optimization_problem.objectives[0].minimize,
logger=logger
)
else:
utility_function_config = multi_objective_probability_of_improvement_utility_function_config_store.default
utility_function = MultiObjectiveProbabilityOfImprovementUtilityFunction(
function_config=utility_function_config,
pareto_frontier=pareto_frontier,
surrogate_model=model,
logger=logger
)
return optimization_problem, model, utility_function, pareto_frontier
def test_optimizers_against_untrained_models(self, objective_function_config_name, utility_function_type_name, utility_function_optimizer_type_name):
"""Tests that the utility function optimizers throw appropriate exceptions when the utility function cannot be evaluated.
:return:
"""
self.logger.info(f"Creating test artifacts for objective function: {objective_function_config_name}, utility_function: {utility_function_optimizer_type_name}, optimizer: {utility_function_optimizer_type_name}.")
model_config = homogeneous_random_forest_config_store.default
objective_function_config = objective_function_config_store.get_config_by_name(objective_function_config_name)
objective_function = ObjectiveFunctionFactory.create_objective_function(objective_function_config=objective_function_config)
optimization_problem = objective_function.default_optimization_problem
model = MultiObjectiveHomogeneousRandomForest(
model_config=model_config,
input_space=optimization_problem.feature_space,
output_space=optimization_problem.objective_space,
logger=self.logger
)
pareto_frontier = ParetoFrontier(optimization_problem=optimization_problem)
if utility_function_type_name == ConfidenceBoundUtilityFunction.__name__:
utility_function_config = Point(utility_function_name="upper_confidence_bound_on_improvement", alpha=0.05)
utility_function = ConfidenceBoundUtilityFunction(
function_config=utility_function_config,
surrogate_model=model,
minimize=optimization_problem.objectives[0].minimize,
logger=self.logger
)
elif utility_function_type_name == MultiObjectiveProbabilityOfImprovementUtilityFunction.__name__:
utility_function_config = multi_objective_probability_of_improvement_utility_function_config_store.default
utility_function = MultiObjectiveProbabilityOfImprovementUtilityFunction(
function_config=utility_function_config,
pareto_frontier=pareto_frontier,
surrogate_model=model,
logger=self.logger
)
else:
assert False
if utility_function_optimizer_type_name == RandomSearchOptimizer.__name__:
utility_function_optimizer_config = random_search_optimizer_config_store.default
elif utility_function_optimizer_type_name == GlowWormSwarmOptimizer.__name__:
utility_function_optimizer_config = glow_worm_swarm_optimizer_config_store.default
elif utility_function_optimizer_type_name == RandomNearIncumbentOptimizer.__name__:
utility_function_optimizer_config = random_near_incumbent_optimizer_config_store.default
else:
assert False, f"Unknown utility_function_optimizer_type_name: {utility_function_optimizer_type_name}"
utility_function_optimizer = UtilityFunctionOptimizerFactory.create_utility_function_optimizer(
utility_function=utility_function,
optimizer_type_name=utility_function_optimizer_type_name,
optimizer_config=utility_function_optimizer_config,
optimization_problem=optimization_problem,
pareto_frontier=pareto_frontier,
logger=self.logger
)
assert not model.trained
self.logger.info("Asserting the optimizer is throwing appropriate exceptions.")
num_failed_suggestions = 3
for i in range(num_failed_suggestions):
with pytest.raises(expected_exception=UnableToProduceGuidedSuggestionException):
utility_function_optimizer.suggest()
self.logger.info(f"[{i+1}/{num_failed_suggestions}] worked.")
# Now let's train the model a bit and make sure that we can produce the suggestions afterwards
#
random_params_df = optimization_problem.parameter_space.random_dataframe(1000)
objectives_df = objective_function.evaluate_dataframe(random_params_df)
features_df = optimization_problem.construct_feature_dataframe(parameters_df=random_params_df)
self.logger.info("Training the model")
model.fit(features_df=features_df, targets_df=objectives_df, iteration_number=1000)
assert model.trained
self.logger.info("Model trained.")
self.logger.info("Updating pareto.")
pareto_frontier.update_pareto(objectives_df=objectives_df, parameters_df=random_params_df)
self.logger.info("Pareto updated.")
self.logger.info("Asserting suggestions work.")
num_successful_suggestions = 3
for i in range(num_successful_suggestions):
suggestion = utility_function_optimizer.suggest()
assert suggestion in optimization_problem.parameter_space
self.logger.info(f"[{i+1}/{num_successful_suggestions}] successfully produced suggestion: {suggestion}")
self.logger.info(f"Done testing. Objective function: {objective_function_config_name}, utility_function: {utility_function_optimizer_type_name}, optimizer: {utility_function_optimizer_type_name}.")
def setUpClass(cls):
""" Set's up all the objects needed to test the RandomSearchOptimizer
To test the RandomSearchOptimizer we need to first construct:
* an optimization problem
* a utility function
To construct a utility function we need the same set up as in the TestConfidenceBoundUtilityFunction
test.
:return:
"""
global_values.declare_singletons()
cls.input_space = SimpleHypergrid(name="input",
dimensions=[
ContinuousDimension(name='x_1',
min=-100,
max=100),
ContinuousDimension(name='x_2',
min=-100,
max=100)
])
cls.output_space = SimpleHypergrid(name="output",
dimensions=[
ContinuousDimension(
name='y',
min=-math.inf,
max=math.inf)
])
x_1, x_2 = np.meshgrid(cls.input_space['x_1'].linspace(num=201),
cls.input_space['x_2'].linspace(num=201))
y = -quadratic(x_1=x_1, x_2=x_2)
cls.input_values_dataframe = pd.DataFrame({
'x_1': x_1.reshape(-1),
'x_2': x_2.reshape(-1)
})
cls.output_values_dataframe = pd.DataFrame({'y': y.reshape(-1)})
cls.model_config = HomogeneousRandomForestRegressionModelConfig()
cls.model = HomogeneousRandomForestRegressionModel(
model_config=cls.model_config,
input_space=cls.input_space,
output_space=cls.output_space)
cls.model.fit(cls.input_values_dataframe,
cls.output_values_dataframe,
iteration_number=len(cls.input_values_dataframe.index))
cls.utility_function_config = ConfidenceBoundUtilityFunctionConfig(
utility_function_name="upper_confidence_bound_on_improvement",
alpha=0.05)
cls.optimization_problem = OptimizationProblem(
parameter_space=cls.input_space,
objective_space=cls.output_space,
objectives=[Objective(name='y', minimize=True)])
cls.utility_function = ConfidenceBoundUtilityFunction(
function_config=cls.utility_function_config,
surrogate_model=cls.model,
minimize=cls.optimization_problem.objectives[0].minimize)