def test_lower_confidence_bound(self): """Tests if the lower confidence bound utility function is behaving properly.""" utility_function_config = Point( utility_function_name="lower_confidence_bound_on_improvement", alpha=0.01) utility_function = ConfidenceBoundUtilityFunction( function_config=utility_function_config, surrogate_model=self.model, minimize=False) predicted_value_col = Prediction.LegalColumnNames.PREDICTED_VALUE.value predicted_value_var_col = Prediction.LegalColumnNames.PREDICTED_VALUE_VARIANCE.value dof_col = Prediction.LegalColumnNames.PREDICTED_VALUE_DEGREES_OF_FREEDOM.value prediction_df = self.sample_predictions.get_dataframe() t_values = t.ppf(1 - utility_function_config.alpha / 2.0, prediction_df[dof_col]) confidence_interval_radii = t_values * prediction_df[ predicted_value_var_col].apply('sqrt') expected_utility_function_values = prediction_df[ predicted_value_col] - confidence_interval_radii utility_function_values = utility_function( self.sample_inputs_pandas_dataframe)['utility'] for expected, actual in zip(expected_utility_function_values, utility_function_values): assert (expected == actual) or (np.isnan(expected) and np.isnan(actual))
def test_random_function_configs(self): for i in range(100): minimize = [True, False][i % 2] utility_function_config = confidence_bound_utility_function_config_store.parameter_space.random( ) utility_function = ConfidenceBoundUtilityFunction( function_config=utility_function_config, surrogate_model=self.model, minimize=minimize) predicted_value_col = Prediction.LegalColumnNames.PREDICTED_VALUE.value predicted_value_var_col = Prediction.LegalColumnNames.PREDICTED_VALUE_VARIANCE.value dof_col = Prediction.LegalColumnNames.PREDICTED_VALUE_DEGREES_OF_FREEDOM.value sign = -1 if minimize else 1 prediction_df = self.sample_predictions.get_dataframe() t_values = t.ppf(1 - utility_function_config.alpha / 2.0, prediction_df[dof_col]) confidence_interval_radii = t_values * prediction_df[ predicted_value_var_col].apply('sqrt') if utility_function_config.utility_function_name == 'lower_confidence_bound_on_improvement': expected_utility_function_values = sign * prediction_df[ predicted_value_col] - confidence_interval_radii else: expected_utility_function_values = sign * prediction_df[ predicted_value_col] + confidence_interval_radii utility_function_values = utility_function( self.sample_inputs_pandas_dataframe)['utility'] for expected, actual in zip(expected_utility_function_values, utility_function_values): assert (expected == actual) or (np.isnan(expected) and np.isnan(actual))
def setup_class(cls): """ Set's up all the objects needed to test the RandomSearchOptimizer To test the RandomSearchOptimizer we need to first construct: * an optimization problem * a utility function To construct a utility function we need the same set up as in the TestConfidenceBoundUtilityFunction test. :return: """ global_values.declare_singletons() global_values.tracer = Tracer(actor_id=cls.__name__, thread_id=0) objective_function_config = objective_function_config_store.get_config_by_name( '2d_quadratic_concave_up') objective_function = ObjectiveFunctionFactory.create_objective_function( objective_function_config=objective_function_config) cls.input_space = objective_function.parameter_space cls.output_space = objective_function.output_space cls.input_values_dataframe = objective_function.parameter_space.random_dataframe( num_samples=2500) cls.output_values_dataframe = objective_function.evaluate_dataframe( cls.input_values_dataframe) cls.model_config = homogeneous_random_forest_config_store.default print(cls.model_config) cls.model = MultiObjectiveHomogeneousRandomForest( model_config=cls.model_config, input_space=cls.input_space, output_space=cls.output_space) cls.model.fit(cls.input_values_dataframe, cls.output_values_dataframe, iteration_number=len(cls.input_values_dataframe.index)) cls.utility_function_config = Point( utility_function_name="upper_confidence_bound_on_improvement", alpha=0.05) cls.optimization_problem = OptimizationProblem( parameter_space=cls.input_space, objective_space=cls.output_space, objectives=[Objective(name='y', minimize=True)]) cls.utility_function = ConfidenceBoundUtilityFunction( function_config=cls.utility_function_config, surrogate_model=cls.model, minimize=cls.optimization_problem.objectives[0].minimize)
def setup_class(cls): """ Set's up all the objects needed to test the UtilityFunctionOptimizers To test the UtilityFunctionOptimizers we need to first construct: * an objective function for the model to approximate and its corresponding parameter and output spaces * an optimization problem * a regression model, then train it on some random parameters to the objective function * a utility function that utilizes the model * a pareto frontier over the random parameters And only then do we get to test our utility function optimizers. This is a lot of work and a somewhat cleaner approach would be to simply create an instance of the BayesianOptimizer to do all of the above for us, but then we might not be able to test the utility function optimizers as thoroughly as we need to. :return: """ global_values.declare_singletons() global_values.tracer = Tracer(actor_id=cls.__name__, thread_id=0) cls.logger = create_logger("TestUtilityFunctionOptimizers") cls.model_config = multi_objective_pass_through_model_config_store.default cls.model = MultiObjectivePassThroughModelForTesting( model_config=cls.model_config, logger=cls.logger ) cls.objective_function = cls.model.objective_function cls.parameter_space = cls.objective_function.parameter_space cls.objective_space = cls.objective_function.output_space cls.optimization_problem = cls.objective_function.default_optimization_problem cls.utility_function_config = Point(utility_function_name="upper_confidence_bound_on_improvement", alpha=0.05) cls.utility_function = ConfidenceBoundUtilityFunction( function_config=cls.utility_function_config, surrogate_model=cls.model, minimize=cls.optimization_problem.objectives[0].minimize, logger=cls.logger ) # To make the pareto frontier we have to generate some random points. # cls.parameters_df = cls.objective_function.parameter_space.random_dataframe(1000) cls.objectives_df = cls.objective_function.evaluate_dataframe(cls.parameters_df) cls.pareto_frontier = ParetoFrontier( optimization_problem=cls.optimization_problem, objectives_df=cls.objectives_df, parameters_df=cls.parameters_df )
def test_glow_worm_on_three_level_quadratic(self): output_space = SimpleHypergrid(name="output", dimensions=[ ContinuousDimension(name='y', min=-math.inf, max=math.inf) ]) objective_function_config = objective_function_config_store.get_config_by_name( 'three_level_quadratic') objective_function = ObjectiveFunctionFactory.create_objective_function( objective_function_config=objective_function_config) # Let's warm up the model a bit # num_warmup_samples = 1000 random_params_df = objective_function.parameter_space.random_dataframe( num_samples=num_warmup_samples) y = objective_function.evaluate_dataframe(random_params_df) model = HomogeneousRandomForestRegressionModel( model_config=self.model_config, input_space=objective_function.parameter_space, output_space=output_space) model.fit(feature_values_pandas_frame=random_params_df, target_values_pandas_frame=y, iteration_number=num_warmup_samples) optimization_problem = OptimizationProblem( parameter_space=objective_function.parameter_space, objective_space=output_space, objectives=[Objective(name='y', minimize=True)]) utility_function = ConfidenceBoundUtilityFunction( function_config=self.utility_function_config, surrogate_model=model, minimize=optimization_problem.objectives[0].minimize) glow_worm_swarm_optimizer = GlowWormSwarmOptimizer( optimization_problem=optimization_problem, utility_function=utility_function, optimizer_config=glow_worm_swarm_optimizer_config_store.default) num_iterations = 5 for i in range(num_iterations): suggested_params = glow_worm_swarm_optimizer.suggest() print(f"[{i+1}/{num_iterations}] {suggested_params.to_json()}") self.assertTrue( suggested_params in objective_function.parameter_space)
def _prepare_dummy_model_based_test_artifacts(self, dummy_model_config, logger): """Prepares all the artifacts we need to create and run a utility function optimizer. I chose to create them here rather than in setup class, to avoid unnecessarily creating all possible combinations for all possible tests. It's easier and cheaper to produce this artifacts just in time, rather than upfront. I suspect that pytest has a functionality to accomplish just this, but haven't found it yet. We need to produce: * an optimization problem * a model * a utility function * pareto frontier """ model = MultiObjectivePassThroughModelForTesting(model_config=dummy_model_config, logger=logger) objective_function = model.objective_function optimization_problem = objective_function.default_optimization_problem # Let's create the pareto frontier. # params_df = objective_function.parameter_space.random_dataframe(1000) objectives_df = objective_function.evaluate_dataframe(params_df) pareto_frontier = ParetoFrontier( optimization_problem=optimization_problem, objectives_df=objectives_df, parameters_df=params_df ) if len(optimization_problem.objectives) == 1: utility_function_config = Point(utility_function_name="upper_confidence_bound_on_improvement", alpha=0.05) utility_function=ConfidenceBoundUtilityFunction( function_config=utility_function_config, surrogate_model=model, minimize=optimization_problem.objectives[0].minimize, logger=logger ) else: utility_function_config = multi_objective_probability_of_improvement_utility_function_config_store.default utility_function = MultiObjectiveProbabilityOfImprovementUtilityFunction( function_config=utility_function_config, pareto_frontier=pareto_frontier, surrogate_model=model, logger=logger ) return optimization_problem, model, utility_function, pareto_frontier
def test_optimizers_against_untrained_models(self, objective_function_config_name, utility_function_type_name, utility_function_optimizer_type_name): """Tests that the utility function optimizers throw appropriate exceptions when the utility function cannot be evaluated. :return: """ self.logger.info(f"Creating test artifacts for objective function: {objective_function_config_name}, utility_function: {utility_function_optimizer_type_name}, optimizer: {utility_function_optimizer_type_name}.") model_config = homogeneous_random_forest_config_store.default objective_function_config = objective_function_config_store.get_config_by_name(objective_function_config_name) objective_function = ObjectiveFunctionFactory.create_objective_function(objective_function_config=objective_function_config) optimization_problem = objective_function.default_optimization_problem model = MultiObjectiveHomogeneousRandomForest( model_config=model_config, input_space=optimization_problem.feature_space, output_space=optimization_problem.objective_space, logger=self.logger ) pareto_frontier = ParetoFrontier(optimization_problem=optimization_problem) if utility_function_type_name == ConfidenceBoundUtilityFunction.__name__: utility_function_config = Point(utility_function_name="upper_confidence_bound_on_improvement", alpha=0.05) utility_function = ConfidenceBoundUtilityFunction( function_config=utility_function_config, surrogate_model=model, minimize=optimization_problem.objectives[0].minimize, logger=self.logger ) elif utility_function_type_name == MultiObjectiveProbabilityOfImprovementUtilityFunction.__name__: utility_function_config = multi_objective_probability_of_improvement_utility_function_config_store.default utility_function = MultiObjectiveProbabilityOfImprovementUtilityFunction( function_config=utility_function_config, pareto_frontier=pareto_frontier, surrogate_model=model, logger=self.logger ) else: assert False if utility_function_optimizer_type_name == RandomSearchOptimizer.__name__: utility_function_optimizer_config = random_search_optimizer_config_store.default elif utility_function_optimizer_type_name == GlowWormSwarmOptimizer.__name__: utility_function_optimizer_config = glow_worm_swarm_optimizer_config_store.default elif utility_function_optimizer_type_name == RandomNearIncumbentOptimizer.__name__: utility_function_optimizer_config = random_near_incumbent_optimizer_config_store.default else: assert False, f"Unknown utility_function_optimizer_type_name: {utility_function_optimizer_type_name}" utility_function_optimizer = UtilityFunctionOptimizerFactory.create_utility_function_optimizer( utility_function=utility_function, optimizer_type_name=utility_function_optimizer_type_name, optimizer_config=utility_function_optimizer_config, optimization_problem=optimization_problem, pareto_frontier=pareto_frontier, logger=self.logger ) assert not model.trained self.logger.info("Asserting the optimizer is throwing appropriate exceptions.") num_failed_suggestions = 3 for i in range(num_failed_suggestions): with pytest.raises(expected_exception=UnableToProduceGuidedSuggestionException): utility_function_optimizer.suggest() self.logger.info(f"[{i+1}/{num_failed_suggestions}] worked.") # Now let's train the model a bit and make sure that we can produce the suggestions afterwards # random_params_df = optimization_problem.parameter_space.random_dataframe(1000) objectives_df = objective_function.evaluate_dataframe(random_params_df) features_df = optimization_problem.construct_feature_dataframe(parameters_df=random_params_df) self.logger.info("Training the model") model.fit(features_df=features_df, targets_df=objectives_df, iteration_number=1000) assert model.trained self.logger.info("Model trained.") self.logger.info("Updating pareto.") pareto_frontier.update_pareto(objectives_df=objectives_df, parameters_df=random_params_df) self.logger.info("Pareto updated.") self.logger.info("Asserting suggestions work.") num_successful_suggestions = 3 for i in range(num_successful_suggestions): suggestion = utility_function_optimizer.suggest() assert suggestion in optimization_problem.parameter_space self.logger.info(f"[{i+1}/{num_successful_suggestions}] successfully produced suggestion: {suggestion}") self.logger.info(f"Done testing. Objective function: {objective_function_config_name}, utility_function: {utility_function_optimizer_type_name}, optimizer: {utility_function_optimizer_type_name}.")
def setUpClass(cls): """ Set's up all the objects needed to test the RandomSearchOptimizer To test the RandomSearchOptimizer we need to first construct: * an optimization problem * a utility function To construct a utility function we need the same set up as in the TestConfidenceBoundUtilityFunction test. :return: """ global_values.declare_singletons() cls.input_space = SimpleHypergrid(name="input", dimensions=[ ContinuousDimension(name='x_1', min=-100, max=100), ContinuousDimension(name='x_2', min=-100, max=100) ]) cls.output_space = SimpleHypergrid(name="output", dimensions=[ ContinuousDimension( name='y', min=-math.inf, max=math.inf) ]) x_1, x_2 = np.meshgrid(cls.input_space['x_1'].linspace(num=201), cls.input_space['x_2'].linspace(num=201)) y = -quadratic(x_1=x_1, x_2=x_2) cls.input_values_dataframe = pd.DataFrame({ 'x_1': x_1.reshape(-1), 'x_2': x_2.reshape(-1) }) cls.output_values_dataframe = pd.DataFrame({'y': y.reshape(-1)}) cls.model_config = HomogeneousRandomForestRegressionModelConfig() cls.model = HomogeneousRandomForestRegressionModel( model_config=cls.model_config, input_space=cls.input_space, output_space=cls.output_space) cls.model.fit(cls.input_values_dataframe, cls.output_values_dataframe, iteration_number=len(cls.input_values_dataframe.index)) cls.utility_function_config = ConfidenceBoundUtilityFunctionConfig( utility_function_name="upper_confidence_bound_on_improvement", alpha=0.05) cls.optimization_problem = OptimizationProblem( parameter_space=cls.input_space, objective_space=cls.output_space, objectives=[Objective(name='y', minimize=True)]) cls.utility_function = ConfidenceBoundUtilityFunction( function_config=cls.utility_function_config, surrogate_model=cls.model, minimize=cls.optimization_problem.objectives[0].minimize)