def test_filtering_out_invalid_rows(self): spaces = [ bayesian_optimizer_config_store.parameter_space, glow_worm_swarm_optimizer_config_store.parameter_space ] # Just to make sure we are testing both hierarchical and flat code paths. # self.assertTrue(any(space.is_hierarchical() for space in spaces)) self.assertTrue(any(not space.is_hierarchical() for space in spaces)) num_samples = 1000 for space in spaces: random_dataframe_with_invalid_rows = space.random_dataframe(num_samples=num_samples) for dimension in space.dimensions: if isinstance(dimension, (ContinuousDimension, DiscreteDimension)): # This makes about half of the rows invalid. # random_dataframe_with_invalid_rows.loc[:, [dimension.name]] *= 2 break with traced(scope_name="slow_filtering"): # Let's filter out invalid rows the slow way. # valid_indices = [] for idx in random_dataframe_with_invalid_rows.index: row_as_df = random_dataframe_with_invalid_rows.loc[[idx]] row_as_point = Point.from_dataframe(row_as_df) if row_as_point in space: valid_indices.append(idx) expected_valid_rows_index = pd.Index(valid_indices) print(f"{len(expected_valid_rows_index)}/{len(random_dataframe_with_invalid_rows.index)} rows are valid.") self.assertTrue(0 < len(expected_valid_rows_index)) self.assertTrue(len(expected_valid_rows_index) < num_samples) # Let's filter out invalid rows the fast way. # actual_valid_rows_index = space.filter_out_invalid_rows(original_dataframe=random_dataframe_with_invalid_rows, exclude_extra_columns=True).index self.assertTrue(expected_valid_rows_index.equals(actual_valid_rows_index)) if not space.is_hierarchical(): # For flat spaces we can choose between the column-wise operators and the row-wise validation. This is to get the tracing data to see the # perf difference, but also to validate correctness by computing the desired index in yet another way. # with traced(scope_name="faster_filtering"): expected_valid_rows_index_2 = random_dataframe_with_invalid_rows[random_dataframe_with_invalid_rows.apply( lambda row: Point(**{dim_name: row[i] for i, dim_name in enumerate(space.dimension_names)}) in space, axis=1 )].index self.assertTrue(expected_valid_rows_index_2.equals(actual_valid_rows_index))
def _naive_poi(self, multi_objective_predictions: MultiObjectivePrediction, valid_predictions_index: pd.Index, std_dev_column_name: str): """Naively generates a monte carlo data frame for each of the feature rows and sends them to ParetoFrontier individually. We should be able to substantially improve on this by batching all those dataframes. :return: """ poi_df = pd.DataFrame(index=valid_predictions_index, columns=['utility'], dtype='float') with traced(scope_name="poi_monte_carlo"): for config_idx in valid_predictions_index: monte_carlo_samples_df = self.create_monte_carlo_samples_df( multi_objective_predictions=multi_objective_predictions, config_idx=config_idx, std_dev_column_name=std_dev_column_name) num_samples = len(monte_carlo_samples_df.index) assert num_samples == self.config.num_monte_carlo_samples # At this point we have a dataframe with all the randomly generated points in the objective space. Let's query the pareto # frontier if they are dominated or not. num_dominated_points = self.pareto_frontier.is_dominated( objectives_df=monte_carlo_samples_df).sum() num_non_dominated_points = num_samples - num_dominated_points probability_of_improvement = num_non_dominated_points / num_samples poi_df.loc[config_idx, 'utility'] = probability_of_improvement return poi_df
def test_named_configs(self): """Tests named optimizer configurations against named objective functions. It is prohibitively expensive to test the entire cross product so we test only its subset, but in such a way that each configuration will be tested at least once. """ optimizer_named_configs = bayesian_optimizer_config_store.list_named_configs() num_optimizer_configs = len(optimizer_named_configs) objective_function_named_configs = objective_function_config_store.list_named_configs() num_objective_function_configs = len(objective_function_named_configs) num_tests = max(num_optimizer_configs, num_objective_function_configs) with traced(scope_name="parallel_tests"), concurrent.futures.ProcessPoolExecutor(max_workers=4) as executor: outstanding_futures = set() for i in range(num_tests): named_optimizer_config = optimizer_named_configs[i % num_optimizer_configs] named_objective_function_config = objective_function_named_configs[i % num_objective_function_configs] print("#####################################################################################################") print(named_optimizer_config) print(named_objective_function_config) optimizer_evaluator_config = optimizer_evaluator_config_store.get_config_by_name(name="parallel_unit_tests_config") optimizer_config = named_optimizer_config.config_point objective_function_config = named_objective_function_config.config_point optimizer_evaluator = OptimizerEvaluator( optimizer_evaluator_config=optimizer_evaluator_config, objective_function_config=objective_function_config, optimizer_config=optimizer_config ) future = executor.submit(optimizer_evaluator.evaluate_optimizer) outstanding_futures.add(future) done_futures, outstanding_futures = concurrent.futures.wait(outstanding_futures, return_when=concurrent.futures.ALL_COMPLETED) for future in done_futures: optimizer_evaluation_report = future.result() assert optimizer_evaluation_report.success mlos.global_values.tracer.trace_events.extend(optimizer_evaluation_report.execution_trace) with pd.option_context('display.max_columns', 100): print(optimizer_evaluation_report.regression_model_goodness_of_fit_state.get_goodness_of_fit_dataframe(DataSetType.TRAIN).tail()) for optimum_name, optimum_over_time in optimizer_evaluation_report.optima_over_time.items(): print("#####################################################################################################") print(optimum_name) print(optimum_over_time.get_dataframe().tail(10)) print("#####################################################################################################")
def evaluate_optimizer(self) -> OptimizerEvaluationReport: # pylint: disable=too-many-statements,too-many-branches evaluation_report = OptimizerEvaluationReport( optimizer_configuration=self.optimizer_config, objective_function_configuration=self.objective_function_config, num_optimization_iterations=self.optimizer_evaluator_config.num_iterations, evaluation_frequency=self.optimizer_evaluator_config.evaluation_frequency ) if self.optimizer_evaluator_config.include_execution_trace_in_report: mlos.global_values.declare_singletons() if mlos.global_values.tracer is None: mlos.global_values.tracer = Tracer() mlos.global_values.tracer.clear_events() if self.optimizer_evaluator_config.include_pickled_objective_function_in_report: evaluation_report.pickled_objective_function_initial_state = pickle.dumps(self.objective_function) if self.optimizer_evaluator_config.include_pickled_optimizer_in_report: evaluation_report.pickled_optimizer_initial_state = pickle.dumps(self.optimizer) multi_objective_regression_model_fit_state = MultiObjectiveRegressionModelFitState(objective_names=self.optimizer.optimization_problem.objective_names) for objective_name in self.optimizer.optimization_problem.objective_names: multi_objective_regression_model_fit_state[objective_name] = RegressionModelFitState() optima_over_time = {} optima_over_time[OptimumDefinition.BEST_OBSERVATION.value] = OptimumOverTime( optimization_problem=self.optimizer.optimization_problem, optimum_definition=OptimumDefinition.BEST_OBSERVATION ) optima_over_time[OptimumDefinition.PREDICTED_VALUE_FOR_OBSERVED_CONFIG.value] = OptimumOverTime( optimization_problem=self.optimizer.optimization_problem, optimum_definition=OptimumDefinition.PREDICTED_VALUE_FOR_OBSERVED_CONFIG ) optima_over_time[f"{OptimumDefinition.UPPER_CONFIDENCE_BOUND_FOR_OBSERVED_CONFIG.value}_99"] = OptimumOverTime( optimization_problem=self.optimizer.optimization_problem, optimum_definition=OptimumDefinition.UPPER_CONFIDENCE_BOUND_FOR_OBSERVED_CONFIG, alpha=0.01 ) optima_over_time[f"{OptimumDefinition.LOWER_CONFIDENCE_BOUND_FOR_OBSERVED_CONFIG.value}_99"] = OptimumOverTime( optimization_problem=self.optimizer.optimization_problem, optimum_definition=OptimumDefinition.LOWER_CONFIDENCE_BOUND_FOR_OBSERVED_CONFIG, alpha=0.01 ) ##################################################################################################### evaluation_report.start_time = datetime.utcnow() i = 0 try: with traced(scope_name="optimization_loop"): for i in range(self.optimizer_evaluator_config.num_iterations): parameters = self.optimizer.suggest() objectives = self.objective_function.evaluate_point(parameters) self.optimizer.register(parameters.to_dataframe(), objectives.to_dataframe()) if i % self.optimizer_evaluator_config.evaluation_frequency == 0: self.logger.info(f"[{i + 1}/{self.optimizer_evaluator_config.num_iterations}]") with traced(scope_name="evaluating_optimizer"): if self.optimizer_evaluator_config.include_pickled_optimizer_in_report: evaluation_report.add_pickled_optimizer(iteration=i, pickled_optimizer=pickle.dumps(self.optimizer)) if self.optimizer.trained: multi_objective_gof_metrics = self.optimizer.compute_surrogate_model_goodness_of_fit() for objective_name, gof_metrics in multi_objective_gof_metrics: multi_objective_regression_model_fit_state[objective_name].set_gof_metrics( data_set_type=DataSetType.TRAIN, gof_metrics=gof_metrics ) for optimum_name, optimum_over_time in optima_over_time.items(): try: config, value = self.optimizer.optimum( optimum_definition=optimum_over_time.optimum_definition, alpha=optimum_over_time.alpha ) optima_over_time[optimum_name].add_optimum_at_iteration( iteration=i, optimum_config=config, optimum_value=value ) except ValueError as e: self.logger.info(f"Failed to get {optimum_name} optimum.", exc_info=True) if self.optimizer_evaluator_config.report_pareto_over_time: evaluation_report.pareto_over_time[i] = copy.deepcopy(self.optimizer.optimization_problem) if self.optimizer_evaluator_config.report_pareto_volume_over_time: volume_estimator = self.optimizer.pareto_frontier.approximate_pareto_volume() ci99_on_volume = volume_estimator.get_two_sided_confidence_interval_on_pareto_volume(alpha=0.01) evaluation_report.pareto_volume_over_time[i] = ci99_on_volume evaluation_report.success = True except Exception as e: evaluation_report.success = False evaluation_report.exception = e evaluation_report.exception_traceback = traceback.format_exc() evaluation_report.end_time = datetime.utcnow() with traced(scope_name="evaluating_optimizer"): # Once the optimization is done, we perform a final evaluation of the optimizer. if self.optimizer.trained: multi_objective_gof_metrics = self.optimizer.compute_surrogate_model_goodness_of_fit() for objective_name, gof_metrics in multi_objective_gof_metrics: multi_objective_regression_model_fit_state[objective_name].set_gof_metrics(data_set_type=DataSetType.TRAIN, gof_metrics=gof_metrics) for optimum_name, optimum_over_time in optima_over_time.items(): try: config, value = self.optimizer.optimum(optimum_definition=optimum_over_time.optimum_definition, alpha=optimum_over_time.alpha) optima_over_time[optimum_name].add_optimum_at_iteration( iteration=self.optimizer_evaluator_config.num_iterations, optimum_config=config, optimum_value=value ) except Exception as e: self.logger.info(f"Failed to get {optimum_name} optimum.", exc_info=True) if self.optimizer_evaluator_config.report_pareto_over_time: evaluation_report.pareto_over_time[i] = copy.deepcopy(self.optimizer.optimization_problem) if self.optimizer_evaluator_config.report_pareto_volume_over_time: volume_estimator = self.optimizer.pareto_frontier.approximate_pareto_volume() ci99_on_volume = volume_estimator.get_two_sided_confidence_interval_on_pareto_volume(alpha=0.01) evaluation_report.pareto_volume_over_time[i] = ci99_on_volume if self.optimizer_evaluator_config.include_execution_trace_in_report: evaluation_report.execution_trace = mlos.global_values.tracer.trace_events mlos.global_values.tracer.clear_events() if self.optimizer_evaluator_config.include_pickled_optimizer_in_report: evaluation_report.add_pickled_optimizer(iteration=i, pickled_optimizer=pickle.dumps(self.optimizer)) if self.optimizer_evaluator_config.include_pickled_objective_function_in_report: evaluation_report.pickled_objective_function_final_state = pickle.dumps(self.objective_function) if self.optimizer_evaluator_config.report_regression_model_goodness_of_fit: evaluation_report.regression_model_fit_state = multi_objective_regression_model_fit_state if self.optimizer_evaluator_config.report_optima_over_time: evaluation_report.optima_over_time = optima_over_time return evaluation_report
def run_iteration(self, worms: pd.DataFrame): with traced(scope_name="numpy_matrix_operations"): positions = worms[self.dimension_names].to_numpy() # At this point many glowworms will have NaNs in their position vectors: for every column that's invalid. # Glowworms with the same set of valid columns, belong to the same subgrids in the hypergrid, but glowworms # with different set of valid columns belong to - essentially - different search spaces, and the idea of # distance ceases to make sense. But their positions are all cast onto this really high dimensional space. # How can we keep them apart? # # Here is the trick: glowworms should only see other glowworms in their own search space. One way to # accomplish that is to fill in the NaNs with a value larger than the max_sensory_radius. Now, glowworms # in different subgrids will never see each other (they can't see that far in this space), but glowworms in # the same subgrids will have the same large placeholder in their invalid dimensions, so it will not contribute # anything to the distance between them. positions = np.nan_to_num(x=positions, copy=False, nan=2 * self.optimizer_config.max_sensory_radius) distances = euclidean_distances(positions, positions) decision_radii = worms['decision_radius'].to_numpy().transpose() # Subtract the sensory radius from each row. Everything in the row, that's negative is your neighbor (if they # also have a higher luciferin level). # distances = (distances - decision_radii).transpose() # Now let's compute the difference in luciferin. Numpy's broadcasting is hard to read, but fast and convenient. # We are basically doing exactly the same thing with luciferin as with distances: whatever is left negative in # your row is your neighbor (if they are also close enough). # luciferin = worms['luciferin'].to_numpy() luciferin_diffs = luciferin[:, np.newaxis] - luciferin # So worms are neighbors if both signs are negative. # distances_signs = np.sign(distances) luciferin_signs = np.sign(luciferin_diffs) summed_signs = distances_signs + luciferin_signs # Now let's put together a matrix, such that in each row for each column we have either: # 0 - if the worm in that column is not a neighbor (too far or too dim) # or luciferin difference between that neighbor and us. # unnormalized_probability = np.where(summed_signs == -2, -luciferin_diffs, 0) # We will have to iterate over all rows anyway, to invoke the np.random.choice() since it operates on # 1-D arrays so we might as well iterate over unnormalized probabilities, check if there is anything # non-zero in there, select the target, compute, and take the step. # for row, unnormalized_probability_row in enumerate( unnormalized_probability): row_sum = unnormalized_probability_row.sum() num_neighbors = np.count_nonzero(unnormalized_probability_row) if row_sum == 0: # nobody is close enough and bright enough continue normalized_probability = unnormalized_probability_row / row_sum col = np.random.choice(len(normalized_probability), size=1, p=normalized_probability)[0] our_position = positions[row] their_position = positions[col] distance = distances[row][col] step_unit_vector = (their_position - our_position) / distance step = self.optimizer_config.step_size * step_unit_vector our_new_position = our_position + step # We only set the non-nan values in the worms dataframe. Remember the trick of setting nans to big values? # This is undoing that trick to hide it from the caller. # TODO: this ends up being pretty slow. See if we can improve. # is_nan = np.isnan(worms.loc[row, self.dimension_names].to_numpy()) our_new_position[is_nan] = np.nan worms.loc[row, self.dimension_names] = our_new_position current_decision_radius = decision_radii[row] decision_radius_update = self.optimizer_config.decision_radius_adjustment_constant * \ (self.optimizer_config.desired_num_neighbors - num_neighbors) new_decision_radius = min( self.optimizer_config.max_sensory_radius, max(0, current_decision_radius + decision_radius_update)) worms.loc[row, ['decision_radius']] = new_decision_radius return worms
def evaluate_optimizer(self) -> OptimizerEvaluationReport: # pylint: disable=too-many-statements evaluation_report = OptimizerEvaluationReport( optimizer_configuration=self.optimizer_config, objective_function_configuration=self.objective_function_config, num_optimization_iterations=self.optimizer_evaluator_config.num_iterations, evaluation_frequency=self.optimizer_evaluator_config.evaluation_frequency, ) if self.optimizer_evaluator_config.include_execution_trace_in_report: mlos.global_values.declare_singletons() if mlos.global_values.tracer is None: mlos.global_values.tracer = Tracer() mlos.global_values.tracer.clear_events() if self.optimizer_evaluator_config.include_pickled_objective_function_in_report: evaluation_report.pickled_objective_function_initial_state = pickle.dumps(self.objective_function) if self.optimizer_evaluator_config.include_pickled_optimizer_in_report: evaluation_report.pickled_optimizer_initial_state = pickle.dumps(self.optimizer) regression_model_fit_state = RegressionModelFitState() optima_over_time = {} optima_over_time[OptimumDefinition.BEST_OBSERVATION.value] = OptimumOverTime( optimization_problem=self.optimizer.optimization_problem, optimum_definition=OptimumDefinition.BEST_OBSERVATION ) optima_over_time[OptimumDefinition.PREDICTED_VALUE_FOR_OBSERVED_CONFIG.value] = OptimumOverTime( optimization_problem=self.optimizer.optimization_problem, optimum_definition=OptimumDefinition.PREDICTED_VALUE_FOR_OBSERVED_CONFIG ) optima_over_time[f"{OptimumDefinition.UPPER_CONFIDENCE_BOUND_FOR_OBSERVED_CONFIG.value}_99"] = OptimumOverTime( optimization_problem=self.optimizer.optimization_problem, optimum_definition=OptimumDefinition.UPPER_CONFIDENCE_BOUND_FOR_OBSERVED_CONFIG, alpha=0.01 ) optima_over_time[f"{OptimumDefinition.LOWER_CONFIDENCE_BOUND_FOR_OBSERVED_CONFIG.value}_99"] = OptimumOverTime( optimization_problem=self.optimizer.optimization_problem, optimum_definition=OptimumDefinition.LOWER_CONFIDENCE_BOUND_FOR_OBSERVED_CONFIG, alpha=0.01 ) ##################################################################################################### i = 0 try: with traced(scope_name="optimization_loop"): for i in range(self.optimizer_evaluator_config.num_iterations): parameters = self.optimizer.suggest() objectives = self.objective_function.evaluate_point(parameters) self.optimizer.register(parameters.to_dataframe(), objectives.to_dataframe()) if i % self.optimizer_evaluator_config.evaluation_frequency == 0: print(f"[{i + 1}/{self.optimizer_evaluator_config.num_iterations}]") with traced(scope_name="evaluating_optimizer"): if self.optimizer_evaluator_config.include_pickled_optimizer_in_report: evaluation_report.add_pickled_optimizer(iteration=i, pickled_optimizer=pickle.dumps(self.optimizer)) if self.optimizer.trained: gof_metrics = self.optimizer.compute_surrogate_model_goodness_of_fit() regression_model_fit_state.set_gof_metrics(data_set_type=DataSetType.TRAIN, gof_metrics=gof_metrics) for optimum_name, optimum_over_time in optima_over_time.items(): try: config, value = self.optimizer.optimum( optimum_definition=optimum_over_time.optimum_definition, alpha=optimum_over_time.alpha ) optima_over_time[optimum_name].add_optimum_at_iteration( iteration=i, optimum_config=config, optimum_value=value ) except ValueError as e: print(e) evaluation_report.success = True except Exception as e: evaluation_report.success = False evaluation_report.exception = e evaluation_report.exception_traceback = traceback.format_exc() with traced(scope_name="evaluating_optimizer"): """Once the optimization is done, we performa final evaluation of the optimizer.""" if self.optimizer.trained: gof_metrics = self.optimizer.compute_surrogate_model_goodness_of_fit() regression_model_fit_state.set_gof_metrics(data_set_type=DataSetType.TRAIN, gof_metrics=gof_metrics) for optimum_name, optimum_over_time in optima_over_time.items(): try: config, value = self.optimizer.optimum(optimum_definition=optimum_over_time.optimum_definition, alpha=optimum_over_time.alpha) optima_over_time[optimum_name].add_optimum_at_iteration( iteration=self.optimizer_evaluator_config.num_iterations, optimum_config=config, optimum_value=value ) except Exception as e: print(e) if self.optimizer_evaluator_config.include_execution_trace_in_report: evaluation_report.execution_trace = mlos.global_values.tracer.trace_events mlos.global_values.tracer.clear_events() if self.optimizer_evaluator_config.include_pickled_optimizer_in_report: evaluation_report.add_pickled_optimizer(iteration=i, pickled_optimizer=pickle.dumps(self.optimizer)) if self.optimizer_evaluator_config.include_pickled_objective_function_in_report: evaluation_report.pickled_objective_function_final_state = pickle.dumps(self.objective_function) if self.optimizer_evaluator_config.report_regression_model_goodness_of_fit: evaluation_report.regression_model_goodness_of_fit_state = regression_model_fit_state if self.optimizer_evaluator_config.report_optima_over_time: evaluation_report.optima_over_time = optima_over_time return evaluation_report