def test_pending_observations_as_array(self): # Mark a trial dispatched so that there are pending observations. self.trial.mark_running(no_runner_required=True) # If outcome names are respected, unlisted metrics should be filtered out. self.assertEqual( [ x.tolist() for x in pending_observations_as_array( pending_observations=get_pending_observation_features( self.experiment), outcome_names=["m2", "m1"], param_names=["x", "y", "z", "w"], ) ], [[["1", "foo", "True", "4"]], [["1", "foo", "True", "4"]]], ) self.experiment.attach_data( Data.from_evaluations({self.trial.arm.name: { "m2": (1, 0) }}, trial_index=self.trial.index)) # There should be no pending observations for metric m2 now, since the # only trial there is, has been updated with data for it. self.assertEqual( [ x.tolist() for x in pending_observations_as_array( pending_observations=get_pending_observation_features( self.experiment), outcome_names=["m2", "m1"], param_names=["x", "y", "z", "w"], ) ], [[], [["1", "foo", "True", "4"]]], )
def _get_transformed_model_gen_args( self, search_space: SearchSpace, pending_observations: Dict[str, List[ObservationFeatures]], fixed_features: ObservationFeatures, model_gen_options: Optional[TConfig] = None, optimization_config: Optional[OptimizationConfig] = None, ) -> ArrayModelGenArgs: # Validation if not self.parameters: # pragma: no cover raise ValueError(FIT_MODEL_ERROR.format(action="_gen")) # Extract search space info search_space_digest = extract_search_space_digest( search_space=search_space, param_names=self.parameters ) if optimization_config is None: raise ValueError( "ArrayModelBridge requires an OptimizationConfig to be specified" ) if self.outcomes is None or len(self.outcomes) == 0: # pragma: no cover raise ValueError("No outcomes found during model fit--data are missing.") validate_optimization_config(optimization_config, self.outcomes) objective_weights = extract_objective_weights( objective=optimization_config.objective, outcomes=self.outcomes ) outcome_constraints = extract_outcome_constraints( outcome_constraints=optimization_config.outcome_constraints, outcomes=self.outcomes, ) extra_model_gen_kwargs = self._get_extra_model_gen_kwargs( optimization_config=optimization_config ) linear_constraints = extract_parameter_constraints( search_space.parameter_constraints, self.parameters ) fixed_features_dict = get_fixed_features(fixed_features, self.parameters) pending_array = pending_observations_as_array( pending_observations, self.outcomes, self.parameters ) return ArrayModelGenArgs( search_space_digest=search_space_digest, objective_weights=objective_weights, outcome_constraints=outcome_constraints, linear_constraints=linear_constraints, fixed_features=fixed_features_dict, pending_observations=pending_array, rounding_func=transform_callback(self.parameters, self.transforms), extra_model_gen_kwargs=extra_model_gen_kwargs, )
def _gen( self, n: int, search_space: SearchSpace, pending_observations: Dict[str, List[ObservationFeatures]], fixed_features: ObservationFeatures, model_gen_options: Optional[TConfig] = None, optimization_config: Optional[OptimizationConfig] = None, ) -> Tuple[List[ObservationFeatures], List[float], Optional[ObservationFeatures], TGenMetadata, ]: """Generate new candidates according to search_space and optimization_config. The outcome constraints should be transformed to no longer be relative. """ # Validation if not self.parameters: # pragma: no cover raise ValueError(FIT_MODEL_ERROR.format(action="_gen")) # Extract bounds bounds, _, target_fidelities = get_bounds_and_task( search_space=search_space, param_names=self.parameters) target_fidelities = { i: float(v) for i, v in target_fidelities.items() # pyre-ignore [6] } if optimization_config is None: raise ValueError( "ArrayModelBridge requires an OptimizationConfig to be specified" ) if self.outcomes is None or len( self.outcomes) == 0: # pragma: no cover raise ValueError( "No outcomes found during model fit--data are missing.") validate_optimization_config(optimization_config, self.outcomes) objective_weights = extract_objective_weights( objective=optimization_config.objective, outcomes=self.outcomes) outcome_constraints = extract_outcome_constraints( outcome_constraints=optimization_config.outcome_constraints, outcomes=self.outcomes, ) extra_model_gen_kwargs = self._get_extra_model_gen_kwargs( optimization_config=optimization_config) linear_constraints = extract_parameter_constraints( search_space.parameter_constraints, self.parameters) fixed_features_dict = get_fixed_features(fixed_features, self.parameters) pending_array = pending_observations_as_array(pending_observations, self.outcomes, self.parameters) # Generate the candidates X, w, gen_metadata, candidate_metadata = self._model_gen( n=n, bounds=bounds, objective_weights=objective_weights, outcome_constraints=outcome_constraints, linear_constraints=linear_constraints, fixed_features=fixed_features_dict, pending_observations=pending_array, model_gen_options=model_gen_options, rounding_func=transform_callback(self.parameters, self.transforms), target_fidelities=target_fidelities, **extra_model_gen_kwargs, ) # Transform array to observations observation_features = parse_observation_features( X=X, param_names=self.parameters, candidate_metadata=candidate_metadata) xbest = self._model_best_point( bounds=bounds, objective_weights=objective_weights, outcome_constraints=outcome_constraints, linear_constraints=linear_constraints, fixed_features=fixed_features_dict, model_gen_options=model_gen_options, target_fidelities=target_fidelities, ) best_obsf = (None if xbest is None else ObservationFeatures( parameters={ p: float(xbest[i]) for i, p in enumerate(self.parameters) })) return observation_features, w.tolist(), best_obsf, gen_metadata