def _validate_and_maybe_get_default_metric_names( metric_names: Optional[Tuple[str, str]], optimization_config: Optional[OptimizationConfig], ) -> Tuple[str, str]: # Default metric_names is all metrics, producing an error if more than 2 if metric_names is None: if not_none(optimization_config).is_moo_problem: multi_objective = checked_cast( MultiObjective, not_none(optimization_config).objective) metric_names = tuple(obj.metric.name for obj in multi_objective.objectives) else: raise UserInputError( "Inference of `metric_names` failed. Expected `MultiObjective` but " f"got {not_none(optimization_config).objective}. Please specify " "`metric_names` of length 2 or provide an experiment whose " "`optimization_config` has 2 objective metrics.") if metric_names is not None and len(metric_names) == 2: return metric_names raise UserInputError( f"Expected 2 metrics but got {len(metric_names or [])}: {metric_names}. " "Please specify `metric_names` of length 2 or provide an experiment whose " "`optimization_config` has 2 objective metrics.")
def get_pareto_optimal_parameters( experiment: Experiment, generation_strategy: GenerationStrategy, use_model_predictions: bool = True, ) -> Optional[Dict[int, Tuple[TParameterization, TModelPredictArm]]]: """Identifies the best parameterizations tried in the experiment so far, using model predictions if ``use_model_predictions`` is true and using observed values from the experiment otherwise. By default, uses model predictions to account for observation noise. NOTE: The format of this method's output is as follows: { trial_index --> (parameterization, (means, covariances) }, where means are a dictionary of form { metric_name --> metric_mean } and covariances are a nested dictionary of form { one_metric_name --> { another_metric_name: covariance } }. Args: experiment: Experiment, from which to find Pareto-optimal arms. generation_strategy: Generation strategy containing the modelbridge. use_model_predictions: Whether to extract the Pareto frontier using model predictions or directly observed values. If ``True``, the metric means and covariances in this method's output will also be based on model predictions and may differ from the observed values. Returns: ``None`` if it was not possible to extract the Pareto frontier, otherwise a mapping from trial index to the tuple of: - the parameterization of the arm in that trial, - two-item tuple of metric means dictionary and covariance matrix (model-predicted if ``use_model_predictions=True`` and observed otherwise). """ # Validate aspects of the experiment: that it is a MOO experiment and # that the current model can be used to produce the Pareto frontier. if not not_none(experiment.optimization_config).is_moo_problem: raise UnsupportedError( "Please use `get_best_parameters` for single-objective problems.") moo_optimization_config = checked_cast(MultiObjectiveOptimizationConfig, experiment.optimization_config) if moo_optimization_config.outcome_constraints: # TODO[drfreund]: Test this flow and remove error. raise NotImplementedError( "Support for outcome constraints is currently under development.") # Extract or instantiate modelbridge to use for Pareto frontier extraction. mb = generation_strategy.model if mb is None or not isinstance(mb, MultiObjectiveTorchModelBridge): logger.info( "Can only extract a Pareto frontier using a multi-objective model bridge" f", but currently used model bridge is: {mb} of type {type(mb)}. Will " "use `Models.MOO` instead to extract Pareto frontier.") mb = checked_cast( MultiObjectiveTorchModelBridge, Models.MOO(experiment=experiment, data=checked_cast(Data, experiment.lookup_data())), ) else: # Make sure the model is up-to-date with the most recent data. generation_strategy._set_or_update_current_model(data=None) # If objective thresholds are not specified in optimization config, extract # the inferred ones if possible or infer them anew if not. objective_thresholds_override = None if not moo_optimization_config.objective_thresholds: lgr = generation_strategy.last_generator_run if lgr and lgr.gen_metadata and "objective_thresholds" in lgr.gen_metadata: objective_thresholds_override = lgr.gen_metadata[ "objective_thresholds"] objective_thresholds_override = mb.infer_objective_thresholds( search_space=experiment.search_space, optimization_config=experiment.optimization_config, fixed_features=None, ) logger.info( f"Using inferred objective thresholds: {objective_thresholds_override}, " "as objective thresholds were not specified as part of the optimization " "configuration on the experiment.") # Extract the Pareto frontier and format it as follows: # { trial_index --> (parameterization, (means, covariances) } pareto_util = predicted_pareto if use_model_predictions else observed_pareto pareto_optimal_observations = pareto_util( modelbridge=mb, objective_thresholds=objective_thresholds_override) return { int(not_none(obs.features.trial_index)): ( obs.features.parameters, (obs.data.means_dict, obs.data.covariance_matrix), ) for obs in pareto_optimal_observations }
def f(self, x: np.ndarray) -> float: x1, x2, fidelity = x fidelity_penalty = random() * math.pow(1.0 - fidelity, 2.0) return checked_cast(float, branin(x1=x1, x2=x2)) - fidelity_penalty
def f(self, x: np.ndarray) -> float: return checked_cast(float, aug_branin(x))
def gen( self, n: int, bounds: List[Tuple[float, float]], objective_weights: Tensor, # objective_directions outcome_constraints: Optional[Tuple[Tensor, Tensor]] = None, objective_thresholds: Optional[Tensor] = None, linear_constraints: Optional[Tuple[Tensor, Tensor]] = None, fixed_features: Optional[Dict[int, float]] = None, pending_observations: Optional[List[Tensor]] = None, model_gen_options: Optional[TConfig] = None, rounding_func: Optional[Callable[[Tensor], Tensor]] = None, target_fidelities: Optional[Dict[int, float]] = None, ) -> Tuple[Tensor, Tensor, TGenMetadata, Optional[List[TCandidateMetadata]]]: options = model_gen_options or {} acf_options = options.get("acquisition_function_kwargs", {}) optimizer_options = options.get("optimizer_kwargs", {}) if target_fidelities: raise NotImplementedError( "target_fidelities not implemented for base BotorchModel") if (objective_thresholds is not None and objective_weights.shape[0] != objective_thresholds.shape[0]): raise AxError( "Objective weights and thresholds most both contain an element for" " each modeled metric.") X_pending, X_observed = _get_X_pending_and_observed( Xs=self.Xs, pending_observations=pending_observations, objective_weights=objective_weights, outcome_constraints=outcome_constraints, bounds=bounds, linear_constraints=linear_constraints, fixed_features=fixed_features, ) model = not_none(self.model) full_objective_thresholds = objective_thresholds full_objective_weights = objective_weights full_outcome_constraints = outcome_constraints # subset model only to the outcomes we need for the optimization if options.get(Keys.SUBSET_MODEL, True): full_objective_weights subset_model_results = subset_model( model=model, objective_weights=objective_weights, outcome_constraints=outcome_constraints, objective_thresholds=objective_thresholds, ) model = subset_model_results.model objective_weights = subset_model_results.objective_weights outcome_constraints = subset_model_results.outcome_constraints objective_thresholds = subset_model_results.objective_thresholds idcs = subset_model_results.indices else: idcs = None if objective_thresholds is None: full_objective_thresholds = infer_objective_thresholds( model=model, X_observed=not_none(X_observed), objective_weights=full_objective_weights, outcome_constraints=full_outcome_constraints, subset_idcs=idcs, ) # subset the objective thresholds objective_thresholds = (full_objective_thresholds if idcs is None else full_objective_thresholds[idcs].clone()) bounds_ = torch.tensor(bounds, dtype=self.dtype, device=self.device) bounds_ = bounds_.transpose(0, 1) botorch_rounding_func = get_rounding_func(rounding_func) if acf_options.get("random_scalarization", False) or acf_options.get( "chebyshev_scalarization", False): # If using a list of acquisition functions, the algorithm to generate # that list is configured by acquisition_function_kwargs. objective_weights_list = [ randomize_objective_weights(objective_weights, **acf_options) for _ in range(n) ] acquisition_function_list = [ self.acqf_constructor( # pyre-ignore: [28] model=model, objective_weights=objective_weights, outcome_constraints=outcome_constraints, X_observed=X_observed, X_pending=X_pending, **acf_options, ) for objective_weights in objective_weights_list ] acquisition_function_list = [ checked_cast(AcquisitionFunction, acq_function) for acq_function in acquisition_function_list ] # Multiple acquisition functions require a sequential optimizer # always use scipy_optimizer_list. # TODO(jej): Allow any optimizer. candidates, expected_acquisition_value = scipy_optimizer_list( acq_function_list=acquisition_function_list, bounds=bounds_, inequality_constraints=_to_inequality_constraints( linear_constraints=linear_constraints), fixed_features=fixed_features, rounding_func=botorch_rounding_func, **optimizer_options, ) else: acquisition_function = self.acqf_constructor( # pyre-ignore: [28] model=model, objective_weights=objective_weights, objective_thresholds=objective_thresholds, outcome_constraints=outcome_constraints, X_observed=X_observed, X_pending=X_pending, **acf_options, ) acquisition_function = checked_cast(AcquisitionFunction, acquisition_function) # pyre-ignore: [28] candidates, expected_acquisition_value = self.acqf_optimizer( acq_function=checked_cast(AcquisitionFunction, acquisition_function), bounds=bounds_, n=n, inequality_constraints=_to_inequality_constraints( linear_constraints=linear_constraints), fixed_features=fixed_features, rounding_func=botorch_rounding_func, **optimizer_options, ) gen_metadata = { "expected_acquisition_value": expected_acquisition_value.tolist(), "objective_thresholds": not_none(full_objective_thresholds).cpu(), } return ( candidates.detach().cpu(), torch.ones(n, dtype=self.dtype), gen_metadata, None, )
def get_standard_plots( experiment: Experiment, model: Optional[ModelBridge], data: Optional[Union[Data, MapData]] = None, model_transitions: Optional[List[int]] = None, ) -> List[go.Figure]: """Extract standard plots for single-objective optimization. Extracts a list of plots from an ``Experiment`` and ``ModelBridge`` of general interest to an Ax user. Currently not supported are - TODO: multi-objective optimization - TODO: ChoiceParameter plots Args: - experiment: The ``Experiment`` from which to obtain standard plots. - model: The ``ModelBridge`` used to suggest trial parameters. - data: If specified, data, to which to fit the model before generating plots. - model_transitions: The arm numbers at which shifts in generation_strategy occur. Returns: - a plot of objective value vs. trial index, to show experiment progression - a plot of objective value vs. range parameter values, only included if the model associated with generation_strategy can create predictions. This consists of: - a plot_slice plot if the search space contains one range parameter - an interact_contour plot if the search space contains multiple range parameters """ objective = not_none(experiment.optimization_config).objective if isinstance(objective, ScalarizedObjective): logger.warning( "get_standard_plots does not currently support ScalarizedObjective " "optimization experiments. Returning an empty list.") return [] if data is None: data = experiment.lookup_data() if isinstance(data, MapData): data = data.deduplicate_data() if data.df.empty: logger.info( f"Experiment {experiment} does not yet have data, nothing to plot." ) return [] output_plot_list = [] output_plot_list.append( _get_objective_trace_plot( experiment=experiment, data=checked_cast(Data, data) if isinstance(data, Data) else checked_cast(MapData, data), model_transitions=model_transitions if model_transitions is not None else [], )) # Objective vs. parameter plot requires a `Model`, so add it only if model # is alrady available. In cases where initially custom trials are attached, # model might not yet be set on the generation strategy. if model: # TODO: Check if model can predict in favor of try/catch. try: output_plot_list.extend( _get_objective_v_param_plots( experiment=experiment, model=model, )) output_plot_list.extend(_get_cross_validation_plots(model=model)) feature_importance_plot = plot_feature_importance_by_feature_plotly( model=model, relative=False, caption=feature_importance_caption) feature_importance_plot.layout.title = "[ADVANCED] " + str( # pyre-fixme[16]: go.Figure has no attribute `layout` feature_importance_plot.layout.title.text) output_plot_list.append(feature_importance_plot) except NotImplementedError: # Model does not implement `predict` method. pass return [plot for plot in output_plot_list if plot is not None]
def get_factorial(search_space: SearchSpace) -> DiscreteModelBridge: """Instantiates a factorial generator.""" return checked_cast( DiscreteModelBridge, Models.FACTORIAL(search_space=search_space, fit_out_of_design=True), )
def f(self, x: np.ndarray) -> float: x1, x2 = x return checked_cast(float, branin(x1=x1, x2=x2))
def get_MTGP_PAREGO( experiment: Experiment, data: Data, trial_index: Optional[int] = None, objective_thresholds: Optional[TRefPoint] = None, search_space: Optional[SearchSpace] = None, dtype: torch.dtype = torch.double, device: torch.device = DEFAULT_TORCH_DEVICE, ) -> MultiObjectiveTorchModelBridge: """Instantiates a multi-objective, multi-task model that uses qParEGO. qParEGO optimizes random augmented chebyshev scalarizations of the multiple objectives. This allows it to explore non-convex pareto frontiers. """ # pyre-ignore: [16] `Optional` has no attribute `objective`. if not isinstance(experiment.optimization_config.objective, MultiObjective): raise ValueError("Multi-objective optimization requires multiple objectives.") elif data.df.empty: # pragma: no cover raise ValueError("MultiObjectiveOptimization requires non-empty data.") if isinstance(experiment, MultiTypeExperiment): trial_index_to_type = { t.index: t.trial_type for t in experiment.trials.values() } transforms = MT_MTGP_trans transform_configs = { "ConvertMetricNames": tconfig_from_mt_experiment(experiment), "TrialAsTask": {"trial_level_map": {"trial_type": trial_index_to_type}}, } else: # Set transforms for a Single-type MTGP model. transforms = ST_MTGP_trans transform_configs = None # Choose the status quo features for the experiment from the selected trial. # If trial_index is None, we will look for a status quo from the last # experiment trial to use as a status quo for the experiment. if trial_index is None: trial_index = len(experiment.trials) - 1 elif trial_index >= len(experiment.trials): raise ValueError("trial_index is bigger than the number of experiment trials") # pyre-fixme[16]: `ax.core.base_trial.BaseTrial` has no attribute `status_quo`. status_quo = experiment.trials[trial_index].status_quo if status_quo is None: status_quo_features = None else: status_quo_features = ObservationFeatures( parameters=status_quo.parameters, # pyre-fixme[6]: Expected `Optional[numpy.int64]` for 2nd param but got # `int`. trial_index=trial_index, ) return checked_cast( MultiObjectiveTorchModelBridge, Models.MOO( experiment=experiment, data=data, objective_thresholds=objective_thresholds, search_space=search_space or experiment.search_space, torch_dtype=dtype, torch_device=device, acqf_constructor=get_NEI, status_quo_features=status_quo_features, transforms=transforms, transform_configs=transform_configs, default_model_gen_options={ "acquisition_function_kwargs": { "chebyshev_scalarization": True, "sequential": True, } }, ), )
def get_MTGP_NEHVI( experiment: Experiment, data: Data, objective_thresholds: Optional[List[ObjectiveThreshold]] = None, search_space: Optional[SearchSpace] = None, dtype: torch.dtype = torch.double, device: torch.device = DEFAULT_TORCH_DEVICE, trial_index: Optional[int] = None, ) -> TorchModelBridge: """Instantiates a Multi-task Gaussian Process (MTGP) model that generates points with qNEHVI. If the input experiment is a MultiTypeExperiment then a Multi-type Multi-task GP model will be instantiated. Otherwise, the model will be a Single-type Multi-task GP. """ # pyre-ignore: [16] `Optional` has no attribute `objective`. if not isinstance(experiment.optimization_config.objective, MultiObjective): raise ValueError("Multi-objective optimization requires multiple objectives.") elif data.df.empty: # pragma: no cover raise ValueError("MultiObjectiveOptimization requires non-empty data.") if isinstance(experiment, MultiTypeExperiment): trial_index_to_type = { t.index: t.trial_type for t in experiment.trials.values() } transforms = MT_MTGP_trans transform_configs = { "ConvertMetricNames": tconfig_from_mt_experiment(experiment), "TrialAsTask": {"trial_level_map": {"trial_type": trial_index_to_type}}, } else: # Set transforms for a Single-type MTGP model. transforms = ST_MTGP_trans transform_configs = None # Choose the status quo features for the experiment from the selected trial. # If trial_index is None, we will look for a status quo from the last # experiment trial to use as a status quo for the experiment. if trial_index is None: trial_index = len(experiment.trials) - 1 elif trial_index >= len(experiment.trials): raise ValueError("trial_index is bigger than the number of experiment trials") # pyre-fixme[16]: `ax.core.base_trial.BaseTrial` has no attribute `status_quo`. status_quo = experiment.trials[trial_index].status_quo if status_quo is None: status_quo_features = None else: status_quo_features = ObservationFeatures( parameters=status_quo.parameters, # pyre-fixme[6]: Expected `Optional[numpy.int64]` for 2nd param but got # `int`. trial_index=trial_index, ) return checked_cast( MultiObjectiveTorchModelBridge, Models.MOO( experiment=experiment, data=data, objective_thresholds=objective_thresholds, search_space=search_space or experiment.search_space, transforms=transforms, transform_configs=transform_configs, torch_dtype=dtype, torch_device=device, status_quo_features=status_quo_features, default_model_gen_options={ "optimizer_kwargs": { # having a batch limit is very important for avoiding # memory issues in the initialization "batch_limit": DEFAULT_EHVI_BATCH_LIMIT, "sequential": True, }, }, ), )
def _get_cutoffs_from_transform_config( metric_name: str, metric_values: List[float], winsorization_config: Union[WinsorizationConfig, Dict[str, WinsorizationConfig]], optimization_config: Optional[OptimizationConfig], ) -> Tuple[float, float]: # (1) Use the same config for all metrics if one WinsorizationConfig was specified if isinstance(winsorization_config, WinsorizationConfig): return _quantiles_to_cutoffs( metric_name=metric_name, metric_values=metric_values, metric_config=winsorization_config, ) # (2) If `winsorization_config` is a dict, use it if `metric_name` is a key, # and the corresponding value is a WinsorizationConfig. if isinstance(winsorization_config, dict) and metric_name in winsorization_config: metric_config = winsorization_config[metric_name] if not isinstance(metric_config, WinsorizationConfig): raise UserInputError( "Expected winsorization config of type " f"`WinsorizationConfig` but got {metric_config} of type " f"{type(metric_config)} for metric {metric_name}.") return _quantiles_to_cutoffs( metric_name=metric_name, metric_values=metric_values, metric_config=metric_config, ) # (3) For constraints and objectives that don't have a pre-specified config we # choose the cutoffs automatically using the optimization config (if supplied). # We ignore ScalarizedOutcomeConstraint and ScalarizedObjective for now. An # exception is raised if we encounter relative constraints. if optimization_config: if metric_name in optimization_config.objective.metric_names: if isinstance(optimization_config.objective, ScalarizedObjective): warnings.warn( "Automatic winsorization isn't supported for ScalarizedObjective. " "Specify the winsorization settings manually if you want to " f"winsorize metric {metric_name}.") return DEFAULT_CUTOFFS # Don't winsorize a ScalarizedObjective elif optimization_config.is_moo_problem: # We deal with a multi-objective function the same way as we deal # with an output constraint. It may be worth investigating setting # the winsorization cutoffs based on the Pareto frontier in the future. optimization_config = checked_cast( MultiObjectiveOptimizationConfig, optimization_config) objective_threshold = _get_objective_threshold_from_moo_config( optimization_config=optimization_config, metric_name=metric_name) if objective_threshold: return _get_auto_winsorization_cutoffs_outcome_constraint( metric_values=metric_values, outcome_constraints=objective_threshold, ) warnings.warn( "Automatic winsorization isn't supported for an objective in " "`MultiObjective` without objective thresholds. Specify the " "winsorization settings manually if you want to winsorize " f"metric {metric_name}.") return DEFAULT_CUTOFFS # Don't winsorize if there is no threshold else: # Single objective return _get_auto_winsorization_cutoffs_single_objective( metric_values=metric_values, minimize=optimization_config.objective.minimize, ) # Get all outcome constraints for metric_name that aren't relative or scalarized outcome_constraints = _get_outcome_constraints_from_config( optimization_config=optimization_config, metric_name=metric_name) if outcome_constraints: return _get_auto_winsorization_cutoffs_outcome_constraint( metric_values=metric_values, outcome_constraints=outcome_constraints, ) # If none of the above, we don't winsorize. return DEFAULT_CUTOFFS
def get_trial_parameters(self, trial_index: int) -> TParameterization: """Retrieve the parameterization of the trial by the given index.""" if trial_index not in self.experiment.trials: raise ValueError(f"Trial {trial_index} does not yet exist.") trial = checked_cast(Trial, self.experiment.trials.get(trial_index)) return not_none(trial.arm).parameters
def test_default_generation_strategy(self) -> None: """Test that Sobol+GPEI is used if no GenerationStrategy is provided.""" ax_client = AxClient() ax_client.create_experiment( parameters=[ # pyre-fixme[6]: expected union that should include { "name": "x1", "type": "range", "bounds": [-5.0, 10.0] }, { "name": "x2", "type": "range", "bounds": [0.0, 15.0] }, ], objective_name="branin", minimize=True, ) self.assertEqual( [s.model for s in not_none(ax_client.generation_strategy)._steps], [Models.SOBOL, Models.GPEI], ) with self.assertRaisesRegex(ValueError, ".* no trials."): ax_client.get_optimization_trace(objective_optimum=branin.fmin) for i in range(6): parameterization, trial_index = ax_client.get_next_trial() x1, x2 = parameterization.get("x1"), parameterization.get("x2") ax_client.complete_trial( trial_index, raw_data={ "branin": ( checked_cast( float, branin(checked_cast(float, x1), checked_cast(float, x2)), ), 0.0, ) }, sample_size=i, ) if i < 5: with self.assertRaisesRegex(ValueError, "Could not obtain contour"): ax_client.get_contour_plot(param_x="x1", param_y="x2") ax_client.get_optimization_trace(objective_optimum=branin.fmin) ax_client.get_contour_plot() self.assertIn("x1", ax_client.get_trials_data_frame()) self.assertIn("x2", ax_client.get_trials_data_frame()) self.assertIn("branin", ax_client.get_trials_data_frame()) self.assertEqual(len(ax_client.get_trials_data_frame()), 6) # Test that Sobol is chosen when all parameters are choice. ax_client = AxClient() ax_client.create_experiment( parameters=[ # pyre-fixme[6]: expected union that should include { "name": "x1", "type": "choice", "values": [1, 2, 3] }, { "name": "x2", "type": "choice", "values": [1, 2, 3] }, ]) self.assertEqual( [s.model for s in not_none(ax_client.generation_strategy)._steps], [Models.SOBOL], ) self.assertEqual(ax_client.get_recommended_max_parallelism(), [(-1, -1)]) self.assertTrue(ax_client.get_trials_data_frame().empty)
def test_early_stopping_with_unaligned_results(self): # test case 1 exp = get_branin_experiment_with_timestamp_map_metric(rate=0.5) for i in range(5): trial = exp.new_trial().add_arm( arm=get_branin_arms(n=1, seed=i)[0]) trial.run() for _ in range(3): # each time we call fetch, we grab another timestamp exp.fetch_data() for trial in exp.trials.values(): trial.mark_as(status=TrialStatus.COMPLETED) # manually "unalign" timestamps to simulate real-world scenario # where each curve reports results at different steps data = checked_cast(MapData, exp.fetch_data()) unaligned_timestamps = [0, 1, 4, 1, 2, 3, 1, 3, 4, 0, 1, 2, 0, 2, 4] data.map_df.loc[data.map_df["metric_name"] == "branin_map", "timestamp"] = unaligned_timestamps exp.attach_data(data=data) """ Dataframe after interpolation: 0 1 2 3 4 timestamp 0 146.138620 NaN NaN 143.375669 65.033535 1 117.388086 113.057480 44.627226 115.168704 58.636359 2 111.575393 90.815154 40.237365 98.060315 52.239184 3 105.762700 77.324501 35.847504 NaN 48.359101 4 99.950007 NaN 30.522333 NaN 44.479018 """ # We consider trials 0, 2, and 4 for early stopping at progression 4, # and choose to stop trial 0. # We consider trial 1 for early stopping at progression 3, and # choose to stop it. # We consider trial 3 for early stopping at progression 2, and # choose to stop it. early_stopping_strategy = PercentileEarlyStoppingStrategy( percentile_threshold=50, min_curves=3, ) should_stop = _evaluate_early_stopping_with_df( early_stopping_strategy=early_stopping_strategy, experiment=exp, df=data.map_df, ) self.assertEqual(set(should_stop), {0, 1, 3}) # test case 2, where trial 3 has only 1 data point exp = get_branin_experiment_with_timestamp_map_metric(rate=0.5) for i in range(5): trial = exp.new_trial().add_arm( arm=get_branin_arms(n=1, seed=i)[0]) trial.run() for _ in range(3): # each time we call fetch, we grab another timestamp exp.fetch_data() for trial in exp.trials.values(): trial.mark_as(status=TrialStatus.COMPLETED) # manually "unalign" timestamps to simulate real-world scenario # where each curve reports results at different steps data = checked_cast(MapData, exp.fetch_data()) unaligned_timestamps = [0, 1, 4, 1, 2, 3, 1, 3, 4, 0, 1, 2, 0, 2, 4] data.map_df.loc[data.map_df["metric_name"] == "branin_map", "timestamp"] = unaligned_timestamps # manually remove timestamps 1 and 2 for arm 3 data.map_df.drop( [15, 16], inplace=True ) # TODO this wont work once we make map_df immutable (which we should) exp.attach_data(data=data) """ Dataframe after interpolation: 0 1 2 3 4 timestamp 0 146.138620 NaN NaN 143.375669 65.033535 1 117.388086 113.057480 44.627226 NaN 58.636359 2 111.575393 90.815154 40.237365 NaN 52.239184 3 105.762700 77.324501 35.847504 NaN 48.359101 4 99.950007 NaN 30.522333 NaN 44.479018 """ # We consider trials 0, 2, and 4 for early stopping at progression 4, # and choose to stop trial 0. # We consider trial 1 for early stopping at progression 3, and # choose to stop it. # We consider trial 3 for early stopping at progression 0, and # choose not to stop it. early_stopping_strategy = PercentileEarlyStoppingStrategy( percentile_threshold=50, min_curves=3, ) should_stop = _evaluate_early_stopping_with_df( early_stopping_strategy=early_stopping_strategy, experiment=exp, df=data.map_df, ) self.assertEqual(set(should_stop), {0, 1})
def gen( self, n: int, bounds: List[Tuple[float, float]], objective_weights: Tensor, outcome_constraints: Optional[Tuple[Tensor, Tensor]] = None, linear_constraints: Optional[Tuple[Tensor, Tensor]] = None, fixed_features: Optional[Dict[int, float]] = None, pending_observations: Optional[List[Tensor]] = None, model_gen_options: Optional[TConfig] = None, rounding_func: Optional[Callable[[Tensor], Tensor]] = None, target_fidelities: Optional[Dict[int, float]] = None, ) -> Tuple[Tensor, Tensor, TGenMetadata, Optional[List[TCandidateMetadata]]]: options = model_gen_options or {} acf_options = options.get(Keys.ACQF_KWARGS, {}) optimizer_options = options.get(Keys.OPTIMIZER_KWARGS, {}) if target_fidelities: raise NotImplementedError( "target_fidelities not implemented for base BotorchModel" ) X_pending, X_observed = _get_X_pending_and_observed( Xs=self.Xs, pending_observations=pending_observations, objective_weights=objective_weights, outcome_constraints=outcome_constraints, bounds=bounds, linear_constraints=linear_constraints, fixed_features=fixed_features, ) model = self.model # subset model only to the outcomes we need for the optimization 357 if options.get(Keys.SUBSET_MODEL, True): model, objective_weights, outcome_constraints, _ = subset_model( model=model, # pyre-ignore [6] objective_weights=objective_weights, outcome_constraints=outcome_constraints, ) bounds_ = torch.tensor(bounds, dtype=self.dtype, device=self.device) bounds_ = bounds_.transpose(0, 1) botorch_rounding_func = get_rounding_func(rounding_func) acquisition_function = self.acqf_constructor( # pyre-ignore: [28] model=model, objective_weights=objective_weights, outcome_constraints=outcome_constraints, X_observed=X_observed, X_pending=X_pending, **acf_options, ) acquisition_function = checked_cast(AcquisitionFunction, acquisition_function) # pyre-ignore: [28] candidates, expected_acquisition_value = self.acqf_optimizer( acq_function=checked_cast(AcquisitionFunction, acquisition_function), bounds=bounds_, n=n, inequality_constraints=_to_inequality_constraints( linear_constraints=linear_constraints ), fixed_features=fixed_features, rounding_func=botorch_rounding_func, **optimizer_options, ) return ( candidates.detach().cpu(), torch.ones(n, dtype=self.dtype), {"expected_acquisition_value": expected_acquisition_value.tolist()}, None, )
def test_checked_cast(self): self.assertEqual(checked_cast(float, 2.0), 2.0) with self.assertRaises(ValueError): checked_cast(float, 2)
def gen( self, n: int, bounds: List[Tuple[float, float]], objective_weights: Tensor, outcome_constraints: Optional[Tuple[Tensor, Tensor]] = None, linear_constraints: Optional[Tuple[Tensor, Tensor]] = None, fixed_features: Optional[Dict[int, float]] = None, pending_observations: Optional[List[Tensor]] = None, model_gen_options: Optional[TConfig] = None, rounding_func: Optional[Callable[[Tensor], Tensor]] = None, target_fidelities: Optional[Dict[int, float]] = None, ) -> Tuple[Tensor, Tensor, TGenMetadata]: options = model_gen_options or {} acf_options = options.get("acquisition_function_kwargs", {}) optimizer_options = options.get("optimizer_kwargs", {}) if target_fidelities: raise NotImplementedError( "target_fidelities not implemented for base BotorchModel") X_pending, X_observed = _get_X_pending_and_observed( Xs=self.Xs, pending_observations=pending_observations, objective_weights=objective_weights, outcome_constraints=outcome_constraints, bounds=bounds, linear_constraints=linear_constraints, fixed_features=fixed_features, ) model = self.model # subset model only to the outcomes we need for the optimization if options.get("subset_model", True): model, objective_weights, outcome_constraints = subset_model( model=model, # pyre-ignore [6] objective_weights=objective_weights, outcome_constraints=outcome_constraints, ) bounds_ = torch.tensor(bounds, dtype=self.dtype, device=self.device) bounds_ = bounds_.transpose(0, 1) if linear_constraints is not None: A, b = linear_constraints inequality_constraints = [] k, d = A.shape for i in range(k): indicies = A[i, :].nonzero().view(-1) coefficients = -A[i, indicies] rhs = -b[i, 0] inequality_constraints.append((indicies, coefficients, rhs)) else: inequality_constraints = None acquisition_function = self.acqf_constructor( # pyre-ignore: [28] model=model, objective_weights=objective_weights, outcome_constraints=outcome_constraints, X_observed=X_observed, X_pending=X_pending, **acf_options, ) botorch_rounding_func = get_rounding_func(rounding_func) # pyre-ignore: [28] candidates, expected_acquisition_value = self.acqf_optimizer( acq_function=checked_cast(AcquisitionFunction, acquisition_function), bounds=bounds_, n=n, inequality_constraints=inequality_constraints, fixed_features=fixed_features, rounding_func=botorch_rounding_func, **optimizer_options, ) return ( candidates.detach().cpu(), torch.ones(n, dtype=self.dtype), { "expected_acquisition_value": expected_acquisition_value.tolist() }, )
def gen( self, n: int, bounds: List[Tuple[float, float]], objective_weights: Tensor, # objective_directions outcome_constraints: Optional[Tuple[Tensor, Tensor]] = None, objective_thresholds: Optional[Tensor] = None, linear_constraints: Optional[Tuple[Tensor, Tensor]] = None, fixed_features: Optional[Dict[int, float]] = None, pending_observations: Optional[List[Tensor]] = None, model_gen_options: Optional[TConfig] = None, rounding_func: Optional[Callable[[Tensor], Tensor]] = None, target_fidelities: Optional[Dict[int, float]] = None, ) -> Tuple[Tensor, Tensor, TGenMetadata, Optional[List[TCandidateMetadata]]]: options = model_gen_options or {} acf_options = options.get("acquisition_function_kwargs", {}) optimizer_options = options.get("optimizer_kwargs", {}) if target_fidelities: raise NotImplementedError( "target_fidelities not implemented for base BotorchModel") X_pending, X_observed = _get_X_pending_and_observed( Xs=self.Xs, pending_observations=pending_observations, objective_weights=objective_weights, outcome_constraints=outcome_constraints, bounds=bounds, linear_constraints=linear_constraints, fixed_features=fixed_features, ) model = self.model # subset model only to the outcomes we need for the optimization if options.get(Keys.SUBSET_MODEL, True): model, objective_weights, outcome_constraints, Ys = subset_model( model=model, # pyre-ignore [6] objective_weights=objective_weights, outcome_constraints=outcome_constraints, Ys=self.Ys, ) else: Ys = self.Ys bounds_ = torch.tensor(bounds, dtype=self.dtype, device=self.device) bounds_ = bounds_.transpose(0, 1) botorch_rounding_func = get_rounding_func(rounding_func) if acf_options.get("random_scalarization", False) or acf_options.get( "chebyshev_scalarization", False): # If using a list of acquisition functions, the algorithm to generate # that list is configured by acquisition_function_kwargs. objective_weights_list = [ randomize_objective_weights(objective_weights, **acf_options) for _ in range(n) ] acquisition_function_list = [ self.acqf_constructor( # pyre-ignore: [28] model=model, objective_weights=objective_weights, outcome_constraints=outcome_constraints, X_observed=X_observed, X_pending=X_pending, Ys=Ys, # Required for chebyshev scalarization calculations. **acf_options, ) for objective_weights in objective_weights_list ] acquisition_function_list = [ checked_cast(AcquisitionFunction, acq_function) for acq_function in acquisition_function_list ] # Multiple acquisition functions require a sequential optimizer # always use scipy_optimizer_list. # TODO(jej): Allow any optimizer. candidates, expected_acquisition_value = scipy_optimizer_list( acq_function_list=acquisition_function_list, bounds=bounds_, inequality_constraints=_to_inequality_constraints( linear_constraints=linear_constraints), fixed_features=fixed_features, rounding_func=botorch_rounding_func, **optimizer_options, ) else: acquisition_function = self.acqf_constructor( # pyre-ignore: [28] model=model, objective_weights=objective_weights, objective_thresholds=objective_thresholds, outcome_constraints=outcome_constraints, X_observed=X_observed, X_pending=X_pending, Ys=self.Ys, # Required for qEHVI calculations. **acf_options, ) acquisition_function = checked_cast(AcquisitionFunction, acquisition_function) # pyre-ignore: [28] candidates, expected_acquisition_value = self.acqf_optimizer( acq_function=checked_cast(AcquisitionFunction, acquisition_function), bounds=bounds_, n=n, inequality_constraints=_to_inequality_constraints( linear_constraints=linear_constraints), fixed_features=fixed_features, rounding_func=botorch_rounding_func, **optimizer_options, ) return ( candidates.detach().cpu(), torch.ones(n, dtype=self.dtype), { "expected_acquisition_value": expected_acquisition_value.tolist() }, None, )
def gen( self, n: int, bounds: List[Tuple[float, float]], objective_weights: Tensor, outcome_constraints: Optional[Tuple[Tensor, Tensor]] = None, linear_constraints: Optional[Tuple[Tensor, Tensor]] = None, fixed_features: Optional[Dict[int, float]] = None, pending_observations: Optional[List[Tensor]] = None, model_gen_options: Optional[TConfig] = None, rounding_func: Optional[Callable[[Tensor], Tensor]] = None, ) -> Tuple[Tensor, Tensor]: """Generate new candidates. An initialized acquisition function can be passed in as model_gen_options["acquisition_function"]. Args: n: Number of candidates to generate. bounds: A list of (lower, upper) tuples for each column of X. objective_weights: The objective is to maximize a weighted sum of the columns of f(x). These are the weights. outcome_constraints: A tuple of (A, b). For k outcome constraints and m outputs at f(x), A is (k x m) and b is (k x 1) such that A f(x) <= b. (Not used by single task models) linear_constraints: A tuple of (A, b). For k linear constraints on d-dimensional x, A is (k x d) and b is (k x 1) such that A x <= b. fixed_features: A map {feature_index: value} for features that should be fixed to a particular value during generation. pending_observations: A list of m (k_i x d) feature tensors X for m outcomes and k_i pending observations for outcome i. model_gen_options: A config dictionary that can contain model-specific options. rounding_func: A function that rounds an optimization result appropriately (i.e., according to `round-trip` transformations). Returns: Tensor: `n x d`-dim Tensor of generated points. Tensor: `n`-dim Tensor of weights for each point. """ options = model_gen_options or {} acf_options = options.get("acquisition_function_kwargs", {}) optimizer_options = options.get("optimizer_kwargs", {}) X_pending, X_observed = _get_X_pending_and_observed( Xs=self.Xs, pending_observations=pending_observations, objective_weights=objective_weights, outcome_constraints=outcome_constraints, bounds=bounds, linear_constraints=linear_constraints, fixed_features=fixed_features, ) acquisition_function = self.acqf_constructor( # pyre-ignore: [28] model=self.model, objective_weights=objective_weights, outcome_constraints=outcome_constraints, X_observed=X_observed, X_pending=X_pending, **acf_options, ) bounds_ = torch.tensor(bounds, dtype=self.dtype, device=self.device) bounds_ = bounds_.transpose(0, 1) if linear_constraints is not None: A, b = linear_constraints inequality_constraints = [] k, d = A.shape for i in range(k): indicies = A[i, :].nonzero().squeeze() coefficients = -A[i, indicies] rhs = -b[i, 0] inequality_constraints.append((indicies, coefficients, rhs)) else: inequality_constraints = None botorch_rounding_func = get_rounding_func(rounding_func) candidates = self.acqf_optimizer( # pyre-ignore: [28] acq_function=checked_cast(AcquisitionFunction, acquisition_function), bounds=bounds_, n=n, inequality_constraints=inequality_constraints, fixed_features=fixed_features, rounding_func=botorch_rounding_func, **optimizer_options, ) return candidates.detach().cpu(), torch.ones(n, dtype=self.dtype)
def run(self) -> "BatchTrial": return checked_cast(BatchTrial, super().run())
def f(self, x: np.ndarray) -> float: return checked_cast(float, aug_hartmann6(x))