def testIsFactorial(self): self.assertFalse(self.batch.is_factorial) # Insufficient factors small_experiment = Experiment( name="small_test", search_space=SearchSpace( [FixedParameter("a", ParameterType.INT, 4)]), ) small_trial = small_experiment.new_batch_trial().add_arm(Arm({"a": 4})) self.assertFalse(small_trial.is_factorial) new_batch_trial = self.experiment.new_batch_trial() new_batch_trial.add_arms_and_weights(arms=[ Arm(parameters={ "w": 0.75, "x": 1, "y": "foo", "z": True }), Arm(parameters={ "w": 0.75, "x": 2, "y": "foo", "z": True }), Arm(parameters={ "w": 0.77, "x": 1, "y": "foo", "z": True }), ]) self.assertFalse(new_batch_trial.is_factorial) new_batch_trial = self.experiment.new_batch_trial() new_batch_trial.add_arms_and_weights(arms=[ Arm(parameters={ "w": 0.77, "x": 1, "y": "foo", "z": True }), Arm(parameters={ "w": 0.77, "x": 2, "y": "foo", "z": True }), Arm(parameters={ "w": 0.75, "x": 1, "y": "foo", "z": True }), Arm(parameters={ "w": 0.75, "x": 2, "y": "foo", "z": True }), ]) self.assertTrue(new_batch_trial.is_factorial)
def experiment_from_json(object_json: Dict[str, Any]) -> Experiment: """Load Ax Experiment from JSON.""" time_created_json = object_json.pop("time_created") trials_json = object_json.pop("trials") experiment_type_json = object_json.pop("experiment_type") data_by_trial_json = object_json.pop("data_by_trial") experiment = Experiment( **{k: object_from_json(v) for k, v in object_json.items()}) experiment._time_created = object_from_json(time_created_json) experiment._trials = trials_from_json(experiment, trials_json) experiment._arms_by_name = {} for trial in experiment._trials.values(): for arm in trial.arms: experiment._register_arm(arm) if trial.ttl_seconds is not None: experiment._trials_have_ttl = True if experiment.status_quo is not None: sq = not_none(experiment.status_quo) experiment._register_arm(sq) experiment._experiment_type = object_from_json(experiment_type_json) experiment._data_by_trial = data_from_json(data_by_trial_json) return experiment
def _benchmark_replication_Async_Scheduler( problem: BenchmarkProblem, method: GenerationStrategy, num_trials: int, experiment_name: str, batch_size: int = 1, raise_all_exceptions: bool = False, benchmark_trial: FunctionType = benchmark_trial, verbose_logging: bool = True, # Number of trials that need to fail for a replication to be considered failed. failed_trials_tolerated: int = 5, async_benchmark_options: Optional[AsyncBenchmarkOptions] = None, ) -> Tuple[Experiment, List[Exception]]: """Run a benchmark replication with asynchronous evaluations through Scheduler. The Scheduler interacts with a BackendSimulator. """ if async_benchmark_options is None: raise NonRetryableBenchmarkingError( "`async_benchmark_options` required for Scheduler benchmarks." ) backend_options = ( async_benchmark_options.backend_options or BackendSimulatorOptions( internal_clock=0.0, max_concurrency=async_benchmark_options.max_pending_trials, ) ) backend_simulator = BackendSimulator( options=backend_options, verbose_logging=verbose_logging ) experiment = Experiment( name=experiment_name, search_space=problem.search_space, optimization_config=problem.optimization_config, runner=SimulatedBackendRunner( simulator=backend_simulator, sample_runtime_func=async_benchmark_options.sample_runtime_func, ), ) scheduler_options = async_benchmark_options.scheduler_options or SchedulerOptions( total_trials=None, init_seconds_between_polls=1, min_seconds_before_poll=1.0, seconds_between_polls_backoff_factor=1.0, logging_level=logging.INFO if verbose_logging else logging.WARNING, early_stopping_strategy=async_benchmark_options.early_stopping_strategy, ) scheduler = AsyncSimulatedBackendScheduler( experiment=experiment, generation_strategy=method, max_pending_trials=async_benchmark_options.max_pending_trials, options=scheduler_options, ) scheduler.run_n_trials( max_trials=num_trials, timeout_hours=async_benchmark_options.timeout_hours ) # update the trial metadata with start time # Note: we could also do it in the BackendSimulator if it got access to the Trial for sim_trial in backend_simulator._completed: metadata_dict = { "start_time": sim_trial.sim_start_time, "queued_time": sim_trial.sim_queued_time, "completed_time": sim_trial.sim_completed_time, } experiment.trials[sim_trial.trial_index].update_run_metadata(metadata_dict) return experiment, []
def get_best_from_model_predictions_with_trial_index( experiment: Experiment, ) -> Optional[Tuple[int, TParameterization, Optional[TModelPredictArm]]]: """Given an experiment, returns the best predicted parameterization and corresponding prediction based on the most recent Trial with predictions. If no trials have predictions returns None. Only some models return predictions. For instance GPEI does while Sobol does not. TModelPredictArm is of the form: ({metric_name: mean}, {metric_name_1: {metric_name_2: cov_1_2}}) Args: experiment: Experiment, on which to identify best raw objective arm. Returns: Tuple of parameterization and model predictions for it. """ # pyre-ignore [16] if isinstance(experiment.optimization_config.objective, MultiObjective): logger.warning( "get_best_from_model_predictions is deprecated for multi-objective " "optimization configs. This method will return an arbitrary point on " "the pareto frontier.") for idx, trial in sorted(experiment.trials.items(), key=lambda x: x[0], reverse=True): gr = None if isinstance(trial, Trial): gr = trial.generator_run elif isinstance(trial, BatchTrial): if len(trial.generator_run_structs) > 0: # In theory batch_trial can have >1 gr, grab the first gr = trial.generator_run_structs[0].generator_run if gr is not None and gr.best_arm_predictions is not None: # pragma: no cover data = experiment.lookup_data() if not isinstance(data, Data): return _gr_to_prediction_with_trial_index(idx, gr) model = get_model_from_generator_run(generator_run=gr, experiment=experiment, data=data) # If model is not ArrayModelBridge, just use the best arm frmo the # last good generator run if not isinstance(model, ArrayModelBridge): return _gr_to_prediction_with_trial_index(idx, gr) # Check to see if the model is worth using cv_results = cross_validate(model=model) diagnostics = compute_diagnostics(result=cv_results) assess_model_fit_results = assess_model_fit( diagnostics=diagnostics) objective_name = experiment.optimization_config.objective.metric.name # If model fit is bad use raw results if (objective_name in assess_model_fit_results.bad_fit_metrics_to_fisher_score): logger.warning( "Model fit is poor; falling back on raw data for best point." ) if not _is_all_noiseless(df=data.df, metric_name=objective_name): logger.warning( "Model fit is poor and data on objective metric " + f"{objective_name} is noisy; interpret best points " + "results carefully.") return _get_best_poor_model_fit(experiment=experiment) res = model.model_best_point() if res is None: return _gr_to_prediction_with_trial_index(idx, gr) best_arm, best_arm_predictions = res return idx, not_none(best_arm).parameters, best_arm_predictions return None
def _init_experiment_from_sqa(self, experiment_sqa: SQAExperiment) -> Experiment: """First step of conversion within experiment_from_sqa.""" opt_config, tracking_metrics = self.opt_config_and_tracking_metrics_from_sqa( metrics_sqa=experiment_sqa.metrics ) search_space = self.search_space_from_sqa( parameters_sqa=experiment_sqa.parameters, parameter_constraints_sqa=experiment_sqa.parameter_constraints, ) if search_space is None: raise SQADecodeError( # pragma: no cover "Experiment SearchSpace cannot be None." ) status_quo = ( Arm( # pyre-fixme[6]: Expected `Dict[str, Optional[Union[bool, float, # int, str]]]` for 1st param but got `Optional[Dict[str, # Optional[Union[bool, float, int, str]]]]`. parameters=experiment_sqa.status_quo_parameters, name=experiment_sqa.status_quo_name, ) if experiment_sqa.status_quo_parameters is not None else None ) if len(experiment_sqa.runners) == 0: runner = None elif len(experiment_sqa.runners) == 1: runner = self.runner_from_sqa(experiment_sqa.runners[0]) else: raise ValueError( # pragma: no cover "Multiple runners on experiment " "only supported for MultiTypeExperiment." ) subclass = (experiment_sqa.properties or {}).get("subclass") if subclass == "SimpleExperiment": if opt_config is None: raise SQADecodeError( # pragma: no cover "SimpleExperiment must have an optimization config." ) experiment = SimpleExperiment( name=experiment_sqa.name, search_space=search_space, objective_name=opt_config.objective.metric.name, minimize=opt_config.objective.minimize, outcome_constraints=opt_config.outcome_constraints, status_quo=status_quo, ) experiment.description = experiment_sqa.description experiment.is_test = experiment_sqa.is_test else: experiment = Experiment( name=experiment_sqa.name, description=experiment_sqa.description, search_space=search_space, optimization_config=opt_config, tracking_metrics=tracking_metrics, runner=runner, status_quo=status_quo, is_test=experiment_sqa.is_test, ) return experiment
def compute_pareto_frontier( experiment: Experiment, primary_objective: Metric, secondary_objective: Metric, data: Optional[Data] = None, outcome_constraints: Optional[List[OutcomeConstraint]] = None, absolute_metrics: Optional[List[str]] = None, num_points: int = 10, trial_index: Optional[int] = None, chebyshev: bool = True, ) -> ParetoFrontierResults: """Compute the Pareto frontier between two objectives. For experiments with batch trials, a trial index or data object must be provided. Args: experiment: The experiment to compute a pareto frontier for. primary_objective: The primary objective to optimize. secondary_objective: The secondary objective against which to trade off the primary objective. outcome_constraints: Outcome constraints to be respected by the optimization. Can only contain constraints on metrics that are not primary or secondary objectives. absolute_metrics: List of outcome metrics that should NOT be relativized w.r.t. the status quo (all other outcomes will be in % relative to status_quo). num_points: The number of points to compute on the Pareto frontier. chebyshev: Whether to use augmented_chebyshev_scalarization when computing Pareto Frontier points. Returns: ParetoFrontierResults: A NamedTuple with the following fields: - param_dicts: The parameter dicts of the points generated on the Pareto Frontier. - means: The posterior mean predictions of the model for each metric (same order as the param dicts). - sems: The posterior sem predictions of the model for each metric (same order as the param dicts). - primary_metric: The name of the primary metric. - secondary_metric: The name of the secondary metric. - absolute_metrics: List of outcome metrics that are NOT be relativized w.r.t. the status quo (all other metrics are in % relative to status_quo). """ # TODO(jej): Implement using MultiObjectiveTorchModelBridge's _pareto_frontier model_gen_options = { "acquisition_function_kwargs": { "chebyshev_scalarization": chebyshev } } if (trial_index is None and data is None and any( isinstance(t, BatchTrial) for t in experiment.trials.values())): raise UnsupportedError( "Must specify trial index or data for experiment with batch trials" ) absolute_metrics = [] if absolute_metrics is None else absolute_metrics for metric in absolute_metrics: if metric not in experiment.metrics: raise ValueError(f"Model was not fit on metric `{metric}`") if outcome_constraints is None: outcome_constraints = [] else: # ensure we don't constrain an objective _validate_outcome_constraints( outcome_constraints=outcome_constraints, primary_objective=primary_objective, secondary_objective=secondary_objective, ) # build posterior mean model if not data: try: data = (experiment.trials[trial_index].fetch_data() if trial_index else experiment.fetch_data()) except Exception as e: logger.info(f"Could not fetch data from experiment or trial: {e}") oc = _build_new_optimization_config( weights=np.array([0.5, 0.5]), primary_objective=primary_objective, secondary_objective=secondary_objective, outcome_constraints=outcome_constraints, ) model = Models.MOO( experiment=experiment, data=data, acqf_constructor=get_PosteriorMean, optimization_config=oc, ) status_quo = experiment.status_quo if status_quo: try: status_quo_prediction = model.predict([ ObservationFeatures( parameters=status_quo.parameters, # pyre-fixme [6]: Expected `Optional[np.int64]` for trial_index trial_index=trial_index, ) ]) except ValueError as e: logger.warning(f"Could not predict OOD status_quo outcomes: {e}") status_quo = None status_quo_prediction = None else: status_quo_prediction = None param_dicts: List[TParameterization] = [] # Construct weightings with linear angular spacing. # TODO: Verify whether 0, 1 weights cause problems because of subset_model. alpha = np.linspace(0 + 0.01, np.pi / 2 - 0.01, num_points) primary_weight = (-1 if primary_objective.lower_is_better else 1) * np.cos(alpha) secondary_weight = (-1 if secondary_objective.lower_is_better else 1) * np.sin(alpha) weights_list = np.stack([primary_weight, secondary_weight]).transpose() for weights in weights_list: outcome_constraints = outcome_constraints oc = _build_new_optimization_config( weights=weights, primary_objective=primary_objective, secondary_objective=secondary_objective, outcome_constraints=outcome_constraints, ) # TODO: (jej) T64002590 Let this serve as a starting point for optimization. # ex. Add global spacing criterion. Implement on BoTorch side. # pyre-fixme [6]: Expected different type for model_gen_options run = model.gen(1, model_gen_options=model_gen_options, optimization_config=oc) param_dicts.append(run.arms[0].parameters) # Call predict on points to get their decomposed metrics. means, cov = model.predict( [ObservationFeatures(parameters) for parameters in param_dicts]) return _extract_pareto_frontier_results( param_dicts=param_dicts, means=means, variances=cov, primary_metric=primary_objective.name, secondary_metric=secondary_objective.name, absolute_metrics=absolute_metrics, outcome_constraints=outcome_constraints, status_quo_prediction=status_quo_prediction, )
def make_experiment( parameters: List[TParameterRepresentation], name: Optional[str] = None, parameter_constraints: Optional[List[str]] = None, outcome_constraints: Optional[List[str]] = None, status_quo: Optional[TParameterization] = None, experiment_type: Optional[str] = None, # Single-objective optimization arguments: objective_name: Optional[str] = None, minimize: bool = False, # Multi-objective optimization arguments: objectives: Optional[Dict[str, str]] = None, objective_thresholds: Optional[List[str]] = None, ) -> Experiment: """Instantiation wrapper that allows for Ax `Experiment` creation without importing or instantiating any Ax classes. Args: parameters: List of dictionaries representing parameters in the experiment search space. Required elements in the dictionaries are: 1. "name" (name of parameter, string), 2. "type" (type of parameter: "range", "fixed", or "choice", string), and one of the following: 3a. "bounds" for range parameters (list of two values, lower bound first), 3b. "values" for choice parameters (list of values), or 3c. "value" for fixed parameters (single value). Optional elements are: 1. "log_scale" (for float-valued range parameters, bool), 2. "value_type" (to specify type that values of this parameter should take; expects "float", "int", "bool" or "str"), 3. "is_fidelity" (bool) and "target_value" (float) for fidelity parameters, 4. "is_ordered" (bool) for choice parameters, 5. "is_task" (bool) for task parameters, and 6. "digits" (int) for float-valued range parameters. name: Name of the experiment to be created. parameter_constraints: List of string representation of parameter constraints, such as "x3 >= x4" or "-x3 + 2*x4 - 3.5*x5 >= 2". For the latter constraints, any number of arguments is accepted, and acceptable operators are "<=" and ">=". parameter_constraints: List of string representation of parameter constraints, such as "x3 >= x4" or "-x3 + 2*x4 - 3.5*x5 >= 2". For the latter constraints, any number of arguments is accepted, and acceptable operators are "<=" and ">=". outcome_constraints: List of string representation of outcome constraints of form "metric_name >= bound", like "m1 <= 3." status_quo: Parameterization of the current state of the system. If set, this will be added to each trial to be evaluated alongside test configurations. experiment_type: String indicating type of the experiment (e.g. name of a product in which it is used), if any. objective_name: Name of the metric used as objective in this experiment, if experiment is single-objective optimization. minimize: Whether this experiment represents a minimization problem, if experiment is a single-objective optimization. objectives: Mapping from an objective name to "minimize" or "maximize" representing the direction for that objective. Used only for multi-objective optimization experiments. objective_thresholds: A list of objective threshold constraints for multi- objective optimization, in the same string format as `outcome_constraints` argument. """ if objective_name is not None and (objectives is not None or objective_thresholds is not None): raise UnsupportedError( "Ambiguous objective definition: for single-objective optimization " "`objective_name` and `minimize` arguments expected. For multi-objective " "optimization `objectives` and `objective_thresholds` arguments expected." ) status_quo_arm = None if status_quo is None else Arm(parameters=status_quo) if objectives is None: optimization_config = OptimizationConfig( objective=Objective( metric=Metric( name=objective_name or DEFAULT_OBJECTIVE_NAME, lower_is_better=minimize, ), minimize=minimize, ), outcome_constraints=make_outcome_constraints( outcome_constraints or [], status_quo_arm is not None), ) else: optimization_config = make_optimization_config( objectives, objective_thresholds or [], outcome_constraints or [], status_quo_arm is not None, ) return Experiment( name=name, search_space=make_search_space(parameters, parameter_constraints or []), optimization_config=optimization_config, status_quo=status_quo_arm, experiment_type=experiment_type, )
def _extract_asynchronous_optimization_trace( experiment: Experiment, start_time: float, end_time: float, delta_t: float, completed_time_key: str, include_only_completed_trials: bool, ) -> np.ndarray: """Extract optimization trace for an asynchronous benchmark run. This involves getting the `completed_time` from the trial `run_metadata`, as described by the `completed_time_key`. From the `start_time`, `end_time`, and `delta_t` arguments, a sequence of times is constructed. The returned optimization trace is the best achieved value so far for each time, amongst completed (or early stopped) trials. Args: experiment: The experiment from which to generate results. start_time: The starting time. end_time: The ending time. delta_t: The increment between successive time points. completed_time_key: The key from which we look up completed run times from trial `run_metadata`. include_only_completed_trials: Include results only from completed trials. This will ignore trials that were early stopped. Returns: An array representing the optimization trace as a function of time. """ if any( isinstance(trial, BatchTrial) for trial in experiment.trials.values()): raise NotImplementedError("Batched trials are not yet supported.") def get_completed_time(row): time = experiment.trials[ row.trial_index].run_metadata[completed_time_key] return pd.Series({"completed_time": time}) if include_only_completed_trials: completed_trials = experiment.trial_indices_by_status[ TrialStatus.COMPLETED] data_df = experiment.fetch_trials_data(trial_indices=completed_trials, noisy=False).df else: data_df = experiment.fetch_data(noisy=False).df minimize = experiment.optimization_config.objective.minimize # pyre-ignore[16] num_periods_running = int((end_time - start_time) // delta_t + 1) # TODO: Currently, the timestamps generated below must exactly match the # `completed_time` column iters_df = pd.DataFrame({ "completed_time": np.arange(num_periods_running) * delta_t + start_time }) true_values = {} for metric, df_m in data_df.groupby("metric_name"): df_m = data_df[data_df["metric_name"] == metric] # only keep the last data point for each arm df_m = (df_m.sort_values(["timestamp"], ascending=True).groupby("arm_name").tail(n=1)) # get completed times from run metadata df_m["completed_time"] = df_m.apply(get_completed_time, axis=1) # for trials that completed at the same time, keep only the best df_m_g = df_m.groupby("completed_time") df_m = (df_m_g.min() if minimize else df_m_g.max()).reset_index() # take cumulative best wrt the completed time df_m = df_m.sort_index() df_m["mean"] = df_m["mean"].cummin( ) if minimize else df_m["mean"].cummax() df_b = pd.merge(iters_df, df_m, how="left", on="completed_time") # replace nans with Infs, which can be handled by `best_feasible_objective` true_values[metric] = df_b["mean"].fillna( np.Inf if minimize else -np.Inf) return best_feasible_objective( # pyre-fixme[6]: Expected `OptimizationConfig` for 1st param but got # `Optional[ax.core.optimization_config.OptimizationConfig]`. optimization_config=experiment.optimization_config, values=true_values, )
def get_hierarchical_search_space_experiment() -> Experiment: return Experiment( search_space=get_hierarchical_search_space(), optimization_config=get_optimization_config(), )
def experiment_from_sqa(self, experiment_sqa: SQAExperiment) -> Experiment: """Convert SQLAlchemy Experiment to Ax Experiment.""" opt_config, tracking_metrics = self.opt_config_and_tracking_metrics_from_sqa( metrics_sqa=experiment_sqa.metrics) search_space = self.search_space_from_sqa( parameters_sqa=experiment_sqa.parameters, parameter_constraints_sqa=experiment_sqa.parameter_constraints, ) if search_space is None: raise SQADecodeError( # pragma: no cover "Experiment SearchSpace cannot be None.") runner = (self.runner_from_sqa(experiment_sqa.runner) if experiment_sqa.runner else None) status_quo = (Arm( parameters=experiment_sqa.status_quo_parameters, name=experiment_sqa.status_quo_name, ) if experiment_sqa.status_quo_parameters is not None else None) if (experiment_sqa.properties is not None and experiment_sqa.properties.get("subclass") == "SimpleExperiment"): if opt_config is None: raise SQADecodeError( # pragma: no cover "SimpleExperiment must have an optimization config.") experiment = SimpleExperiment( name=experiment_sqa.name, search_space=search_space, objective_name=opt_config.objective.metric.name, minimize=opt_config.objective.minimize, outcome_constraints=opt_config.outcome_constraints, status_quo=status_quo, ) experiment.description = experiment_sqa.description experiment.is_test = experiment_sqa.is_test else: experiment = Experiment( name=experiment_sqa.name, description=experiment_sqa.description, search_space=search_space, optimization_config=opt_config, tracking_metrics=tracking_metrics, runner=runner, status_quo=status_quo, is_test=experiment_sqa.is_test, ) trials = [ self.trial_from_sqa(trial_sqa=trial, experiment=experiment) for trial in experiment_sqa.trials ] data_by_trial = defaultdict(dict) for data_sqa in experiment_sqa.data: trial_index = data_sqa.trial_index timestamp = data_sqa.time_created data_by_trial[trial_index][timestamp] = self.data_from_sqa( data_sqa=data_sqa) data_by_trial = { trial_index: OrderedDict(sorted(data_by_timestamp.items())) for trial_index, data_by_timestamp in data_by_trial.items() } experiment._trials = {trial.index: trial for trial in trials} for trial in trials: for arm in trial.arms: experiment._arms_by_signature[arm.signature] = arm if experiment.status_quo is not None: sq_sig = experiment.status_quo.signature experiment._arms_by_signature[sq_sig] = experiment.status_quo experiment._time_created = experiment_sqa.time_created experiment._experiment_type = self.get_enum_name( value=experiment_sqa.experiment_type, enum=self.config.experiment_type_enum) experiment._data_by_trial = dict(data_by_trial) return experiment
def test_REMBOStrategy(self, mock_fit_gpytorch_model, mock_optimize_acqf): # Construct a high-D test experiment with multiple metrics hartmann_search_space = SearchSpace( parameters=[ RangeParameter( name=f"x{i}", parameter_type=ParameterType.FLOAT, lower=0.0, upper=1.0, ) for i in range(20) ] ) exp = Experiment( name="test", search_space=hartmann_search_space, optimization_config=OptimizationConfig( objective=Objective( metric=Hartmann6Metric( name="hartmann6", param_names=[f"x{i}" for i in range(6)] ), minimize=True, ), outcome_constraints=[ OutcomeConstraint( metric=L2NormMetric( name="l2norm", param_names=[f"x{i}" for i in range(6)], noise_sd=0.2, ), op=ComparisonOp.LEQ, bound=1.25, relative=False, ) ], ), runner=SyntheticRunner(), ) # Instantiate the strategy gs = REMBOStrategy(D=20, d=6, k=4, init_per_proj=4) # Check that arms and data are correctly segmented by projection exp.new_batch_trial(generator_run=gs.gen(experiment=exp, n=2)).run() self.assertEqual(len(gs.arms_by_proj[0]), 2) self.assertEqual(len(gs.arms_by_proj[1]), 0) exp.new_batch_trial(generator_run=gs.gen(experiment=exp, n=2)).run() self.assertEqual(len(gs.arms_by_proj[0]), 2) self.assertEqual(len(gs.arms_by_proj[1]), 2) # Iterate until the first projection fits a GP for _ in range(4): exp.new_batch_trial(generator_run=gs.gen(experiment=exp, n=2)).run() mock_fit_gpytorch_model.assert_not_called() self.assertEqual(len(gs.arms_by_proj[0]), 4) self.assertEqual(len(gs.arms_by_proj[1]), 4) self.assertEqual(len(gs.arms_by_proj[2]), 2) self.assertEqual(len(gs.arms_by_proj[3]), 2) # Keep iterating until GP is used for gen for i in range(4): # First two trials will go towards 3rd and 4th proj. getting enough if i < 1: # data for GP. self.assertLess(len(gs.arms_by_proj[2]), 4) if i < 2: self.assertLess(len(gs.arms_by_proj[3]), 4) exp.new_batch_trial(generator_run=gs.gen(experiment=exp, n=2)).run() if i < 2: mock_fit_gpytorch_model.assert_not_called() else: # After all proj. have > 4 arms' worth of data, GP can be fit. self.assertFalse(any(len(x) < 4 for x in gs.arms_by_proj.values())) mock_fit_gpytorch_model.assert_called() self.assertTrue(len(gs.model_transitions) > 0) gs2 = gs.clone_reset() self.assertEqual(gs2.D, 20) self.assertEqual(gs2.d, 6)
def exp_to_df( exp: Experiment, metrics: Optional[List[Metric]] = None, run_metadata_fields: Optional[List[str]] = None, trial_properties_fields: Optional[List[str]] = None, **kwargs: Any, ) -> pd.DataFrame: """Transforms an experiment to a DataFrame. Only supports Experiment and SimpleExperiment. Transforms an Experiment into a dataframe with rows keyed by trial_index and arm_name, metrics pivoted into one row. Args: exp: An Experiment that may have pending trials. metrics: Override list of metrics to return. Return all metrics if None. run_metadata_fields: fields to extract from trial.run_metadata for trial in experiment.trials. If there are multiple arms per trial, these fields will be replicated across the arms of a trial. trial_properties_fields: fields to extract from trial._properties for trial in experiment.trials. If there are multiple arms per trial, these fields will be replicated across the arms of a trial. Output columns names will be prepended with "trial_properties_". **kwargs: Custom named arguments, useful for passing complex objects from call-site to the `fetch_data` callback. Returns: DataFrame: A dataframe of inputs, metadata and metrics by trial and arm. If no trials are available, returns an empty dataframe. If no metric ouputs are available, returns a dataframe of inputs and metadata. """ def prep_return(df: pd.DataFrame, drop_col: str, sort_by: List[str]) -> pd.DataFrame: return not_none( not_none(df.drop(drop_col, axis=1)).sort_values(sort_by)) def merge_trials_dict_with_df(df: pd.DataFrame, trials_dict: Dict[int, Any], column_name: str) -> None: """Add a column ``column_name`` to a DataFrame ``df`` containing a column ``trial_index``. Each value of the new column is given by the element of ``trials_dict`` indexed by ``trial_index``. Args: df: Pandas DataFrame with column ``trial_index``, to be appended with a new column. trials_dict: Dict mapping each ``trial_index`` to a value. The new column of df will be populated with the value corresponding with the ``trial_index`` of each row. column_name: Name of the column to be appended to ``df``. """ if "trial_index" not in df.columns: raise ValueError("df must have trial_index column") if any(trials_dict.values()): # field present for any trial if not all(trials_dict.values()): # not present for all trials logger.warning( f"Column {column_name} missing for some trials. " "Filling with None when missing.") df[column_name] = [ trials_dict[trial_index] for trial_index in df.trial_index ] else: logger.warning(f"Column {column_name} missing for all trials. " "Not appending column.") def get_generation_method_str(trial: BaseTrial) -> str: generation_methods = { not_none(generator_run._model_key) for generator_run in trial.generator_runs if generator_run._model_key is not None } # add "Manual" if any generator_runs are manual if any(generator_run.generator_run_type == GeneratorRunType.MANUAL.name for generator_run in trial.generator_runs): generation_methods.add("Manual") return ", ".join( generation_methods) if generation_methods else "Unknown" # Accept Experiment and SimpleExperiment if isinstance(exp, MultiTypeExperiment): raise ValueError( "Cannot transform MultiTypeExperiments to DataFrames.") key_components = ["trial_index", "arm_name"] # Get each trial-arm with parameters arms_df = pd.DataFrame() for trial_index, trial in exp.trials.items(): for arm in trial.arms: arms_df = arms_df.append( { "arm_name": arm.name, "trial_index": trial_index, **arm.parameters }, ignore_index=True, ) # Fetch results; in case arms_df is empty, return empty results (legacy behavior) results = exp.fetch_data(metrics, **kwargs).df if len(arms_df.index) == 0: if len(results.index) != 0: raise ValueError( "exp.fetch_data().df returned more rows than there are experimental " "arms. This is an inconsistent experimental state. Please report to " "Ax support.") return results # Create key column from key_components arms_df["trial_index"] = arms_df["trial_index"].astype(int) key_col = "-".join(key_components) key_vals = arms_df[key_components[0]].astype("str") + arms_df[ key_components[1]].astype("str") arms_df[key_col] = key_vals # Add trial status trials = exp.trials.items() trial_to_status = {index: trial.status.name for index, trial in trials} merge_trials_dict_with_df(df=arms_df, trials_dict=trial_to_status, column_name="trial_status") # Add generation_method, accounting for the generic case that generator_runs is of # arbitrary length. Repeated methods within a trial are condensed via `set` and an # empty set will yield "Unknown" as the method. trial_to_generation_method = { trial_index: get_generation_method_str(trial) for trial_index, trial in trials } merge_trials_dict_with_df( df=arms_df, trials_dict=trial_to_generation_method, column_name="generation_method", ) # Add any trial properties fields to arms_df if trial_properties_fields is not None: # add trial._properties fields for field in trial_properties_fields: trial_to_properties_field = { trial_index: (trial._properties[field] if field in trial._properties else None) for trial_index, trial in trials } merge_trials_dict_with_df( df=arms_df, trials_dict=trial_to_properties_field, column_name="trial_properties_" + field, ) # Add any run_metadata fields to arms_df if run_metadata_fields is not None: # add run_metadata fields for field in run_metadata_fields: trial_to_metadata_field = { trial_index: (trial.run_metadata[field] if field in trial.run_metadata else None) for trial_index, trial in trials } merge_trials_dict_with_df( df=arms_df, trials_dict=trial_to_metadata_field, column_name=field, ) if len(results.index) == 0: logger.info( f"No results present for the specified metrics `{metrics}`. " "Returning arm parameters and metadata only.") exp_df = arms_df elif not all(col in results.columns for col in key_components): logger.warn( f"At least one of key columns `{key_components}` not present in results df " f"`{results}`. Returning arm parameters and metadata only.") exp_df = arms_df else: # prepare results for merge key_vals = results[key_components[0]].astype("str") + results[ key_components[1]].astype("str") results[key_col] = key_vals metric_vals = results.pivot(index=key_col, columns="metric_name", values="mean").reset_index() # dedupe results by key_components metadata = results[key_components + [key_col]].drop_duplicates() metrics_df = pd.merge(metric_vals, metadata, on=key_col) # merge and return exp_df = pd.merge(metrics_df, arms_df, on=key_components + [key_col], how="outer") return prep_return(df=exp_df, drop_col=key_col, sort_by=["arm_name"])
def get_standard_plots( experiment: Experiment, model: Optional[ModelBridge], model_transitions: Optional[List[int]] = None, ) -> List[go.Figure]: """Extract standard plots for single-objective optimization. Extracts a list of plots from an ``Experiment`` and ``ModelBridge`` of general interest to an Ax user. Currently not supported are - TODO: multi-objective optimization - TODO: ChoiceParameter plots Args: - experiment: The ``Experiment`` from which to obtain standard plots. - model: The ``ModelBridge`` used to suggest trial parameters. - data: If specified, data, to which to fit the model before generating plots. - model_transitions: The arm numbers at which shifts in generation_strategy occur. Returns: - a plot of objective value vs. trial index, to show experiment progression - a plot of objective value vs. range parameter values, only included if the model associated with generation_strategy can create predictions. This consists of: - a plot_slice plot if the search space contains one range parameter - an interact_contour plot if the search space contains multiple range parameters """ objective = not_none(experiment.optimization_config).objective if isinstance(objective, MultiObjective): logger.warning( "get_standard_plots does not currently support MultiObjective " "optimization experiments. Returning an empty list.") return [] if isinstance(objective, ScalarizedObjective): logger.warning( "get_standard_plots does not currently support ScalarizedObjective " "optimization experiments. Returning an empty list.") return [] if experiment.fetch_data().df.empty: logger.info( f"Experiment {experiment} does not yet have data, nothing to plot." ) return [] output_plot_list = [] output_plot_list.append( _get_objective_trace_plot( experiment=experiment, metric_name=not_none( experiment.optimization_config).objective.metric.name, model_transitions=model_transitions if model_transitions is not None else [], optimization_direction=("minimize" if not_none( experiment.optimization_config).objective.minimize else "maximize"), )) # Objective vs. parameter plot requires a `Model`, so add it only if model # is alrady available. In cases where initially custom trials are attached, # model might not yet be set on the generation strategy. if model: # TODO: Check if model can predict in favor of try/catch. try: output_plot_list.append( _get_objective_v_param_plot( search_space=experiment.search_space, model=model, metric_name=not_none( experiment.optimization_config).objective.metric.name, trials=experiment.trials, )) output_plot_list.append(_get_cross_validation_plot(model)) except NotImplementedError: # Model does not implement `predict` method. pass return [plot for plot in output_plot_list if plot is not None]
def get_experiment_for_value() -> Experiment: return Experiment(get_search_space_for_value(), "test")
def get_standard_plots( experiment: Experiment, generation_strategy: GenerationStrategy ) -> List[go.Figure]: """Extract standard plots for single-objective optimization. Extracts a list of plots from an Experiment and GenerationStrategy of general interest to an Ax user. Currently not supported are - TODO: multi-objective optimization - TODO: ChoiceParameter plots Args: - experiment: the Experiment from which to obtain standard plots. - generation_strategy: the GenerationStrategy used to suggest trial parameters in experiment Returns: - a plot of objective value vs. trial index, to show experiment progression - a plot of objective value vs. range parameter values, only included if the model associated with generation_strategy can create predictions. This consists of: - a plot_slice plot if the search space contains one range parameter - an interact_contour plot if the search space contains multiple range parameters """ objective = not_none(experiment.optimization_config).objective if isinstance(objective, MultiObjective): logger.warning( "get_standard_plots does not currently support MultiObjective " "optimization experiments. Returning an empty list." ) return [] if isinstance(objective, ScalarizedObjective): logger.warning( "get_standard_plots does not currently support ScalarizedObjective " "optimization experiments. Returning an empty list." ) return [] if experiment.fetch_data().df.empty: logger.info(f"Experiment {experiment} does not yet have data, nothing to plot.") return [] output_plot_list = [] output_plot_list.append( _get_objective_trace_plot( experiment=experiment, metric_name=not_none(experiment.optimization_config).objective.metric.name, model_transitions=generation_strategy.model_transitions, optimization_direction=( "minimize" if not_none(experiment.optimization_config).objective.minimize else "maximize" ), ) ) try: output_plot_list.append( _get_objective_v_param_plot( search_space=experiment.search_space, model=not_none(generation_strategy.model), metric_name=not_none( experiment.optimization_config ).objective.metric.name, trials=experiment.trials, ) ) except NotImplementedError: # Model does not implement `predict` method. pass return [plot for plot in output_plot_list if plot is not None]
def get_observed_pareto_frontiers( experiment: Experiment, data: Optional[Data] = None, rel: bool = True, ) -> List[ParetoFrontierResults]: """ Find all Pareto points from an experiment. Uses only values as observed in the data; no modeling is involved. Makes no assumption about the search space or types of parameters. If "data" is provided will use that, otherwise will use all data attached to the experiment. Uses all arms present in data; does not filter according to experiment search space. Assumes experiment has a multiobjective optimization config from which the objectives and outcome constraints will be extracted. Will generate a ParetoFrontierResults for every pair of metrics in the experiment's multiobjective optimization config. """ if data is None: data = experiment.fetch_data() if experiment.optimization_config is None: raise ValueError("Experiment must have an optimization config") mb = get_tensor_converter_model(experiment=experiment, data=data) pareto_observations = observed_pareto_frontier(modelbridge=mb) # Convert to ParetoFrontierResults metric_names = [ metric.name for metric in experiment.optimization_config.objective.metrics # pyre-ignore ] pfr_means = {name: [] for name in metric_names} pfr_sems = {name: [] for name in metric_names} for obs in pareto_observations: for i, name in enumerate(obs.data.metric_names): pfr_means[name].append(obs.data.means[i]) pfr_sems[name].append(np.sqrt(obs.data.covariance[i, i])) # Relativize as needed if rel and experiment.status_quo is not None: # Get status quo values sq_df = data.df[data.df["arm_name"] == experiment.status_quo.name # pyre-ignore ] sq_df = sq_df.to_dict(orient="list") # pyre-ignore sq_means = {} sq_sems = {} for i, metric in enumerate(sq_df["metric_name"]): sq_means[metric] = sq_df["mean"][i] sq_sems[metric] = sq_df["sem"][i] # Relativize for name in metric_names: if np.isnan(sq_sems[name]) or np.isnan(pfr_sems[name]).any(): # Just relativize means pfr_means[name] = [(mu / sq_means[name] - 1) * 100 for mu in pfr_means[name]] else: # Use delta method pfr_means[name], pfr_sems[name] = relativize( means_t=pfr_means[name], sems_t=pfr_sems[name], mean_c=sq_means[name], sem_c=sq_sems[name], as_percent=True, ) absolute_metrics = [] else: absolute_metrics = metric_names objective_thresholds = {} if experiment.optimization_config.objective_thresholds is not None: # pyre-ignore for objth in experiment.optimization_config.objective_thresholds: is_rel = objth.metric.name not in absolute_metrics if objth.relative != is_rel: raise ValueError( f"Objective threshold for {objth.metric.name} has " f"rel={objth.relative} but was specified here as rel={is_rel}" ) objective_thresholds[objth.metric.name] = objth.bound # Construct ParetoFrontResults for each pair pfr_list = [] param_dicts = [obs.features.parameters for obs in pareto_observations] arm_names = [obs.arm_name for obs in pareto_observations] for metric_a, metric_b in combinations(metric_names, 2): pfr_list.append( ParetoFrontierResults( param_dicts=param_dicts, means=pfr_means, sems=pfr_sems, primary_metric=metric_a, secondary_metric=metric_b, absolute_metrics=absolute_metrics, objective_thresholds=objective_thresholds, arm_names=arm_names, )) return pfr_list
def exp_to_df( exp: Experiment, metrics: Optional[List[Metric]] = None, key_components: Optional[List[str]] = None, run_metadata_fields: Optional[List[str]] = None, **kwargs: Any, ) -> pd.DataFrame: """Transforms an experiment to a DataFrame. Only supports Experiment and SimpleExperiment. Transforms an Experiment into a dataframe with rows keyed by trial_index and arm_name, metrics pivoted into one row. Args: exp: An Experiment that may have pending trials. metrics: Override list of metrics to return. Return all metrics if None. key_components: fields that combine to make a unique key corresponding to rows, similar to the list of fields passed to a GROUP BY. Defaults to ['arm_name', 'trial_index']. run_metadata_fields: fields to extract from trial.run_metadata for trial in experiment.trials. If there are multiple arms per trial, these fields will be replicated across the arms of a trial. **kwargs: Custom named arguments, useful for passing complex objects from call-site to the `fetch_data` callback. Returns: DataFrame: A dataframe of inputs and metrics by trial and arm. """ def prep_return( df: pd.DataFrame, drop_col: str, sort_by: List[str] ) -> pd.DataFrame: return not_none(not_none(df.drop(drop_col, axis=1)).sort_values(sort_by)) key_components = key_components or ["trial_index", "arm_name"] # Accept Experiment and SimpleExperiment if isinstance(exp, MultiTypeExperiment): raise ValueError("Cannot transform MultiTypeExperiments to DataFrames.") results = exp.fetch_data(metrics, **kwargs).df if len(results.index) == 0: # Handle empty case return results # create key column from key_components key_col = "-".join(key_components) key_vals = results[key_components[0]].astype("str") for key in key_components[1:]: key_vals = key_vals + results[key].astype("str") results[key_col] = key_vals # pivot dataframe from long to wide metric_vals = results.pivot( index=key_col, columns="metric_name", values="mean" ).reset_index() # dedupe results by key_components metadata = results[key_components + [key_col]].drop_duplicates() metric_and_metadata = pd.merge(metric_vals, metadata, on=key_col) # get params of each arm and merge with deduped results arm_names_and_params = pd.DataFrame( [{"arm_name": name, **arm.parameters} for name, arm in exp.arms_by_name.items()] ) exp_df = pd.merge(metric_and_metadata, arm_names_and_params, on="arm_name") # add trial status trials = exp.trials.items() trial_to_status = {index: trial.status.name for index, trial in trials} exp_df["trial_status"] = [trial_to_status[key] for key in exp_df.trial_index] # if no run_metadata fields are requested, return exp_df so far if run_metadata_fields is None: return prep_return(df=exp_df, drop_col=key_col, sort_by=key_components) if not isinstance(run_metadata_fields, list): raise ValueError("run_metadata_fields must be List[str] or None.") # add additional run_metadata fields for field in run_metadata_fields: trial_to_metadata_field = { index: (trial.run_metadata[field] if field in trial.run_metadata else None) for index, trial in trials } if any(trial_to_metadata_field.values()): # field present for any trial if not all(trial_to_metadata_field.values()): # not present for all trials logger.warning( f"Field {field} missing for some trials' run_metadata. " "Returning None when missing." ) exp_df[field] = [trial_to_metadata_field[key] for key in exp_df.trial_index] else: logger.warning( f"Field {field} missing for all trials' run_metadata. " "Not appending column." ) return prep_return(df=exp_df, drop_col=key_col, sort_by=key_components)
def testStatusQuoSetter(self): sq_parameters = self.experiment.status_quo.parameters self.experiment.status_quo = None self.assertIsNone(self.experiment.status_quo) # Verify normal update sq_parameters["w"] = 3.5 self.experiment.status_quo = Arm(sq_parameters) self.assertEqual(self.experiment.status_quo.parameters["w"], 3.5) self.assertEqual(self.experiment.status_quo.name, "status_quo") # Verify all None values self.experiment.status_quo = Arm({n: None for n in sq_parameters.keys()}) self.assertIsNone(self.experiment.status_quo.parameters["w"]) # Try extra param sq_parameters["a"] = 4 with self.assertRaises(ValueError): self.experiment.status_quo = Arm(sq_parameters) # Try wrong type sq_parameters.pop("a") sq_parameters["w"] = "hello" with self.assertRaises(ValueError): self.experiment.status_quo = Arm(sq_parameters) # Verify arms_by_signature only contains status_quo self.assertEqual(len(self.experiment.arms_by_signature), 1) # Change status quo, verify still just 1 arm sq_parameters["w"] = 3.6 self.experiment.status_quo = Arm(sq_parameters) self.assertEqual(len(self.experiment.arms_by_signature), 1) # Make a batch, add status quo to it, then change exp status quo, verify 2 arms batch = self.experiment.new_batch_trial() batch.set_status_quo_with_weight(self.experiment.status_quo, 1) sq_parameters["w"] = 3.7 self.experiment.status_quo = Arm(sq_parameters) self.assertEqual(len(self.experiment.arms_by_signature), 2) self.assertEqual(self.experiment.status_quo.name, "status_quo_e0") # Try missing param sq_parameters.pop("w") with self.assertRaises(ValueError): self.experiment.status_quo = Arm(sq_parameters) # Actually name the status quo. exp = Experiment( name="test3", search_space=get_branin_search_space(), tracking_metrics=[BraninMetric(name="b", param_names=["x1", "x2"])], runner=SyntheticRunner(), ) batch = exp.new_batch_trial() arms = get_branin_arms(n=1, seed=0) batch.add_arms_and_weights(arms=arms) self.assertIsNone(exp.status_quo) exp.status_quo = arms[0] self.assertEqual(exp.status_quo.name, "0_0") # Try setting sq to existing arm with different name with self.assertRaises(ValueError): exp.status_quo = Arm(arms[0].parameters, name="new_name")
def testEmptyMetrics(self): empty_experiment = Experiment(name="test_experiment", search_space=get_search_space()) self.assertEqual(empty_experiment.num_trials, 0) with self.assertRaises(ValueError): empty_experiment.fetch_data() batch = empty_experiment.new_batch_trial() batch.mark_running(no_runner_required=True) self.assertEqual(empty_experiment.num_trials, 1) with self.assertRaises(ValueError): batch.fetch_data() empty_experiment.add_tracking_metric(Metric(name="ax_test_metric")) self.assertTrue(empty_experiment.fetch_data().df.empty) empty_experiment.attach_data(get_data()) batch.mark_completed() self.assertFalse(empty_experiment.fetch_data().df.empty)
def testDerelativizeTransform(self, mock_predict, mock_fit, mock_observations_from_data): t = Derelativize(search_space=None, observation_features=None, observation_data=None) # ModelBridge with in-design status quo search_space = SearchSpace(parameters=[ RangeParameter("x", ParameterType.FLOAT, 0, 20), RangeParameter("y", ParameterType.FLOAT, 0, 20), ]) g = ModelBridge( search_space=search_space, model=None, transforms=[], experiment=Experiment(search_space, "test"), data=Data(), status_quo_name="1_1", ) # Test with no relative constraints objective = Objective(Metric("c")) oc = OptimizationConfig( objective=objective, outcome_constraints=[ OutcomeConstraint(Metric("a"), ComparisonOp.LEQ, bound=2, relative=False) ], ) oc2 = t.transform_optimization_config(oc, g, None) self.assertTrue(oc == oc2) # Test with relative constraint, in-design status quo oc = OptimizationConfig( objective=objective, outcome_constraints=[ OutcomeConstraint(Metric("a"), ComparisonOp.LEQ, bound=2, relative=False), OutcomeConstraint(Metric("b"), ComparisonOp.LEQ, bound=-10, relative=True), ], ) oc = t.transform_optimization_config(oc, g, None) self.assertTrue(oc.outcome_constraints == [ OutcomeConstraint( Metric("a"), ComparisonOp.LEQ, bound=2, relative=False), OutcomeConstraint( Metric("b"), ComparisonOp.LEQ, bound=4.5, relative=False), ]) obsf = mock_predict.mock_calls[0][1][1][0] obsf2 = ObservationFeatures(parameters={"x": 2.0, "y": 10.0}) self.assertTrue(obsf == obsf2) # Test with relative constraint, out-of-design status quo mock_predict.side_effect = Exception() g = ModelBridge( search_space=search_space, model=None, transforms=[], experiment=Experiment(search_space, "test"), data=Data(), status_quo_name="1_2", ) oc = OptimizationConfig( objective=objective, outcome_constraints=[ OutcomeConstraint(Metric("a"), ComparisonOp.LEQ, bound=2, relative=False), OutcomeConstraint(Metric("b"), ComparisonOp.LEQ, bound=-10, relative=True), ], ) oc = t.transform_optimization_config(oc, g, None) self.assertTrue(oc.outcome_constraints == [ OutcomeConstraint( Metric("a"), ComparisonOp.LEQ, bound=2, relative=False), OutcomeConstraint( Metric("b"), ComparisonOp.LEQ, bound=3.6, relative=False), ]) self.assertEqual(mock_predict.call_count, 2) # Raises error if predict fails with in-design status quo g = ModelBridge(search_space, None, [], status_quo_name="1_1") oc = OptimizationConfig( objective=objective, outcome_constraints=[ OutcomeConstraint(Metric("a"), ComparisonOp.LEQ, bound=2, relative=False), OutcomeConstraint(Metric("b"), ComparisonOp.LEQ, bound=-10, relative=True), ], ) with self.assertRaises(Exception): oc = t.transform_optimization_config(oc, g, None) # Raises error with relative constraint, no status quo exp = Experiment(search_space, "name") g = ModelBridge(search_space, None, [], exp) with self.assertRaises(ValueError): oc = t.transform_optimization_config(oc, g, None) # Raises error with relative constraint, no modelbridge with self.assertRaises(ValueError): oc = t.transform_optimization_config(oc, None, None)
def get_standard_plots( experiment: Experiment, model: Optional[ModelBridge], data: Optional[Data] = None, model_transitions: Optional[List[int]] = None, true_objective_metric_name: Optional[str] = None, ) -> List[go.Figure]: """Extract standard plots for single-objective optimization. Extracts a list of plots from an ``Experiment`` and ``ModelBridge`` of general interest to an Ax user. Currently not supported are - TODO: multi-objective optimization - TODO: ChoiceParameter plots Args: - experiment: The ``Experiment`` from which to obtain standard plots. - model: The ``ModelBridge`` used to suggest trial parameters. - data: If specified, data, to which to fit the model before generating plots. - model_transitions: The arm numbers at which shifts in generation_strategy occur. Returns: - a plot of objective value vs. trial index, to show experiment progression - a plot of objective value vs. range parameter values, only included if the model associated with generation_strategy can create predictions. This consists of: - a plot_slice plot if the search space contains one range parameter - an interact_contour plot if the search space contains multiple range parameters """ if (true_objective_metric_name is not None and true_objective_metric_name not in experiment.metrics.keys()): raise ValueError( f"true_objective_metric_name='{true_objective_metric_name}' is not present " f"in experiment.metrics={experiment.metrics}. Please add a valid " "true_objective_metric_name or remove the optional parameter to get " "standard plots.") objective = not_none(experiment.optimization_config).objective if isinstance(objective, ScalarizedObjective): logger.warning( "get_standard_plots does not currently support ScalarizedObjective " "optimization experiments. Returning an empty list.") return [] if data is None: data = experiment.fetch_data() if data.df.empty: logger.info( f"Experiment {experiment} does not yet have data, nothing to plot." ) return [] output_plot_list = [] output_plot_list.extend( _get_objective_trace_plot( experiment=experiment, data=data, model_transitions=model_transitions if model_transitions is not None else [], true_objective_metric_name=true_objective_metric_name, )) # Objective vs. parameter plot requires a `Model`, so add it only if model # is alrady available. In cases where initially custom trials are attached, # model might not yet be set on the generation strategy. if model: # TODO: Check if model can predict in favor of try/catch. try: if true_objective_metric_name is not None: output_plot_list.append( _objective_vs_true_objective_scatter( model=model, objective_metric_name=objective.metric_names[0], true_objective_metric_name=true_objective_metric_name, )) output_plot_list.extend( _get_objective_v_param_plots( experiment=experiment, model=model, )) output_plot_list.extend(_get_cross_validation_plots(model=model)) feature_importance_plot = plot_feature_importance_by_feature_plotly( model=model, relative=False, caption=FEATURE_IMPORTANCE_CAPTION) feature_importance_plot.layout.title = "[ADVANCED] " + str( # pyre-fixme[16]: go.Figure has no attribute `layout` feature_importance_plot.layout.title.text) output_plot_list.append(feature_importance_plot) output_plot_list.append( interact_fitted_plotly(model=model, rel=False)) except NotImplementedError: # Model does not implement `predict` method. pass return [plot for plot in output_plot_list if plot is not None]
def make_experiment( parameters: List[TParameterRepresentation], name: Optional[str] = None, parameter_constraints: Optional[List[str]] = None, outcome_constraints: Optional[List[str]] = None, status_quo: Optional[TParameterization] = None, experiment_type: Optional[str] = None, tracking_metric_names: Optional[List[str]] = None, # Single-objective optimization arguments: objective_name: Optional[str] = None, minimize: bool = False, # Multi-objective optimization arguments: objectives: Optional[Dict[str, str]] = None, objective_thresholds: Optional[List[str]] = None, support_intermediate_data: Optional[bool] = False, immutable_search_space_and_opt_config: Optional[bool] = True, ) -> Experiment: """Instantiation wrapper that allows for Ax `Experiment` creation without importing or instantiating any Ax classes. Args: parameters: List of dictionaries representing parameters in the experiment search space. Required elements in the dictionaries are: 1. "name" (name of parameter, string), 2. "type" (type of parameter: "range", "fixed", or "choice", string), and one of the following: 3a. "bounds" for range parameters (list of two values, lower bound first), 3b. "values" for choice parameters (list of values), or 3c. "value" for fixed parameters (single value). Optional elements are: 1. "log_scale" (for float-valued range parameters, bool), 2. "value_type" (to specify type that values of this parameter should take; expects "float", "int", "bool" or "str"), 3. "is_fidelity" (bool) and "target_value" (float) for fidelity parameters, 4. "is_ordered" (bool) for choice parameters, 5. "is_task" (bool) for task parameters, and 6. "digits" (int) for float-valued range parameters. name: Name of the experiment to be created. parameter_constraints: List of string representation of parameter constraints, such as "x3 >= x4" or "-x3 + 2*x4 - 3.5*x5 >= 2". For the latter constraints, any number of arguments is accepted, and acceptable operators are "<=" and ">=". parameter_constraints: List of string representation of parameter constraints, such as "x3 >= x4" or "-x3 + 2*x4 - 3.5*x5 >= 2". For the latter constraints, any number of arguments is accepted, and acceptable operators are "<=" and ">=". outcome_constraints: List of string representation of outcome constraints of form "metric_name >= bound", like "m1 <= 3." status_quo: Parameterization of the current state of the system. If set, this will be added to each trial to be evaluated alongside test configurations. experiment_type: String indicating type of the experiment (e.g. name of a product in which it is used), if any. tracking_metric_names: Names of additional tracking metrics not used for optimization. objective_name: Name of the metric used as objective in this experiment, if experiment is single-objective optimization. minimize: Whether this experiment represents a minimization problem, if experiment is a single-objective optimization. objectives: Mapping from an objective name to "minimize" or "maximize" representing the direction for that objective. Used only for multi-objective optimization experiments. objective_thresholds: A list of objective threshold constraints for multi- objective optimization, in the same string format as `outcome_constraints` argument. support_intermediate_data: whether trials may report metrics results for incomplete runs. immutable_search_space_and_opt_config: Whether it's possible to update the search space and optimization config on this experiment after creation. Defaults to True. If set to True, we won't store or load copies of the search space and optimization config on each generator run, which will improve storage performance. """ if objective_name is not None and (objectives is not None or objective_thresholds is not None): raise UnsupportedError( "Ambiguous objective definition: for single-objective optimization " "`objective_name` and `minimize` arguments expected. For multi-objective " "optimization `objectives` and `objective_thresholds` arguments expected." ) status_quo_arm = None if status_quo is None else Arm(parameters=status_quo) # TODO(jej): Needs to be decided per-metric when supporting heterogenous data. metric_cls = MapMetric if support_intermediate_data else Metric if objectives is None: optimization_config = OptimizationConfig( objective=Objective( metric=metric_cls( name=objective_name or DEFAULT_OBJECTIVE_NAME, lower_is_better=minimize, ), minimize=minimize, ), outcome_constraints=make_outcome_constraints( outcome_constraints or [], status_quo_arm is not None), ) else: optimization_config = make_optimization_config( objectives, objective_thresholds or [], outcome_constraints or [], status_quo_arm is not None, ) tracking_metrics = (None if tracking_metric_names is None else [ Metric(name=metric_name) for metric_name in tracking_metric_names ]) default_data_type = (DataType.MAP_DATA if support_intermediate_data else DataType.DATA) immutable_ss_and_oc = immutable_search_space_and_opt_config properties = ({} if not immutable_search_space_and_opt_config else { Keys.IMMUTABLE_SEARCH_SPACE_AND_OPT_CONF.value: immutable_ss_and_oc }) return Experiment( name=name, search_space=make_search_space(parameters, parameter_constraints or []), optimization_config=optimization_config, status_quo=status_quo_arm, experiment_type=experiment_type, tracking_metrics=tracking_metrics, default_data_type=default_data_type, properties=properties, )
def exp_to_df( exp: Experiment, metrics: Optional[List[Metric]] = None, run_metadata_fields: Optional[List[str]] = None, trial_properties_fields: Optional[List[str]] = None, **kwargs: Any, ) -> pd.DataFrame: """Transforms an experiment to a DataFrame with rows keyed by trial_index and arm_name, metrics pivoted into one row. If the pivot results in more than one row per arm (or one row per ``arm * map_keys`` combination if ``map_keys`` are present), results are omitted and warning is produced. Only supports ``Experiment``. Transforms an ``Experiment`` into a ``pd.DataFrame``. Args: exp: An ``Experiment`` that may have pending trials. metrics: Override list of metrics to return. Return all metrics if ``None``. run_metadata_fields: fields to extract from ``trial.run_metadata`` for trial in ``experiment.trials``. If there are multiple arms per trial, these fields will be replicated across the arms of a trial. trial_properties_fields: fields to extract from ``trial._properties`` for trial in ``experiment.trials``. If there are multiple arms per trial, these fields will be replicated across the arms of a trial. Output columns names will be prepended with ``"trial_properties_"``. **kwargs: Custom named arguments, useful for passing complex objects from call-site to the `fetch_data` callback. Returns: DataFrame: A dataframe of inputs, metadata and metrics by trial and arm (and ``map_keys``, if present). If no trials are available, returns an empty dataframe. If no metric ouputs are available, returns a dataframe of inputs and metadata. """ # Accept Experiment and SimpleExperiment if isinstance(exp, MultiTypeExperiment): raise ValueError( "Cannot transform MultiTypeExperiments to DataFrames.") key_components = ["trial_index", "arm_name"] # Get each trial-arm with parameters arms_df = pd.DataFrame() for trial_index, trial in exp.trials.items(): for arm in trial.arms: arms_df = arms_df.append( { "arm_name": arm.name, "trial_index": trial_index, **arm.parameters }, ignore_index=True, ) # Fetch results; in case arms_df is empty, return empty results (legacy behavior) data = exp.fetch_data(metrics, **kwargs) results = data.df if len(arms_df.index) == 0: if len(results.index) != 0: raise ValueError( "exp.fetch_data().df returned more rows than there are experimental " "arms. This is an inconsistent experimental state. Please report to " "Ax support.") return results # Create key column from key_components arms_df["trial_index"] = arms_df["trial_index"].astype(int) # Add trial status trials = exp.trials.items() trial_to_status = {index: trial.status.name for index, trial in trials} _merge_trials_dict_with_df(df=arms_df, trials_dict=trial_to_status, column_name="trial_status") # Add generation_method, accounting for the generic case that generator_runs is of # arbitrary length. Repeated methods within a trial are condensed via `set` and an # empty set will yield "Unknown" as the method. trial_to_generation_method = { trial_index: _get_generation_method_str(trial) for trial_index, trial in trials } _merge_trials_dict_with_df( df=arms_df, trials_dict=trial_to_generation_method, column_name="generation_method", ) # Add any trial properties fields to arms_df if trial_properties_fields is not None: # add trial._properties fields for field in trial_properties_fields: trial_to_properties_field = { trial_index: (trial._properties[field] if field in trial._properties else None) for trial_index, trial in trials } _merge_trials_dict_with_df( df=arms_df, trials_dict=trial_to_properties_field, column_name="trial_properties_" + field, ) # Add any run_metadata fields to arms_df if run_metadata_fields is not None: # add run_metadata fields for field in run_metadata_fields: trial_to_metadata_field = { trial_index: (trial.run_metadata[field] if field in trial.run_metadata else None) for trial_index, trial in trials } _merge_trials_dict_with_df( df=arms_df, trials_dict=trial_to_metadata_field, column_name=field, ) exp_df = _merge_results_if_no_duplicates( arms_df=arms_df, data=data, key_components=key_components, metrics=metrics or list(exp.metrics.values()), ) return not_none(not_none(exp_df).sort_values(["trial_index"]))
def benchmark_minimize_callable( problem: BenchmarkProblem, num_trials: int, method_name: str, replication_index: Optional[int] = None, ) -> Tuple[Experiment, Callable[[List[float]], float]]: """ An interface for evaluating external methods on Ax benchmark problems. The arms run and performance will be tracked by Ax, so the external method can be evaluated alongside Ax methods. It is designed around methods that implement an interface like scipy.optimize.minimize. This function will return a callable evaluation function that takes in an array of parameter values and returns a float objective value. The evaluation function should always be minimized: if the benchmark problem is a maximization problem, then the value returned by the evaluation function will be negated so it can be used directly by methods that minimize. This callable can be given to an external minimization function, and Ax will track all of the calls made to it and the arms that were evaluated. This will also return an Experiment object that will track the arms evaluated by the external method in the same way as done for Ax internal benchmarks. This function should thus be used for each benchmark replication. Args: problem: The Ax benchmark problem to be used to construct the evalutaion function. num_trials: The maximum number of trials for a benchmark run. method_name: Name of the method being tested. replication_index: Replicate number, if multiple replicates are being run. """ # Some validation if isinstance(problem, SimpleBenchmarkProblem): raise NonRetryableBenchmarkingError( "`SimpleBenchmarkProblem` not supported.") if not all( isinstance(p, RangeParameter) for p in problem.search_space.parameters.values()): raise NonRetryableBenchmarkingError( "Only continuous search spaces supported.") if any(p.log_scale for p in problem.search_space.parameters.values() # pyre-ignore ): raise NonRetryableBenchmarkingError( "Log-scale parameters not supported.") # Create Ax experiment experiment_name = f"{method_name}_on_{problem.name}" if replication_index is not None: experiment_name += f"__v{replication_index}" experiment = Experiment( name=experiment_name, search_space=problem.search_space, optimization_config=problem.optimization_config, runner=SyntheticRunner(), ) max_trials = num_trials # to be used below # Construct the evaluation function def evaluation_function(x: List[float]) -> float: # Check if we have exhuasted the evaluation budget if len(experiment.trials) >= max_trials: raise ValueError( f"Evaluation budget ({max_trials} trials) exhuasted.") # Create an ObservationFeatures param_dict = { pname: x[i] for i, pname in enumerate(problem.search_space.parameters.keys()) } obsf = ObservationFeatures(parameters=param_dict) # pyre-ignore # Get the time since last call num_trials = len(experiment.trials) if num_trials == 0: gen_time = None else: previous_ts = experiment.trials[num_trials - 1].time_created.timestamp() gen_time = time.time() - previous_ts # Create a GR arms, candidate_metadata_by_arm_signature = gen_arms( observation_features=[obsf], arms_by_signature=experiment.arms_by_signature) gr = GeneratorRun( arms=arms, gen_time=gen_time, candidate_metadata_by_arm_signature= candidate_metadata_by_arm_signature, ) # Add it as a trial trial = experiment.new_trial().add_generator_run(gr).run() # Evaluate function df = trial.fetch_data().df if len(df) > 1: raise Exception( "Does not support multiple outcomes") # pragma: no cover obj = float(df["mean"].values[0]) if not problem.optimization_config.objective.minimize: obj = -obj return obj return experiment, evaluation_function
def gen( self, experiment: Experiment, data: Optional[Data] = None, n: int = 1, **kwargs: Any, ) -> GeneratorRun: """Produce the next points in the experiment.""" self._set_experiment(experiment=experiment) new_arm_signatures = set() data = data or experiment.fetch_data() if data is not None and not data.df.empty: if self._data.df.empty: new_data = data.df else: # Select only the new data to determine how many new arms were # evaluated since the generation strategy was last updated with # data (find rows that are in `data.df`, but not in `self._data.df`) merged = data.df.merge( self._data.df, on=[ "arm_name", "trial_index", "metric_name", "mean", "sem" ], how="left", indicator=True, ) new_data = merged[merged["_merge"] == "left_only"] # Get arm signatures for each entry in data that the GS hasn't seen yet. new_arm_signatures = { not_none(experiment.arms_by_name.get( row["arm_name"])).signature for _, row in new_data.iterrows() if (row["arm_name"] in experiment.arms_by_name and not not_none(experiment.trials.get( row["trial_index"])).status.is_failed) } enough_observed = (len(self._observed) + len(new_arm_signatures) ) >= self._curr.min_arms_observed unlimited_arms = self._curr.num_arms == -1 enough_generated = (not unlimited_arms and len(self._generated) >= self._curr.num_arms) # Check that minimum observed_arms is satisfied if it's enforced. if self._curr.enforce_num_arms and enough_generated and not enough_observed: raise DataRequiredError( "All trials for current model have been generated, but not enough " "data has been observed to fit next model. Try again when more data " "are available.") # TODO[Lena, T44021164]: take into account failed trials. Potentially # reduce `_generated` count when a trial mentioned in new data failed. lgr = self.last_generator_run if enough_generated and enough_observed: # Change to the next model. self._change_model(experiment=experiment, data=data) elif lgr is not None and lgr._model_state_after_gen is not None: model_state = not_none(lgr._model_state_after_gen) self._set_current_model(experiment=experiment, data=data, **model_state) else: self._set_current_model(experiment=experiment, data=data) model = not_none(self._model) kwargs = consolidate_kwargs( kwargs_iterable=[self._curr.model_gen_kwargs, kwargs], keywords=get_function_argument_names(not_none(self._model).gen), ) gen_run = model.gen(n=n, **kwargs) # If nothing failed, update known data, _generated, and _observed. self._data = data self._generated.extend([arm.signature for arm in gen_run.arms]) self._observed.extend(new_arm_signatures) self._generator_runs.append(gen_run) return gen_run
def get_pareto_optimal_parameters( experiment: Experiment, generation_strategy: GenerationStrategy, use_model_predictions: bool = True, ) -> Optional[Dict[int, Tuple[TParameterization, TModelPredictArm]]]: """Identifies the best parameterizations tried in the experiment so far, using model predictions if ``use_model_predictions`` is true and using observed values from the experiment otherwise. By default, uses model predictions to account for observation noise. NOTE: The format of this method's output is as follows: { trial_index --> (parameterization, (means, covariances) }, where means are a dictionary of form { metric_name --> metric_mean } and covariances are a nested dictionary of form { one_metric_name --> { another_metric_name: covariance } }. Args: experiment: Experiment, from which to find Pareto-optimal arms. generation_strategy: Generation strategy containing the modelbridge. use_model_predictions: Whether to extract the Pareto frontier using model predictions or directly observed values. If ``True``, the metric means and covariances in this method's output will also be based on model predictions and may differ from the observed values. Returns: ``None`` if it was not possible to extract the Pareto frontier, otherwise a mapping from trial index to the tuple of: - the parameterization of the arm in that trial, - two-item tuple of metric means dictionary and covariance matrix (model-predicted if ``use_model_predictions=True`` and observed otherwise). """ # Validate aspects of the experiment: that it is a MOO experiment and # that the current model can be used to produce the Pareto frontier. if not not_none(experiment.optimization_config).is_moo_problem: raise UnsupportedError( "Please use `get_best_parameters` for single-objective problems.") moo_optimization_config = checked_cast(MultiObjectiveOptimizationConfig, experiment.optimization_config) if moo_optimization_config.outcome_constraints: # TODO[drfreund]: Test this flow and remove error. raise NotImplementedError( "Support for outcome constraints is currently under development.") # Extract or instantiate modelbridge to use for Pareto frontier extraction. mb = generation_strategy.model if mb is None or not isinstance(mb, MultiObjectiveTorchModelBridge): logger.info( "Can only extract a Pareto frontier using a multi-objective model bridge" f", but currently used model bridge is: {mb} of type {type(mb)}. Will " "use `Models.MOO` instead to extract Pareto frontier.") mb = checked_cast( MultiObjectiveTorchModelBridge, Models.MOO(experiment=experiment, data=checked_cast(Data, experiment.lookup_data())), ) else: # Make sure the model is up-to-date with the most recent data. generation_strategy._set_or_update_current_model(data=None) # If objective thresholds are not specified in optimization config, extract # the inferred ones if possible or infer them anew if not. objective_thresholds_override = None if not moo_optimization_config.objective_thresholds: lgr = generation_strategy.last_generator_run if lgr and lgr.gen_metadata and "objective_thresholds" in lgr.gen_metadata: objective_thresholds_override = lgr.gen_metadata[ "objective_thresholds"] objective_thresholds_override = mb.infer_objective_thresholds( search_space=experiment.search_space, optimization_config=experiment.optimization_config, fixed_features=None, ) logger.info( f"Using inferred objective thresholds: {objective_thresholds_override}, " "as objective thresholds were not specified as part of the optimization " "configuration on the experiment.") # Extract the Pareto frontier and format it as follows: # { trial_index --> (parameterization, (means, covariances) } pareto_util = predicted_pareto if use_model_predictions else observed_pareto pareto_optimal_observations = pareto_util( modelbridge=mb, objective_thresholds=objective_thresholds_override) return { int(not_none(obs.features.trial_index)): ( obs.features.parameters, (obs.data.means_dict, obs.data.covariance_matrix), ) for obs in pareto_optimal_observations }
def should_stop_trials_early( self, trial_indices: Set[int], experiment: Experiment, **kwargs: Dict[str, Any], ) -> Dict[int, Optional[str]]: """Stop a trial if its performance is in the bottom `percentile_threshold` of the trials at the same step. Args: trial_indices: Indices of candidate trials to consider for early stopping. experiment: Experiment that contains the trials and other contextual data. Returns: A dictionary mapping trial indices that should be early stopped to (optional) messages with the associated reason. An empty dictionary means no suggested updates to any trial's status. """ if experiment.optimization_config is None: raise UnsupportedError( # pragma: no cover "Experiment must have an optimization config in order to use an " "early stopping strategy." ) optimization_config = not_none(experiment.optimization_config) objective_name = optimization_config.objective.metric.name minimize = optimization_config.objective.minimize data = experiment.lookup_data(keep_latest_map_values_only=False) if data.df.empty: logger.info( "PercentileEarlyStoppingStrategy received empty data. " "Not stopping any trials." ) return {} if not isinstance(data, MapData): raise ValueError( "PercentileEarlyStoppingStrategy expects MapData, but the " f"data attached to experiment is of type {type(data)}." ) map_keys = data.map_keys if len(map_keys) > 1: raise ValueError( # pragma: no cover "PercentileEarlyStoppingStrategy expects MapData with a single " "map key, but the data attached to the experiment has multiple: " f"{data.map_keys}." ) map_key = map_keys[0] df = data.df metric_to_aligned_means, _ = align_partial_results( df=df, progr_key=map_key, metrics=[objective_name], ) aligned_means = metric_to_aligned_means[objective_name] decisions = { trial_index: self.should_stop_trial_early( trial_index=trial_index, experiment=experiment, df=aligned_means, percentile_threshold=self.percentile_threshold, map_key=map_key, minimize=minimize, ) for trial_index in trial_indices } return { trial_index: reason for trial_index, (should_stop, reason) in decisions.items() if should_stop }
def _init_experiment_from_sqa(self, experiment_sqa: SQAExperiment) -> Experiment: """First step of conversion within experiment_from_sqa.""" opt_config, tracking_metrics = self.opt_config_and_tracking_metrics_from_sqa( metrics_sqa=experiment_sqa.metrics) search_space = self.search_space_from_sqa( parameters_sqa=experiment_sqa.parameters, parameter_constraints_sqa=experiment_sqa.parameter_constraints, ) if search_space is None: raise SQADecodeError( # pragma: no cover "Experiment SearchSpace cannot be None.") status_quo = ( Arm( # pyre-fixme[6]: Expected `Dict[str, Optional[Union[bool, float, # int, str]]]` for 1st param but got `Optional[Dict[str, # Optional[Union[bool, float, int, str]]]]`. parameters=experiment_sqa.status_quo_parameters, name=experiment_sqa.status_quo_name, ) if experiment_sqa.status_quo_parameters is not None else None) if len(experiment_sqa.runners) == 0: runner = None elif len(experiment_sqa.runners) == 1: runner = self.runner_from_sqa(experiment_sqa.runners[0]) else: raise ValueError( # pragma: no cover "Multiple runners on experiment " "only supported for MultiTypeExperiment.") # `experiment_sqa.properties` is `sqlalchemy.ext.mutable.MutableDict` # so need to convert it to regular dict. properties = dict(experiment_sqa.properties or {}) # Remove 'subclass' from experiment's properties, since its only # used for decoding to the correct experiment subclass in storage. subclass = properties.pop(Keys.SUBCLASS, None) if subclass == "SimpleExperiment": if opt_config is None: raise SQADecodeError( # pragma: no cover "SimpleExperiment must have an optimization config.") experiment = SimpleExperiment( name=experiment_sqa.name, search_space=search_space, objective_name=opt_config.objective.metric.name, minimize=opt_config.objective.minimize, outcome_constraints=opt_config.outcome_constraints, status_quo=status_quo, properties=properties, ) experiment.description = experiment_sqa.description experiment.is_test = experiment_sqa.is_test else: experiment = Experiment( name=experiment_sqa.name, description=experiment_sqa.description, search_space=search_space, optimization_config=opt_config, tracking_metrics=tracking_metrics, runner=runner, status_quo=status_quo, is_test=experiment_sqa.is_test, properties=properties, ) return experiment
def test_best_point( self, _mock_gen, _mock_best_point, _mock_fit, _mock_predict, _mock_gen_arms, _mock_unwrap, _mock_obs_from_data, ): exp = Experiment(search_space=get_search_space_for_range_value(), name="test") oc = OptimizationConfig( objective=Objective(metric=Metric("a"), minimize=False), outcome_constraints=[], ) modelbridge = ArrayModelBridge( search_space=get_search_space_for_range_value(), model=NumpyModel(), transforms=[t1, t2], experiment=exp, data=Data(), optimization_config=oc, ) self.assertEqual(list(modelbridge.transforms.keys()), ["Cast", "t1", "t2"]) # test check that optimization config is required with self.assertRaises(ValueError): run = modelbridge.gen(n=1, optimization_config=None) # _fit is mocked, which typically sets this. modelbridge.outcomes = ["a"] run = modelbridge.gen( n=1, optimization_config=oc, ) arm, predictions = run.best_arm_predictions self.assertEqual(arm.parameters, {}) self.assertEqual(predictions[0], {"m": 1.0}) self.assertEqual(predictions[1], {"m": {"m": 2.0}}) model_arm, model_predictions = modelbridge.model_best_point() self.assertEqual(model_predictions[0], {"m": 1.0}) self.assertEqual(model_predictions[1], {"m": {"m": 2.0}}) # test optimization config validation - raise error when # ScalarizedOutcomeConstraint contains a metric that is not in the outcomes with self.assertRaises(ValueError): run = modelbridge.gen( n=1, optimization_config=OptimizationConfig( objective=Objective(metric=Metric("a"), minimize=False), outcome_constraints=[ ScalarizedOutcomeConstraint( metrics=[Metric("wrong_metric_name")], weights=[1.0], op=ComparisonOp.LEQ, bound=0, ) ], ), )