def test_exp_to_df(self): exp = get_branin_experiment(with_batch=True) exp.trials[0].run() df = exp_to_df(exp) self.assertIsInstance(df, pd.DataFrame) df = exp_to_df(exp, run_metadata_fields=["name"]) self.assertIn("name", df.columns)
def prepare_experiment_for_plotting( experiment: Experiment, ignored_names: Optional[List[str]] = None, ) -> pd.DataFrame: """Strip variables not desired in the final plot and truncate names for readability Args: experiment: Experiment containing trials to plot ignored_names: Metrics present in the experiment data we wish to exclude from the final plot. By default we ignore ["generation_method", "trial_status", "arm_name"] Returns: df.DataFrame: data frame ready for ingestion by plotly """ ignored_names = (["generation_method", "trial_status", "arm_name"] if ignored_names is None else ignored_names) df = exp_to_df(experiment) dropped = df.drop(ignored_names, axis=1) renamed = dropped.rename( # pyre-fixme[6] Expected `typing.Union[ # typing.Callable[[Optional[typing.Hashable]], Optional[typing.Hashable]], # None, typing.Mapping[Optional[typing.Hashable], typing.Any]]` for 1st # parameter `columns` to call `pd.core.frame.DataFrame.rename` but got # `typing.Dict[str, str]`. columns=_get_shortest_unique_suffix_dict( [str(c) for c in dropped.columns])) return renamed
def test_get_best_trial(self): exp = get_branin_experiment(with_batch=True, minimize=True) # Hack in `noise_sd` value to ensure full reproducibility. exp.metrics["branin"].noise_sd = 0.0 exp.trials[0].run() df = exp_to_df(exp) best_trial = get_best_trial(exp) pd.testing.assert_frame_equal( df.sort_values("branin").head(1), best_trial)
def test_get_best_trial(self): exp = get_branin_experiment(with_batch=True, minimize=True) # exp with no completed trials should return None self.assertIsNone(get_best_trial(exp)) # exp with completed trials should return optimal row # Hack in `noise_sd` value to ensure full reproducibility. exp.metrics[OBJECTIVE_NAME].noise_sd = 0.0 exp.trials[0].run() df = exp_to_df(exp) best_trial = get_best_trial(exp) pd.testing.assert_frame_equal( df.sort_values(OBJECTIVE_NAME).head(1), best_trial ) # exp with missing rows should return optimal row dummy_struct = namedtuple("dummy_struct", "df") mock_results = dummy_struct( df=pd.DataFrame( { "arm_name": ["0_0"], "metric_name": [OBJECTIVE_NAME], "mean": [DUMMY_OBJECTIVE_MEAN], "sem": [0], "trial_index": [0], "n": [123], "frac_nonnull": [1], } ) ) with patch.object(Experiment, "fetch_data", lambda self, metrics: mock_results): best_trial = get_best_trial(exp=exp) self.assertEqual(best_trial[OBJECTIVE_NAME][0], DUMMY_OBJECTIVE_MEAN) # when optimal objective is shared across multiple trials, # arbitrarily return a single optimal row mock_results = dummy_struct( df=pd.DataFrame( { "arm_name": ["0_0", "0_1"], "metric_name": [OBJECTIVE_NAME] * 2, "mean": [DUMMY_OBJECTIVE_MEAN] * 2, "sem": [0] * 2, "trial_index": [0, 1], "n": [123] * 2, "frac_nonnull": [1] * 2, } ) ) with patch.object(Experiment, "fetch_data", lambda self, metrics: mock_results): best_trial = get_best_trial(exp=exp) self.assertEqual(len(best_trial.index), 1) self.assertEqual(best_trial[OBJECTIVE_NAME][0], DUMMY_OBJECTIVE_MEAN)
def test_exp_to_df(self): # MultiTypeExperiment should fail exp = get_multi_type_experiment() with self.assertRaisesRegex(ValueError, "MultiTypeExperiment"): exp_to_df(exp=exp) # exp with no trials should return empty results exp = get_branin_experiment() df = exp_to_df(exp=exp) self.assertEqual(len(df), 0) # set up experiment exp = get_branin_experiment(with_batch=True) # check that pre-run experiment returns all columns except objective df = exp_to_df(exp) self.assertEqual( set(EXPECTED_COLUMNS) - set(df.columns), {OBJECTIVE_NAME}) self.assertEqual(len(df.index), len(exp.arms_by_name)) exp.trials[0].run() # assert result is df with expected columns and length df = exp_to_df(exp=exp) self.assertIsInstance(df, pd.DataFrame) self.assertListEqual(sorted(df.columns), sorted(EXPECTED_COLUMNS)) self.assertEqual(len(df.index), len(exp.arms_by_name)) # test with run_metadata_fields and trial_properties_fields not empty # add source to properties for _, trial in exp.trials.items(): trial._properties["source"] = DUMMY_SOURCE df = exp_to_df(exp, run_metadata_fields=["name"], trial_properties_fields=["source"]) self.assertIn("name", df.columns) self.assertIn("trial_properties_source", df.columns) # test column values or types self.assertTrue(all(x == 0 for x in df.trial_index)) self.assertTrue(all(x == "RUNNING" for x in df.trial_status)) self.assertTrue(all(x == "Sobol" for x in df.generation_method)) self.assertTrue( all(x == DUMMY_SOURCE for x in df.trial_properties_source)) self.assertTrue(all(x == "branin_test_experiment_0" for x in df.name)) for float_column in FLOAT_COLUMNS: self.assertTrue(all( isinstance(x, float) for x in df[float_column])) # works correctly for failed trials (will need to mock) dummy_struct = namedtuple("dummy_struct", "df") mock_results = dummy_struct(df=pd.DataFrame({ "arm_name": ["0_0"], "metric_name": [OBJECTIVE_NAME], "mean": [DUMMY_OBJECTIVE_MEAN], "sem": [0], "trial_index": [0], "n": [123], "frac_nonnull": [1], })) with patch.object(Experiment, "fetch_data", lambda self, metrics: mock_results): df = exp_to_df(exp=exp) # all but one row should have a metric value of NaN self.assertEqual(pd.isna(df[OBJECTIVE_NAME]).sum(), len(df.index) - 1) # an experiment with more results than arms raises an error with patch.object( Experiment, "fetch_data", lambda self, metrics: mock_results), self.assertRaisesRegex( ValueError, "inconsistent experimental state"): exp_to_df(exp=get_branin_experiment()) # custom added trial has a generation_method of Manual custom_arm = Arm(name="custom", parameters={"x1": 0, "x2": 0}) exp.new_trial().add_arm(custom_arm) df = exp_to_df(exp) self.assertEqual(df[df.arm_name == "custom"].iloc[0].generation_method, "Manual")
def get_trials_data_frame(self) -> pd.DataFrame: return exp_to_df(exp=self.experiment)
def test_infer_objective_thresholds(self, _, cuda=False): # lightweight test exp = get_branin_experiment_with_multi_objective( has_optimization_config=True, with_batch=True, with_status_quo=True, ) for trial in exp.trials.values(): trial.mark_running(no_runner_required=True).mark_completed() exp.attach_data( get_branin_data_multi_objective(trial_indices=exp.trials.keys()) ) data = exp.fetch_data() modelbridge = TorchModelBridge( search_space=exp.search_space, model=MultiObjectiveBotorchModel(), optimization_config=exp.optimization_config, transforms=Cont_X_trans + Y_trans, torch_device=torch.device("cuda" if cuda else "cpu"), experiment=exp, data=data, ) fixed_features = ObservationFeatures(parameters={"x1": 0.0}) search_space = exp.search_space.clone() param_constraints = [ ParameterConstraint(constraint_dict={"x1": 1.0}, bound=10.0) ] search_space.add_parameter_constraints(param_constraints) oc = exp.optimization_config.clone() oc.objective._objectives[0].minimize = True expected_base_gen_args = modelbridge._get_transformed_gen_args( search_space=search_space.clone(), optimization_config=oc, fixed_features=fixed_features, ) with ExitStack() as es: mock_model_infer_obj_t = es.enter_context( patch( "ax.modelbridge.torch.infer_objective_thresholds", wraps=infer_objective_thresholds, ) ) mock_get_transformed_gen_args = es.enter_context( patch.object( modelbridge, "_get_transformed_gen_args", wraps=modelbridge._get_transformed_gen_args, ) ) mock_get_transformed_model_gen_args = es.enter_context( patch.object( modelbridge, "_get_transformed_model_gen_args", wraps=modelbridge._get_transformed_model_gen_args, ) ) mock_untransform_objective_thresholds = es.enter_context( patch.object( modelbridge, "_untransform_objective_thresholds", wraps=modelbridge._untransform_objective_thresholds, ) ) obj_thresholds = modelbridge.infer_objective_thresholds( search_space=search_space, optimization_config=oc, fixed_features=fixed_features, ) expected_obj_weights = torch.tensor([-1.0, 1.0]) ckwargs = mock_model_infer_obj_t.call_args[1] self.assertTrue( torch.equal(ckwargs["objective_weights"], expected_obj_weights) ) # check that transforms have been applied (at least UnitX) self.assertEqual(ckwargs["bounds"], [(0.0, 1.0), (0.0, 1.0)]) lc = ckwargs["linear_constraints"] self.assertTrue(torch.equal(lc[0], torch.tensor([[15.0, 0.0]]))) self.assertTrue(torch.equal(lc[1], torch.tensor([[15.0]]))) self.assertEqual(ckwargs["fixed_features"], {0: 1.0 / 3.0}) mock_get_transformed_gen_args.assert_called_once() mock_get_transformed_model_gen_args.assert_called_once_with( search_space=expected_base_gen_args.search_space, fixed_features=expected_base_gen_args.fixed_features, pending_observations=expected_base_gen_args.pending_observations, optimization_config=expected_base_gen_args.optimization_config, ) mock_untransform_objective_thresholds.assert_called_once() ckwargs = mock_untransform_objective_thresholds.call_args[1] self.assertTrue( torch.equal(ckwargs["objective_weights"], expected_obj_weights) ) self.assertEqual(ckwargs["bounds"], [(0.0, 1.0), (0.0, 1.0)]) self.assertEqual(ckwargs["fixed_features"], {0: 1.0 / 3.0}) self.assertEqual(obj_thresholds[0].metric.name, "branin_a") self.assertEqual(obj_thresholds[1].metric.name, "branin_b") self.assertEqual(obj_thresholds[0].op, ComparisonOp.LEQ) self.assertEqual(obj_thresholds[1].op, ComparisonOp.GEQ) self.assertFalse(obj_thresholds[0].relative) self.assertFalse(obj_thresholds[1].relative) df = exp_to_df(exp) Y = np.stack([df.branin_a.values, df.branin_b.values]).T Y = torch.from_numpy(Y) Y[:, 0] *= -1 pareto_Y = Y[is_non_dominated(Y)] nadir = pareto_Y.min(dim=0).values self.assertTrue( np.all( np.array([-obj_thresholds[0].bound, obj_thresholds[1].bound]) < nadir.numpy() ) ) # test using MTGP sobol_generator = get_sobol( search_space=exp.search_space, seed=TEST_SOBOL_SEED, # set initial position equal to the number of sobol arms generated # so far. This means that new sobol arms will complement the previous # arms in a space-filling fashion init_position=len(exp.arms_by_name) - 1, ) sobol_run = sobol_generator.gen(n=2) trial = exp.new_batch_trial(optimize_for_power=True) trial.add_generator_run(sobol_run) trial.mark_running(no_runner_required=True).mark_completed() data = exp.fetch_data() torch.manual_seed(0) # make model fitting deterministic modelbridge = TorchModelBridge( search_space=exp.search_space, model=MultiObjectiveBotorchModel(), optimization_config=exp.optimization_config, transforms=ST_MTGP_trans, experiment=exp, data=data, ) fixed_features = ObservationFeatures(parameters={}, trial_index=1) expected_base_gen_args = modelbridge._get_transformed_gen_args( search_space=search_space.clone(), optimization_config=exp.optimization_config, fixed_features=fixed_features, ) with ExitStack() as es: mock_model_infer_obj_t = es.enter_context( patch( "ax.modelbridge.torch.infer_objective_thresholds", wraps=infer_objective_thresholds, ) ) mock_untransform_objective_thresholds = es.enter_context( patch.object( modelbridge, "_untransform_objective_thresholds", wraps=modelbridge._untransform_objective_thresholds, ) ) obj_thresholds = modelbridge.infer_objective_thresholds( search_space=search_space, optimization_config=exp.optimization_config, fixed_features=fixed_features, ) ckwargs = mock_model_infer_obj_t.call_args[1] self.assertEqual(ckwargs["fixed_features"], {2: 1.0}) mock_untransform_objective_thresholds.assert_called_once() ckwargs = mock_untransform_objective_thresholds.call_args[1] self.assertEqual(ckwargs["fixed_features"], {2: 1.0}) self.assertEqual(obj_thresholds[0].metric.name, "branin_a") self.assertEqual(obj_thresholds[1].metric.name, "branin_b") self.assertEqual(obj_thresholds[0].op, ComparisonOp.GEQ) self.assertEqual(obj_thresholds[1].op, ComparisonOp.GEQ) self.assertFalse(obj_thresholds[0].relative) self.assertFalse(obj_thresholds[1].relative) df = exp_to_df(exp) trial_mask = df.trial_index == 1 Y = np.stack([df.branin_a.values[trial_mask], df.branin_b.values[trial_mask]]).T Y = torch.from_numpy(Y) pareto_Y = Y[is_non_dominated(Y)] nadir = pareto_Y.min(dim=0).values self.assertTrue( np.all( np.array([obj_thresholds[0].bound, obj_thresholds[1].bound]) < nadir.numpy() ) )
def test_infer_objective_thresholds(self, _, cuda=False): # lightweight test exp = get_branin_experiment_with_multi_objective( has_optimization_config=True, with_batch=True, with_status_quo=True, ) for trial in exp.trials.values(): trial.mark_running(no_runner_required=True).mark_completed() exp.attach_data( get_branin_data_multi_objective(trial_indices=exp.trials.keys())) data = exp.fetch_data() modelbridge = MultiObjectiveTorchModelBridge( search_space=exp.search_space, model=MultiObjectiveBotorchModel(), optimization_config=exp.optimization_config, transforms=Cont_X_trans + Y_trans, torch_device=torch.device("cuda" if cuda else "cpu"), experiment=exp, data=data, ) fixed_features = ObservationFeatures(parameters={"x1": 0.0}) search_space = exp.search_space.clone() param_constraints = [ ParameterConstraint(constraint_dict={"x1": 1.0}, bound=10.0) ] outcome_constraints = [ OutcomeConstraint( metric=exp.metrics["branin_a"], op=ComparisonOp.GEQ, bound=-40.0, relative=False, ) ] search_space.add_parameter_constraints(param_constraints) exp.optimization_config.outcome_constraints = outcome_constraints oc = exp.optimization_config.clone() oc.objective._objectives[0].minimize = True expected_base_gen_args = modelbridge._get_transformed_gen_args( search_space=search_space.clone(), optimization_config=oc, fixed_features=fixed_features, ) with ExitStack() as es: mock_model_infer_obj_t = es.enter_context( patch( "ax.modelbridge.multi_objective_torch.infer_objective_thresholds", wraps=infer_objective_thresholds, )) mock_get_transformed_gen_args = es.enter_context( patch.object( modelbridge, "_get_transformed_gen_args", wraps=modelbridge._get_transformed_gen_args, )) mock_get_transformed_model_gen_args = es.enter_context( patch.object( modelbridge, "_get_transformed_model_gen_args", wraps=modelbridge._get_transformed_model_gen_args, )) mock_untransform_objective_thresholds = es.enter_context( patch.object( modelbridge, "untransform_objective_thresholds", wraps=modelbridge.untransform_objective_thresholds, )) obj_thresholds = modelbridge.infer_objective_thresholds( search_space=search_space, optimization_config=oc, fixed_features=fixed_features, ) expected_obj_weights = torch.tensor([-1.0, 1.0]) ckwargs = mock_model_infer_obj_t.call_args[1] self.assertTrue( torch.equal(ckwargs["objective_weights"], expected_obj_weights)) # check that transforms have been applied (at least UnitX) self.assertEqual(ckwargs["bounds"], [(0.0, 1.0), (0.0, 1.0)]) oc = ckwargs["outcome_constraints"] self.assertTrue(torch.equal(oc[0], torch.tensor([[-1.0, 0.0]]))) self.assertTrue(torch.equal(oc[1], torch.tensor([[45.0]]))) lc = ckwargs["linear_constraints"] self.assertTrue(torch.equal(lc[0], torch.tensor([[15.0, 0.0]]))) self.assertTrue(torch.equal(lc[1], torch.tensor([[15.0]]))) self.assertEqual(ckwargs["fixed_features"], {0: 1.0 / 3.0}) mock_get_transformed_gen_args.assert_called_once() mock_get_transformed_model_gen_args.assert_called_once_with( search_space=expected_base_gen_args.search_space, fixed_features=expected_base_gen_args.fixed_features, pending_observations=expected_base_gen_args. pending_observations, optimization_config=expected_base_gen_args.optimization_config, ) mock_untransform_objective_thresholds.assert_called_once() ckwargs = mock_untransform_objective_thresholds.call_args[1] self.assertTrue( torch.equal(ckwargs["objective_weights"], expected_obj_weights)) self.assertEqual(ckwargs["bounds"], [(0.0, 1.0), (0.0, 1.0)]) self.assertEqual(ckwargs["fixed_features"], {0: 1.0 / 3.0}) self.assertEqual(obj_thresholds[0].metric.name, "branin_a") self.assertEqual(obj_thresholds[1].metric.name, "branin_b") self.assertEqual(obj_thresholds[0].op, ComparisonOp.LEQ) self.assertEqual(obj_thresholds[1].op, ComparisonOp.GEQ) self.assertFalse(obj_thresholds[0].relative) self.assertFalse(obj_thresholds[1].relative) df = exp_to_df(exp) Y = np.stack([df.branin_a.values, df.branin_b.values]).T Y = torch.from_numpy(Y) Y[:, 0] *= -1 pareto_Y = Y[is_non_dominated(Y)] nadir = pareto_Y.min(dim=0).values self.assertTrue( np.all( np.array([-obj_thresholds[0].bound, obj_thresholds[1].bound]) < nadir.numpy())) # test using MTGP sobol_generator = get_sobol(search_space=exp.search_space) sobol_run = sobol_generator.gen(n=5) trial = exp.new_batch_trial(optimize_for_power=True) trial.add_generator_run(sobol_run) trial.mark_running(no_runner_required=True).mark_completed() data = exp.fetch_data() modelbridge = MultiObjectiveTorchModelBridge( search_space=exp.search_space, model=MultiObjectiveBotorchModel(), optimization_config=exp.optimization_config, transforms=ST_MTGP_trans, experiment=exp, data=data, ) fixed_features = ObservationFeatures(parameters={}, trial_index=1) expected_base_gen_args = modelbridge._get_transformed_gen_args( search_space=search_space.clone(), optimization_config=exp.optimization_config, fixed_features=fixed_features, ) with self.assertRaises(ValueError): # Check that a ValueError is raised when MTGP is being used # and trial_index is not specified as a fixed features. # Note: this error is raised by StratifiedStandardizeY modelbridge.infer_objective_thresholds( search_space=search_space, optimization_config=exp.optimization_config, ) with ExitStack() as es: mock_model_infer_obj_t = es.enter_context( patch( "ax.modelbridge.multi_objective_torch.infer_objective_thresholds", wraps=infer_objective_thresholds, )) mock_untransform_objective_thresholds = es.enter_context( patch.object( modelbridge, "untransform_objective_thresholds", wraps=modelbridge.untransform_objective_thresholds, )) obj_thresholds = modelbridge.infer_objective_thresholds( search_space=search_space, optimization_config=exp.optimization_config, fixed_features=fixed_features, ) ckwargs = mock_model_infer_obj_t.call_args[1] self.assertEqual(ckwargs["fixed_features"], {2: 1.0}) mock_untransform_objective_thresholds.assert_called_once() ckwargs = mock_untransform_objective_thresholds.call_args[1] self.assertEqual(ckwargs["fixed_features"], {2: 1.0}) self.assertEqual(obj_thresholds[0].metric.name, "branin_a") self.assertEqual(obj_thresholds[1].metric.name, "branin_b") self.assertEqual(obj_thresholds[0].op, ComparisonOp.GEQ) self.assertEqual(obj_thresholds[1].op, ComparisonOp.GEQ) self.assertFalse(obj_thresholds[0].relative) self.assertFalse(obj_thresholds[1].relative) df = exp_to_df(exp) trial_mask = df.trial_index == 1 Y = np.stack( [df.branin_a.values[trial_mask], df.branin_b.values[trial_mask]]).T Y = torch.from_numpy(Y) pareto_Y = Y[is_non_dominated(Y)] nadir = pareto_Y.min(dim=0).values self.assertTrue( np.all( np.array([obj_thresholds[0].bound, obj_thresholds[1].bound]) < nadir.numpy()))