def test_status_quo_for_non_monolithic_data(self, mock_gen):
mock_gen.return_value = (
[
ObservationFeatures(parameters={
"x1": float(i),
"x2": float(i)
},
trial_index=np.int64(1)) for i in range(5)
],
[1] * 5,
None,
{},
)
exp = get_branin_experiment_with_multi_objective(with_status_quo=True)
sobol = Models.SOBOL(search_space=exp.search_space)
exp.new_batch_trial(sobol.gen(5)).set_status_quo_and_optimize_power(
status_quo=exp.status_quo).run()
# create data where metrics vary in start and end times
data = get_non_monolithic_branin_moo_data()
with warnings.catch_warnings(record=True) as ws:
bridge = ModelBridge(
experiment=exp,
data=data,
model=Model(),
search_space=exp.search_space,
)
# just testing it doesn't error
bridge.gen(5)
self.assertTrue(any("start_time" in str(w.message) for w in ws))
self.assertTrue(any("end_time" in str(w.message) for w in ws))
self.assertEqual(bridge.status_quo.arm_name, "status_quo")
def testOODStatusQuo(self):
# An OOD status quo arm without a trial index will raise an error
experiment = get_branin_experiment()
experiment.add_tracking_metric(
BraninMetric(name="m2", param_names=["x1", "x2"]))
metrics = list(experiment.metrics.values())
sobol = Models.SOBOL(experiment.search_space)
a = sobol.gen(5)
experiment.new_batch_trial(generator_run=a).run()
# Experiments with batch trials must specify a trial index
with self.assertRaises(UnsupportedError):
compute_pareto_frontier(
experiment,
metrics[0],
metrics[1],
absolute_metrics=[m.name for m in metrics],
)
compute_pareto_frontier(
experiment,
metrics[0],
metrics[1],
trial_index=0,
absolute_metrics=[m.name for m in metrics],
)
compute_pareto_frontier(
experiment,
metrics[0],
metrics[1],
data=experiment.fetch_data(),
absolute_metrics=[m.name for m in metrics],
)
def setUp(self):
self.branin_experiment = get_branin_experiment_with_multi_objective()
sobol = Models.SOBOL(search_space=self.branin_experiment.search_space)
sobol_run = sobol.gen(n=20)
self.branin_experiment.new_batch_trial().add_generator_run(
sobol_run
).run().mark_completed()
data = self.branin_experiment.fetch_data()
ms_gpei = ModelSpec(model_enum=Models.GPEI)
ms_gpei.fit(experiment=self.branin_experiment, data=data)
ms_gpkg = ModelSpec(model_enum=Models.GPKG)
ms_gpkg.fit(experiment=self.branin_experiment, data=data)
self.fitted_model_specs = [ms_gpei, ms_gpkg]
self.model_selection_node = GenerationNode(
model_specs=self.fitted_model_specs,
best_model_selector=SingleDiagnosticBestModelSelector(
diagnostic="Fisher exact test p",
criterion=MetricAggregation.MEAN,
metric_aggregation=DiagnosticCriterion.MIN,
),
)
def test_get_model_from_generator_run(self):
"""Tests that it is possible to restore a model from a generator run it
produced, if `Models` registry was used.
"""
exp = get_branin_experiment()
initial_sobol = Models.SOBOL(experiment=exp, seed=239)
gr = initial_sobol.gen(n=1)
# Restore the model as it was before generation.
sobol = get_model_from_generator_run(generator_run=gr,
experiment=exp,
data=exp.fetch_data())
self.assertEqual(sobol.model.init_position, 0)
self.assertEqual(sobol.model.seed, 239)
# Restore the model as it was after generation (to resume generation).
sobol_after_gen = get_model_from_generator_run(generator_run=gr,
experiment=exp,
data=exp.fetch_data(),
after_gen=True)
self.assertEqual(sobol_after_gen.model.init_position, 1)
self.assertEqual(sobol_after_gen.model.seed, 239)
self.assertEqual(
initial_sobol.gen(n=1).arms,
sobol_after_gen.gen(n=1).arms)
exp.new_trial(generator_run=gr)
# Check restoration of GPEI, to ensure proper restoration of callable kwargs
gpei = Models.GPEI(experiment=exp, data=get_branin_data())
# Punch GPEI model + bridge kwargs into the Sobol generator run, to avoid
# a slow call to `gpei.gen`.
gr._model_key = "GPEI"
gr._model_kwargs = gpei._model_kwargs
gr._bridge_kwargs = gpei._bridge_kwargs
gpei_restored = get_model_from_generator_run(gr,
experiment=exp,
data=get_branin_data())
for key in gpei.__dict__:
self.assertIn(key, gpei_restored.__dict__)
original, restored = gpei.__dict__[key], gpei_restored.__dict__[
key]
# Fit times are set in instantiation so not same and model compared below
if key in ["fit_time", "fit_time_since_gen", "model"]:
continue # Fit times are set in instantiation so won't be same
if isinstance(original, OrderedDict) and isinstance(
restored, OrderedDict):
original, restored = list(original.keys()), list(
restored.keys())
if isinstance(original, Model) and isinstance(restored, Model):
continue # Model equality is tough to compare.
self.assertEqual(original, restored)
for key in gpei.model.__dict__:
self.assertIn(key, gpei_restored.model.__dict__)
original, restored = (
gpei.model.__dict__[key],
gpei_restored.model.__dict__[key],
)
# Botorch model equality is tough to compare and training data
# is unnecessary to compare, because data passed to model was the same
if key in ["model", "warm_start_refitting", "Xs", "Ys"]:
continue
self.assertEqual(original, restored)
def setUp(self) -> None:
self.experiment = get_branin_experiment()
sobol = Models.SOBOL(search_space=self.experiment.search_space)
sobol_run = sobol.gen(n=20)
self.experiment.new_batch_trial().add_generator_run(
sobol_run).run().mark_completed()
self.data = self.experiment.fetch_data()
def get_sobol(
search_space: SearchSpace,
seed: Optional[int] = None,
deduplicate: bool = False,
init_position: int = 0,
scramble: bool = True,
) -> RandomModelBridge:
"""Instantiates a Sobol sequence quasi-random generator.
Args:
search_space: Sobol generator search space.
kwargs: Custom args for sobol generator.
Returns:
RandomModelBridge, with SobolGenerator as model.
"""
logger.info(
"Factory functions (like `get_sobol`) will soon be deprecated. Use "
"the model registry instead (`Models.SOBOL(...)`).")
return checked_cast(
RandomModelBridge,
Models.SOBOL(
search_space=search_space,
seed=seed,
deduplicate=deduplicate,
init_position=init_position,
scramble=scramble,
),
)
def get_sobol(
search_space: SearchSpace,
seed: Optional[int] = None,
deduplicate: bool = False,
init_position: int = 0,
scramble: bool = True,
) -> RandomModelBridge:
"""Instantiates a Sobol sequence quasi-random generator.
Args:
search_space: Sobol generator search space.
kwargs: Custom args for sobol generator.
Returns:
RandomModelBridge, with SobolGenerator as model.
"""
return checked_cast(
RandomModelBridge,
Models.SOBOL(
search_space=search_space,
seed=seed,
deduplicate=deduplicate,
init_position=init_position,
scramble=scramble,
),
)
def test_enum_sobol_GPEI(self):
"""Tests Soboland GPEI instantiation through the Models enum."""
exp = get_branin_experiment()
# Check that factory generates a valid sobol modelbridge.
sobol = Models.SOBOL(search_space=exp.search_space)
self.assertIsInstance(sobol, RandomModelBridge)
for _ in range(5):
sobol_run = sobol.gen(n=1)
self.assertEqual(sobol_run._model_key, "Sobol")
exp.new_batch_trial().add_generator_run(sobol_run).run()
# Check that factory generates a valid GP+EI modelbridge.
exp.optimization_config = get_branin_optimization_config()
gpei = Models.GPEI(experiment=exp, data=exp.fetch_data())
self.assertIsInstance(gpei, TorchModelBridge)
self.assertEqual(gpei._model_key, "GPEI")
botorch_defaults = "ax.models.torch.botorch_defaults"
# Check that the callable kwargs and the torch kwargs were recorded.
self.assertEqual(
gpei._model_kwargs,
{
"acqf_constructor": {
"is_callable_as_path": True,
"value": f"{botorch_defaults}.get_NEI",
},
"acqf_optimizer": {
"is_callable_as_path": True,
"value": f"{botorch_defaults}.scipy_optimizer",
},
"model_constructor": {
"is_callable_as_path": True,
"value": f"{botorch_defaults}.get_and_fit_model",
},
"model_predictor": {
"is_callable_as_path": True,
"value": f"{botorch_defaults}.predict_from_model",
},
"refit_on_cv": False,
"refit_on_update": True,
"warm_start_refitting": True,
},
)
self.assertEqual(
gpei._bridge_kwargs,
{
"transform_configs": None,
"torch_dtype": torch_float64,
"torch_device": torch_device(type="cpu"),
"status_quo_name": None,
"status_quo_features": None,
"optimization_config": None,
"transforms": Cont_X_trans + Y_trans,
},
)
gpei = Models.GPEI(experiment=exp,
data=exp.fetch_data(),
search_space=exp.search_space)
self.assertIsInstance(gpei, TorchModelBridge)
def setUp(self):
experiment = get_branin_experiment()
experiment.add_tracking_metric(
BraninMetric(name="m2", param_names=["x1", "x2"]))
sobol = Models.SOBOL(experiment.search_space)
a = sobol.gen(5)
experiment.new_batch_trial(generator_run=a).run()
self.experiment = experiment
self.metrics = list(experiment.metrics.values())
def setUp(self):
super().setUp()
search_space = get_search_space()
gr = Models.SOBOL(search_space=search_space).gen(n=1)
self.model = Mock(
search_space=search_space,
status_quo=Mock(
features=ObservationFeatures(parameters=gr.arms[0].parameters)
),
)
def testPlotMultipleParetoFrontiers(self):
experiment = get_branin_experiment_with_multi_objective(
has_objective_thresholds=True, )
sobol = Models.SOBOL(experiment.search_space)
a = sobol.gen(5)
experiment.new_batch_trial(generator_run=a).run()
pfrs = get_observed_pareto_frontiers(experiment=experiment)
pfrs2 = copy.deepcopy(pfrs)
pfr_lists = {"pfrs 1": pfrs, "pfrs 2": pfrs2}
self.assertIsNotNone(interact_multiple_pareto_frontier(pfr_lists))
def test_cross_validate_raises_not_implemented_error_for_non_cv_model_with_data(
self,
):
exp = get_branin_experiment(with_batch=True)
exp.trials[0].run().complete()
sobol = Models.SOBOL(
experiment=exp, search_space=exp.search_space, data=exp.fetch_data()
)
with self.assertRaises(NotImplementedError):
cross_validate(model=sobol)
def test_enum_model_kwargs(self):
"""Tests that kwargs are passed correctly when instantiating through the
Models enum."""
exp = get_branin_experiment()
sobol = Models.SOBOL(
search_space=exp.search_space, init_position=2, scramble=False, seed=239
)
self.assertIsInstance(sobol, RandomModelBridge)
for _ in range(5):
sobol_run = sobol.gen(1)
exp.new_batch_trial().add_generator_run(sobol_run).run()
def test_ST_MTGP(self):
"""Tests single type MTGP instantiation."""
# Test Single-type MTGP
exp = get_branin_experiment()
sobol = Models.SOBOL(search_space=exp.search_space)
self.assertIsInstance(sobol, RandomModelBridge)
for _ in range(5):
sobol_run = sobol.gen(n=1)
t = exp.new_batch_trial().add_generator_run(sobol_run)
t.set_status_quo_with_weight(status_quo=t.arms[0], weight=0.5)
t.run().mark_completed()
status_quo_features = ObservationFeatures(
parameters=exp.trials[0].status_quo.parameters,
trial_index=0,
)
mtgp = Models.ST_MTGP(
experiment=exp,
data=exp.fetch_data(),
status_quo_features=status_quo_features,
)
self.assertIsInstance(mtgp, TorchModelBridge)
exp = get_branin_experiment()
sobol = Models.SOBOL(search_space=exp.search_space)
self.assertIsInstance(sobol, RandomModelBridge)
sobol_run = sobol.gen(n=1)
t = exp.new_batch_trial().add_generator_run(sobol_run)
t.set_status_quo_with_weight(status_quo=t.arms[0], weight=0.5)
t.run().mark_completed()
with self.assertRaises(ValueError):
status_quo_features = ObservationFeatures(
parameters=exp.trials[0].status_quo.parameters,
trial_index=0,
)
Models.ST_MTGP(
experiment=exp,
data=exp.fetch_data(),
status_quo_features=status_quo_features,
)
def test_ST_MTGP_NEHVI(self):
"""Tests single type MTGP NEHVI instantiation."""
multi_obj_exp = get_branin_experiment_with_multi_objective(
with_batch=True,
with_status_quo=True,
)
metrics = multi_obj_exp.optimization_config.objective.metrics
multi_objective_thresholds = [
ObjectiveThreshold(metric=metrics[0],
bound=0.0,
relative=False,
op=ComparisonOp.GEQ),
ObjectiveThreshold(metric=metrics[1],
bound=0.0,
relative=False,
op=ComparisonOp.GEQ),
]
sobol = Models.SOBOL(search_space=multi_obj_exp.search_space)
self.assertIsInstance(sobol, RandomModelBridge)
for _ in range(2):
sobol_run = sobol.gen(n=1)
t = multi_obj_exp.new_batch_trial().add_generator_run(sobol_run)
t.set_status_quo_with_weight(status_quo=t.arms[0], weight=0.5)
t.run().mark_completed()
status_quo_features = ObservationFeatures(
parameters=multi_obj_exp.trials[0].status_quo.parameters,
trial_index=0,
)
mtgp = Models.ST_MTGP_NEHVI(
experiment=multi_obj_exp,
data=multi_obj_exp.fetch_data(),
status_quo_features=status_quo_features,
objective_thresholds=multi_objective_thresholds,
)
self.assertIsInstance(mtgp, TorchModelBridge)
self.assertIsInstance(mtgp.model, MultiObjectiveBotorchModel)
# test it can generate
mtgp_run = mtgp.gen(n=1,
fixed_features=ObservationFeatures(parameters={},
trial_index=1))
self.assertEqual(len(mtgp_run.arms), 1)
# test a generated trial can be completed
t = multi_obj_exp.new_batch_trial().add_generator_run(mtgp_run)
t.set_status_quo_with_weight(status_quo=t.arms[0], weight=0.5)
t.run().mark_completed()
def test_enum_sobol_GPEI(self):
"""Tests Sobol instantiation through the Models enum."""
exp = get_branin_experiment()
# Check that factory generates a valid sobol modelbridge.
sobol = Models.SOBOL(search_space=exp.search_space)
self.assertIsInstance(sobol, RandomModelBridge)
for _ in range(5):
sobol_run = sobol.gen(n=1)
self.assertEqual(sobol_run._model_key, "Sobol")
exp.new_batch_trial().add_generator_run(sobol_run).run()
# Check that factory generates a valid GP+EI modelbridge.
exp.optimization_config = get_branin_optimization_config()
gpei = Models.GPEI(experiment=exp, data=exp.fetch_data())
self.assertIsInstance(gpei, TorchModelBridge)
gpei = Models.GPEI(experiment=exp,
data=exp.fetch_data(),
search_space=exp.search_space)
self.assertIsInstance(gpei, TorchModelBridge)
def test_relativize_transform_requires_a_modelbridge_to_have_status_quo_data(self):
sobol = Models.SOBOL(search_space=get_search_space())
self.assertIsNone(sobol.status_quo)
with self.assertRaisesRegex(ValueError, "status quo data"):
Relativize(
search_space=None,
observation_features=[],
observation_data=[],
modelbridge=sobol,
).transform_observation_data(
observation_data=[
ObservationData(
metric_names=["foo"],
means=np.array([2]),
covariance=np.array([[0.1]]),
)
],
observation_features=[ObservationFeatures(parameters={"x": 1})],
)
def test_get_model_from_generator_run(self):
"""Tests that it is possible to restore a model from a generator run it
produced, if `Models` registry was used.
"""
exp = get_branin_experiment()
initial_sobol = Models.SOBOL(experiment=exp, seed=239)
gr = initial_sobol.gen(n=1)
# Restore the model as it was before generation.
sobol = get_model_from_generator_run(generator_run=gr,
experiment=exp,
data=exp.fetch_data())
self.assertEqual(sobol.model.init_position, 0)
self.assertEqual(sobol.model.seed, 239)
# Restore the model as it was after generation (to resume generation).
sobol_after_gen = get_model_from_generator_run(generator_run=gr,
experiment=exp,
data=exp.fetch_data(),
after_gen=True)
self.assertEqual(sobol_after_gen.model.init_position, 1)
self.assertEqual(sobol_after_gen.model.seed, 239)
self.assertEqual(
initial_sobol.gen(n=1).arms,
sobol_after_gen.gen(n=1).arms)
def setUp(self) -> None:
self.experiment = get_experiment()
self.arm = Arm({"x": 1, "y": "foo", "z": True, "w": 4})
self.trial = self.experiment.new_trial(GeneratorRun([self.arm]))
self.experiment_2 = get_experiment()
self.batch_trial = self.experiment_2.new_batch_trial(
GeneratorRun([self.arm]))
self.batch_trial.set_status_quo_with_weight(
self.experiment_2.status_quo, 1)
self.obs_feat = ObservationFeatures.from_arm(arm=self.trial.arm,
trial_index=np.int64(
self.trial.index))
self.hss_exp = get_hierarchical_search_space_experiment()
self.hss_sobol = Models.SOBOL(search_space=self.hss_exp.search_space)
self.hss_gr = self.hss_sobol.gen(n=1)
self.hss_trial = self.hss_exp.new_trial(self.hss_gr)
self.hss_arm = not_none(self.hss_trial.arm)
self.hss_cand_metadata = self.hss_trial._get_candidate_metadata(
arm_name=self.hss_arm.name)
self.hss_full_parameterization = self.hss_cand_metadata.get(
Keys.FULL_PARAMETERIZATION).copy()
self.assertTrue(
all(p_name in self.hss_full_parameterization
for p_name in self.hss_exp.search_space.parameters))
self.hss_obs_feat = ObservationFeatures.from_arm(
arm=self.hss_arm,
trial_index=np.int64(self.hss_trial.index),
metadata=self.hss_cand_metadata,
)
self.hss_obs_feat_all_params = ObservationFeatures.from_arm(
arm=Arm(self.hss_full_parameterization),
trial_index=np.int64(self.hss_trial.index),
metadata={
Keys.FULL_PARAMETERIZATION: self.hss_full_parameterization
},
)
def test_cross_validate_gives_a_useful_error_for_model_with_no_data(self):
exp = get_branin_experiment()
sobol = Models.SOBOL(experiment=exp, search_space=exp.search_space)
with self.assertRaisesRegex(ValueError, "no training data"):
cross_validate(model=sobol)
RangeParameter(name="lr", lower=1.0e-5, upper=1.0e-1,
parameter_type=ParameterType.FLOAT),
RangeParameter(name="lr_change", lower=0.5, upper=1.0,
parameter_type=ParameterType.FLOAT),
RangeParameter(name="leafes", lower=2, upper=1000,
parameter_type=ParameterType.INT)]
)
experiment = SimpleExperiment(
name = f"weather_lbgm_{dt.datetime.today().strftime('%d-%m-%Y')}",
search_space = search_space,
evaluation_function = mdl.obj_fun,
)
sobol = Models.SOBOL(experiment.search_space)
for i in range(n_random_trials):
experiment.new_trial(generator_run=sobol.gen(1))
best_arm = None
for i in range(n_searches):
gpei = Models.GPEI(experiment=experiment, data=experiment.eval())
generator_run = gpei.gen(1)
best_arm, _ = generator_run.best_arm_predictions
experiment.new_trial(generator_run=generator_run)
best_para_ax = best_arm.parameters
n_oos = 0
params['num_boost_round'] = 200
def main():
args = parse_args()
function_list = [get_function_by_name[name]
for name in args.function_name_list.split(',')]
weight_list = list(map(float, args.weight_list.split(',')))
covariance_matrix = json.loads(args.covariance_matrix)
evaluate_covariance = args.evaluate_covariance
init_iter = args.init_iter
# if init_iter > 1:
# raise ValueError("init_iter should be 1.")
init_batch_size = args.init_batch_size
update_iter = args.update_iter
batch_size = args.batch_size
var_coef = args.var_coef
var_compute_type = args.var_compute_type
num_random = args.num_random
num_bucket = args.num_bucket
save_path = args.save_path
# num_control = args.num_control
minimize = True
groundtruth_function = get_groundtruth_function(function_list, weight_list)
#evaluation_function = get_evaluation_function(
# function_list, weight_list, covariance_matrix,
# evaluate_covariance, var_coef)
evaluation_function = get_evaluation_function(
function_list, weight_list, covariance_matrix,
var_compute_type, num_random, num_bucket)
exp = SimpleExperiment(
name=args.function_name_list + args.weight_list,
search_space=get_search_space(function_list),
evaluation_function=evaluation_function,
objective_name="objective_name",
minimize=minimize,
)
t_start = time.time()
print(f"Start time: {t_start}")
print(f"Sobol iteration begin...{time.time() - t_start}")
sobol = Models.SOBOL(exp.search_space)
for i in range(init_iter):
if init_batch_size == 1:
exp.new_trial(generator_run=sobol.gen(init_batch_size))
else:
exp.new_batch_trial(generator_run=sobol.gen(init_batch_size))
print(f"Running sobol optimization trial {i+1}/{init_iter}..."
f"{time.time() - t_start}")
print(f"GPEI iteration begin...{time.time() - t_start}")
for i in range(update_iter):
gpei = Models.BOTORCH(experiment=exp, data=exp.eval())
if batch_size == 1:
exp.new_trial(generator_run=gpei.gen(batch_size))
else:
exp.new_batch_trial(generator_run=gpei.gen(batch_size))
print(f"Running GPEI optimization trial {i+1}/{update_iter}..."
f"{time.time() - t_start}")
# Construct Result.
## origin data.
data_df = copy.deepcopy(exp.eval().df)
compare_func = min if minimize else max
arm_name2mean = {}
for _, row in data_df.iterrows():
arm_name2mean[row["arm_name"]] = row["mean"]
## parameters true_mean.
other_columns = {
"arm_name": [], "parameters": [], "true_mean": [],
"cur_trial_best_mean": [], "accum_trials_best_mean": []}
atbm = None # accum_trial_best_mean
for trial in exp.trials.values():
ctbm = None # cur_trial_best_mean
for arm in trial.arms:
other_columns['arm_name'].append(arm.name)
other_columns['parameters'].append(json.dumps(arm.parameters))
other_columns['true_mean'].append(
groundtruth_function(arm.parameters))
if ctbm is None:
ctbm = arm_name2mean[arm.name]
ctbm = compare_func(ctbm, arm_name2mean[arm.name])
if atbm is None:
atbm = ctbm
atbm = compare_func(atbm, ctbm)
other_columns['cur_trial_best_mean'].extend([ctbm] * len(trial.arms))
other_columns['accum_trials_best_mean'].extend([atbm] * len(trial.arms))
other_df = DataFrame(other_columns)
result_df = data_df.set_index('arm_name').join(
other_df.set_index('arm_name')).reset_index()
# Save to file.
print("Save to file.")
sub_dir_name = "_".join([
"ax", args.function_name_list.replace(",", "_"),
args.weight_list.replace(",", "_"), args.covariance_matrix.replace(
"[", "_").replace("]", "_").replace(",", "_").replace(" ", ""),
str(args.evaluate_covariance), str(args.init_iter), str(init_batch_size),
str(args.update_iter), str(args.batch_size), str(args.var_coef),
str(minimize), str(var_compute_type), str(num_random), str(num_bucket)
])
abs_dir_path = os.path.join(save_path, sub_dir_name)
Path(abs_dir_path).mkdir(parents=True, exist_ok=True)
task_id = os.environ.get('TASK_INDEX')
cur_time = pd.Timestamp.now().strftime('%Y%m%d%H%M%S')
filename = cur_time + "_" + str(task_id) + ".csv"
print(os.path.join(abs_dir_path, filename))
result_df.to_csv(os.path.join(abs_dir_path, filename))
print("2021-01-19 19:48:00")
print("Done...")
def sobol(experiment, n_initial_evaluations, seed=None):
print(f'Running Sobol initialisation...')
sobol = Models.SOBOL(experiment.search_space, seed=int(seed))
for i in range(n_initial_evaluations):
print(f'Sobol: {i + 1}/{n_initial_evaluations}')
experiment.new_trial(generator_run=sobol.gen(1))
def test_get_standard_plots(self):
exp = get_branin_experiment()
self.assertEqual(
len(
get_standard_plots(experiment=exp,
model=get_generation_strategy().model)),
0,
)
exp = get_branin_experiment(with_batch=True, minimize=True)
exp.trials[0].run()
plots = get_standard_plots(
experiment=exp,
model=Models.BOTORCH(experiment=exp, data=exp.fetch_data()),
)
self.assertEqual(len(plots), 6)
self.assertTrue(all(isinstance(plot, go.Figure) for plot in plots))
exp = get_branin_experiment_with_multi_objective(with_batch=True)
exp.optimization_config.objective.objectives[0].minimize = False
exp.optimization_config.objective.objectives[1].minimize = True
exp.optimization_config._objective_thresholds = [
ObjectiveThreshold(metric=exp.metrics["branin_a"],
op=ComparisonOp.GEQ,
bound=-100.0),
ObjectiveThreshold(metric=exp.metrics["branin_b"],
op=ComparisonOp.LEQ,
bound=100.0),
]
exp.trials[0].run()
plots = get_standard_plots(experiment=exp,
model=Models.MOO(experiment=exp,
data=exp.fetch_data()))
self.assertEqual(len(plots), 7)
# All plots are successfully created when objective thresholds are absent
exp.optimization_config._objective_thresholds = []
plots = get_standard_plots(experiment=exp,
model=Models.MOO(experiment=exp,
data=exp.fetch_data()))
self.assertEqual(len(plots), 7)
exp = get_branin_experiment_with_timestamp_map_metric(
with_status_quo=True)
exp.new_trial().add_arm(exp.status_quo)
exp.trials[0].run()
exp.new_trial(generator_run=Models.SOBOL(
search_space=exp.search_space).gen(n=1))
exp.trials[1].run()
plots = get_standard_plots(
experiment=exp,
model=Models.BOTORCH(experiment=exp, data=exp.fetch_data()),
true_objective_metric_name="branin",
)
self.assertEqual(len(plots), 9)
self.assertTrue(all(isinstance(plot, go.Figure) for plot in plots))
self.assertIn(
"Objective branin_map vs. True Objective Metric branin",
[p.layout.title.text for p in plots],
)
with self.assertRaisesRegex(
ValueError, "Please add a valid true_objective_metric_name"):
plots = get_standard_plots(
experiment=exp,
model=Models.BOTORCH(experiment=exp, data=exp.fetch_data()),
true_objective_metric_name="not_present",
)
def matbench_fold(fold):
t0 = time()
train_inputs, train_outputs = task.get_train_and_val_data(fold)
train_val_df = pd.DataFrame({
"formula": train_inputs.values,
"target": train_outputs.values
})
if dummy:
train_val_df = train_val_df[:25]
optimization_config = OptimizationConfig(objective=Objective(
metric=CrabNetMetric(name=metric,
train_val_df=train_val_df,
n_splits=n_splits),
minimize=True,
), )
# TODO: use status_quo (Arm) as default CrabNet parameters
exp = Experiment(
name="nested_crabnet_mae_saas",
search_space=search_space,
optimization_config=optimization_config,
runner=SyntheticRunner(),
)
sobol = Models.SOBOL(exp.search_space)
print("evaluating SOBOL points")
for _ in range(n_sobol):
print(_)
trial = exp.new_trial(generator_run=sobol.gen(1))
trial.run()
trial.mark_completed()
data = exp.fetch_data()
j = -1
new_value = np.nan
best_so_far = np.nan
for j in range(n_saas):
saas = Models.FULLYBAYESIAN(
experiment=exp,
data=exp.fetch_data(),
num_samples=
num_samples, # Increasing this may result in better model fits
warmup_steps=
warmup_steps, # Increasing this may result in better model fits
gp_kernel=
"rbf", # "rbf" is the default in the paper, but we also support "matern"
torch_device=tkwargs["device"],
torch_dtype=tkwargs["dtype"],
verbose=False, # Set to True to print stats from MCMC
disable_progbar=
True, # Set to False to print a progress bar from MCMC
)
generator_run = saas.gen(1)
best_arm, _ = generator_run.best_arm_predictions
trial = exp.new_trial(generator_run=generator_run)
trial.run()
trial.mark_completed()
data = Data.from_multiple_data([data, trial.fetch_data()])
new_value = trial.fetch_data().df["mean"].min()
best_so_far = data.df["mean"].min()
tf = time()
print(
f"iter{j}, BestInIter:{new_value:.3f}, BestSoFar:{best_so_far:.3f} elapsed time: {tf - t0}",
)
exp.fetch_data()
best_parameters = best_arm.parameters
experiment_fpath = join(experiment_dir, "experiment" + str(fold) + ".json")
save_experiment(exp, experiment_fpath)
test_pred, default_mae, test_mae, best_parameterization = get_test_results(
task, fold, best_parameters, train_val_df)
print(f"default_mae: {default_mae}")
print(f"test_mae: {test_mae}")
# maes.append(test_mae) # [0.32241879861870626, ...]
# task.record(fold, test_pred, params=best_parameterization)
return test_pred, best_parameterization
def pid(cfg):
env_name = cfg.env.params.name
env = gym.make(env_name)
env.reset()
full_rewards = []
exp_cfg = cfg.experiment
from learn.utils.plotly import hv_characterization
hv_characterization()
def compare_control(env, cfg, save=True):
import torch
from learn.control.pid import PidPolicy
controllers = []
labels = []
metrics = []
# PID baselines
# /Users/nato/Documents/Berkeley/Research/Codebases/dynamics-learn/sweeps/2020-04-14/11-12-02
# from learn.simulate_sac import *
# Rotation policy
sac_policy1 = torch.load(
'/Users/nato/Documents/Berkeley/Research/Codebases/dynamics-learn/outputs/2020-03-24/18-32-26/trial_70000.dat'
)
controllers.append(sac_policy1['policy'])
labels.append("SAC - Rotation")
metrics.append(0)
# Living reward policy
sac_policy2 = torch.load(
'/Users/nato/Documents/Berkeley/Research/Codebases/dynamics-learn/outputs/2020-03-24/18-31-45/trial_35000.dat'
)
controllers.append(sac_policy2['policy'])
labels.append("SAC - Living")
metrics.append(1)
# Square cost policy
# sac_policy2 = torch.load(
# '/Users/nato/Documents/Berkeley/Research/Codebases/dynamics-learn/sweeps/2020-03-25/20-30-47/metric.name=Square,robot=iono_sim/26/trial_40000.dat')
controllers.append(sac_policy2['policy'])
labels.append("SAC - Square")
metrics.append(2)
# un-Optimized PID parameters
pid_params = [[2531.917, 61.358, 33.762], [2531.917, 61.358, 33.762]]
pid = PidPolicy(cfg)
pid.set_params(pid_params)
controllers.append(pid)
labels.append("PID - temp")
metrics.append(0)
controllers.append(pid)
labels.append("PID - temp")
metrics.append(1)
# Optimized PID parameters
pid_params = [[2531.917, 61.358, 3333.762],
[2531.917, 61.358, 3333.762]]
pid = PidPolicy(cfg)
pid.set_params(pid_params)
controllers.append(pid)
labels.append("PID - improved")
metrics.append(2)
from learn.control.mpc import MPController
cfg.policy.mode = 'mpc'
# dynam_model = torch.load(
# '/Users/nato/Documents/Berkeley/Research/Codebases/dynamics-learn/outputs/2020-03-25/10-45-17/trial_1.dat')
dynam_model = torch.load(
'/Users/nato/Documents/Berkeley/Research/Codebases/dynamics-learn/sweeps/2020-03-25/20-30-57/metric.name=Rotation,robot=iono_sim/14/trial_9.dat'
)
mpc = MPController(env, dynam_model['model'], cfg)
controllers.append(mpc)
labels.append("MPC - 1")
metrics.append(0)
controllers.append(mpc)
labels.append("MPC - 2")
metrics.append(1)
controllers.append(mpc)
labels.append("MPC - 3")
metrics.append(2)
import plotly.graph_objects as go
import plotly
colors = [
'#1f77b4', # muted blue
'#ff7f0e', # safety orange
'#2ca02c', # cooked asparagus green
'#d62728', # brick red
'#9467bd', # muted purple
'#8c564b', # chestnut brown
'#e377c2', # raspberry yogurt pink
'#7f7f7f', # middle gray
'#bcbd22', # curry yellow-green
'#17becf' # blue-teal
]
markers = [
"cross",
"circle-open-dot",
"x-open-dot",
"triangle-up-open-dot",
"y-down-open",
"diamond-open-dot",
"hourglass",
"hash",
"star",
"square",
]
m1 = living_reward
m2 = rotation_mat
m3 = squ_cost
eval_metrics = [m1, m2, m3]
metric_names = ["Living", "Rotation", "Square"]
fig = plotly.subplots.make_subplots(
rows=3,
cols=2,
# subplot_titles=["Living", "Rotation", "Square"],
subplot_titles=[
"Pitch",
"Roll",
" ",
" ",
" ",
" ",
],
vertical_spacing=0.03,
horizontal_spacing=0.03,
shared_xaxes=True,
) # go.Figure()
fig_mpc = go.Figure()
fig_sac = go.Figure()
pry = [1, 0, 2]
# state0 = 2*env.reset()
# state0 = env.reset()
state0 = np.array([0, np.deg2rad(15), 0, 0, 0, 0])
for i, (con, lab, m) in enumerate(zip(controllers, labels, metrics)):
print(f"Evaluating controller type {lab}")
_ = env.reset()
env.set_state(np.concatenate((np.zeros(6), state0)))
state = state0
states = []
actions = []
rews = []
done = False
# for t in range(cfg.experiment.r_len + 1):
for t in range(500):
if done:
break
if "SAC" in lab:
with torch.no_grad():
with eval_mode(con):
action = con.select_action(state)
if i < 2:
action = np.array([65535, 65535, 65535, 65535
]) * (action + 1) / 2
else:
action = np.array([3000, 3000, 3000, 3000
]) * (action + 1) / 2
else:
action = con.get_action(state, metric=eval_metrics[m])
states.append(state)
actions.append(action)
state, rew, done, _ = env.step(action)
done = done
states = np.stack(states)
actions = np.stack(actions)
pitch = np.degrees(states[:, pry[0]])
roll = np.degrees(states[:, pry[1]])
# deal with markers
num_mark = np.zeros(len(pitch))
mark_every = 50
m_size = 32
start = np.random.randint(0, int(len(pitch) / 10))
num_mark[start::mark_every] = m_size
if "SAC" in lab:
fig_sac.add_trace(
go.Scatter(
y=pitch,
name=metric_names[m], # legendgroup=lab[:3],
# showlegend=(True if (i % 3 == 0) else False),
line=dict(color=colors[m], width=4),
cliponaxis=False,
mode='lines+markers',
marker=dict(color=colors[m],
symbol=markers[-m],
size=num_mark.tolist())))
elif "MPC" in lab:
fig_mpc.add_trace(
go.Scatter(
y=pitch,
name=metric_names[m], # legendgroup=lab[:3],
# showlegend=(True if (i % 3 == 0) else False),
line=dict(color=colors[m], width=4),
cliponaxis=False,
mode='lines+markers',
marker=dict(color=colors[m],
symbol=markers[-m],
size=num_mark.tolist())))
fig.add_trace(
go.Scatter(
y=pitch,
name=lab[:3] + str(m),
legendgroup=lab[:3],
showlegend=(True if (i % 3 == 0) else False),
line=dict(color=colors[int(i / 3)],
width=2), # mode='lines+markers',
# marker=dict(color=colors[i], symbol=markers[i], size=16)
),
row=m + 1,
col=1)
fig.add_trace(
go.Scatter(
y=roll,
name=lab[:3] + str(m),
legendgroup=lab[:3],
showlegend=(False),
line=dict(color=colors[int(i / 3)],
width=2), # mode='lines+markers',
# marker=dict(color=colors[i], symbol=markers[i], size=16)
),
row=m + 1,
col=2)
fig.update_layout(
title='Comparison of Controllers and Reward Functions',
font=dict(family="Times New Roman, Times, serif",
size=24,
color="black"),
legend_orientation="h",
legend=dict(
x=.6,
y=0.07,
bgcolor='rgba(205, 223, 212, .4)',
bordercolor="Black",
),
# xaxis_title='Timestep',
# yaxis_title='Angle (Degrees)',
plot_bgcolor='white',
width=1600,
height=1000,
# xaxis=dict(
# showline=True,
# showgrid=False,
# showticklabels=True, ),
# yaxis=dict(
# showline=True,
# showgrid=False,
# showticklabels=True, ),
)
fig_sac.update_layout( # title='Comparison of SAC Policies',
font=dict(family="Times New Roman, Times, serif",
size=32,
color="black"),
legend_orientation="h",
legend=dict(
x=.35,
y=0.1,
bgcolor='rgba(205, 223, 212, .4)',
bordercolor="Black",
),
# xaxis_title='Timestep',
# yaxis_title='Angle (Degrees)',
showlegend=False,
plot_bgcolor='white',
width=1600,
height=800,
margin=dict(t=5, r=5),
)
fig_mpc.update_layout( # title='Comparison of MPC Policies',
font=dict(family="Times New Roman, Times, serif",
size=32,
color="black"),
legend_orientation="h",
showlegend=False,
legend=dict(
x=.35,
y=0.1,
bgcolor='rgba(205, 223, 212, .4)',
bordercolor="Black",
# ncol= 2,
),
# xaxis_title='Timestep',
# yaxis_title='Angle (Degrees)',
plot_bgcolor='white',
width=1600,
height=800,
margin=dict(t=5, r=5),
)
reg_color = 'rgba(255,60,60,.15)'
fig_sac.add_trace(
go.Scatter(x=[0, 500],
y=[5, 5],
name='Living Region',
legendgroup='Living Region',
fill='tozeroy',
mode='lines',
fillcolor=reg_color,
line=dict(width=0.0,
color=reg_color))) # fill down to xaxis
fig_sac.add_trace(
go.Scatter(x=[0, 500],
y=[-5, -5],
showlegend=False,
legendgroup='Living Region',
fill='tozeroy',
mode='lines',
fillcolor=reg_color,
line=dict(width=0.0,
color=reg_color))) # fill down to xaxis
fig_mpc.add_trace(
go.Scatter(x=[0, 500],
y=[5, 5],
name='Living Region',
legendgroup='Living Region',
fill='tozeroy',
mode='lines',
fillcolor=reg_color,
line=dict(width=0.0,
color=reg_color))) # fill down to xaxis
fig_mpc.add_trace(
go.Scatter(x=[0, 500],
y=[-5, -5],
showlegend=False,
legendgroup='Living Region',
fill='tozeroy',
mode='lines',
fillcolor=reg_color,
line=dict(width=0.0,
color=reg_color))) # fill down to xaxis
# SOLO
rang_ind = [-20, 20]
fig_sac.update_xaxes(
title_text="Timestep",
range=[0, 500],
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig_sac.update_yaxes(
title_text="Pitch (degrees)",
range=rang_ind,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig_sac.show()
fig_sac.write_image(os.getcwd() + "/compare_sac.pdf")
fig_mpc.update_xaxes(
title_text="Timestep",
range=[0, 500],
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig_mpc.update_yaxes(
title_text="Pitch (degrees)",
range=rang_ind,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig_mpc.show()
fig_mpc.write_image(os.getcwd() + "/compare_mpc.pdf")
# COMPARISON
fig.update_xaxes(
title_text="Timestep",
row=3,
col=1,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_xaxes(
row=2,
col=1,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_xaxes(
row=1,
col=1,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_xaxes(
title_text="Timestep",
row=3,
col=2,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_xaxes(
row=2,
col=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_xaxes(
row=1,
col=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
# fig.update_xaxes(title_text="xaxis 1 title", row=1, col=1)
# fig.update_yaxes(title_text="Roll (Degrees)", row=1, col=1)
rang = [-30, 30]
nticks = 6
fig.update_yaxes(
title_text="Living Rew.",
range=rang,
row=1,
col=1,
nticks=nticks,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_yaxes(
title_text="Rotation Rew.",
range=rang,
row=2,
col=1,
nticks=nticks,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_yaxes(
title_text="Square Cost",
range=rang,
row=3,
col=1,
nticks=nticks,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_yaxes(
range=rang,
row=1,
col=2,
nticks=nticks,
showticklabels=False,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_yaxes(
range=rang,
row=2,
col=2,
nticks=nticks,
showticklabels=False,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
fig.update_yaxes(
range=rang,
row=3,
col=2,
nticks=nticks,
showticklabels=False,
ticks="inside",
tickwidth=2,
zeroline=True,
zerolinecolor='rgba(0,0,0,.5)',
zerolinewidth=1,
)
print(f"Plotting {len(labels)} control responses")
# save = False
# if save:
# fig.write_image(os.getcwd() + "compare.png")
# else:
# fig.show()
#
# return fig
# compare_control(env, cfg, save=True)
# quit()
plot_results(logx=False, save=True, mpc=False)
quit()
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # Evalutation Function # # # # # # # # # # # # # # # # # # # #
def bo_rollout_wrapper(params, weights=None): # env, controller, exp_cfg):
pid_1 = [params["pitch-p"], params["pitch-i"], params["pitch-d"]]
# pid_1 = [params["roll-p"], params["roll-i"],
# params["roll-d"]] # [params["pitch-p"], params["pitch-i"], params["pitch-d"]]
pid_2 = [params["roll-p"], params["roll-i"], params["roll-d"]]
print(
f"Optimizing Parameters {np.round(pid_1, 3)},{np.round(pid_2, 3)}")
pid_params = [[pid_1[0], pid_1[1], pid_1[2]],
[pid_2[0], pid_2[1], pid_2[2]]]
# pid_params = [[1000, 0, 0], [1000, 0, 0]]
pid = PidPolicy(cfg)
pid.set_params(pid_params)
cum_cost = []
r = 0
fncs = [squ_cost, living_reward, rotation_mat]
mult_rewards = [[] for _ in range(len(fncs))]
while r < cfg.experiment.repeat:
pid.reset()
states, actions, rews, sim_error = rollout(env, pid, exp_cfg)
# plot_rollout(states, actions, pry=[1, 0, 2])
rewards_full = get_rewards(states, actions, fncs=fncs)
for i, vec in enumerate(rewards_full):
mult_rewards[i].append(vec)
# if sim_error:
# print("Repeating strange simulation")
# continue
# if len(rews) < 400:
# cum_cost.append(-(cfg.experiment.r_len - len(rews)) / cfg.experiment.r_len)
# else:
rollout_cost = np.sum(rews) / cfg.experiment.r_len # / len(rews)
# if rollout_cost > max_cost:
# max_cost = rollout_cost
# rollout_cost += get_reward_euler(states[-1], actions[-1])
cum_cost.append(rollout_cost)
r += 1
std = np.std(cum_cost)
cum_cost = np.mean(cum_cost)
# print(f"Cum. Cost {cum_cost / max_cost}")
# print(f"- Mean Cum. Cost / Rew: {cum_cost}, std dev: {std}")
eval = {
"Square": (np.mean(rewards_full[0]), np.std(rewards_full[0])),
"Living": (np.mean(rewards_full[1]), np.std(rewards_full[1])),
"Rotation": (np.mean(rewards_full[2]), np.std(rewards_full[2]))
}
for n, (key, value) in enumerate(eval.items()):
if n == 0:
print(f"- Square {np.round(value, 4)}")
elif n == 1:
print(f"- Living {np.round(value, 4)}")
else:
print(f"- Rotn {np.round(value, 4)}")
return eval
# return cum_cost.reshape(1, 1), std
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
from ax import (
ComparisonOp,
ParameterType,
RangeParameter,
SearchSpace,
SimpleExperiment,
OutcomeConstraint,
)
exp = SimpleExperiment(
name="PID Control Robot",
search_space=SearchSpace([
RangeParameter(
name=f"roll-p",
parameter_type=ParameterType.FLOAT,
lower=1.0,
upper=10000.0,
log_scale=True,
),
# FixedParameter(name="roll-i", value=0.0, parameter_type=ParameterType.FLOAT),
RangeParameter(
name=f"roll-i",
parameter_type=ParameterType.FLOAT,
lower=0,
upper=1000.0,
log_scale=False,
),
RangeParameter(
name=f"roll-d",
parameter_type=ParameterType.FLOAT,
lower=.1,
upper=5000.0,
log_scale=True,
),
RangeParameter(
name=f"pitch-p",
parameter_type=ParameterType.FLOAT,
lower=1.0,
upper=10000.0,
log_scale=True,
),
RangeParameter(
name=f"pitch-d",
parameter_type=ParameterType.FLOAT,
lower=0,
upper=1000.0,
log_scale=False,
),
RangeParameter(
name=f"pitch-i",
parameter_type=ParameterType.FLOAT,
lower=.1,
upper=5000.0,
log_scale=True,
),
# FixedParameter(name="pitch-i", value=0.0, parameter_type=ParameterType.FLOAT),
]),
evaluation_function=bo_rollout_wrapper,
objective_name=cfg.metric.name,
minimize=cfg.metric.minimize,
outcome_constraints=[],
)
from ax.storage.metric_registry import register_metric
from ax.storage.runner_registry import register_runner
class GenericMetric(Metric):
def fetch_trial_data(self, trial):
records = []
for arm_name, arm in trial.arms_by_name.items():
params = arm.parameters
mean, sem = bo_rollout_wrapper(params)
records.append({
"arm_name": arm_name,
"metric_name": self.name,
"mean": mean,
"sem": sem,
"trial_index": trial.index,
})
return Data(df=pd.DataFrame.from_records(records))
class MyRunner(Runner):
def run(self, trial):
return {"name": str(trial.index)}
optimization_config = OptimizationConfig(objective=Objective(
metric=GenericMetric(name=cfg.metric.name),
minimize=cfg.metric.minimize,
), )
register_metric(GenericMetric)
register_runner(MyRunner)
exp.runner = MyRunner()
exp.optimization_config = optimization_config
log.info(f"Running experiment, metric name {cfg.metric.name}")
log.info(f"Running Sobol initialization trials...")
sobol = Models.SOBOL(exp.search_space)
num_search = cfg.bo.random
for i in range(num_search):
exp.new_trial(generator_run=sobol.gen(1))
exp.trials[len(exp.trials) - 1].run()
import plotly.graph_objects as go
gpei = Models.BOTORCH(experiment=exp, data=exp.eval())
objectives = ["Living", "Square", "Rotation"]
def plot_all(model, objectives, name="", rend=False):
for o in objectives:
plot = plot_contour(
model=model,
param_x="roll-p",
param_y="roll-d",
metric_name=o,
)
plot[0]['layout']['title'] = o
data = plot[0]['data']
lay = plot[0]['layout']
for i, d in enumerate(data):
if i > 1:
d['cliponaxis'] = False
fig = {
"data": data,
"layout": lay,
}
go.Figure(fig).write_image(name + o + ".png")
if rend: render(plot)
plot_all(gpei, objectives, name="Random fit-")
num_opt = cfg.bo.optimized
for i in range(num_opt):
log.info(f"Running GP+EI optimization trial {i + 1}/{num_opt}...")
# Reinitialize GP+EI model at each step with updated data.
batch = exp.new_trial(generator_run=gpei.gen(1))
gpei = Models.BOTORCH(experiment=exp, data=exp.eval())
if ((i + 1) % 10) == 0:
plot_all(gpei,
objectives,
name=f"optimizing {str(i + 1)}-",
rend=False)
from ax.plot.exp_utils import exp_to_df
best_arm, _ = gpei.gen(1).best_arm_predictions
best_parameters = best_arm.parameters
log.info(f"Best parameters {best_parameters}")
experiment_log = {
"Exp": exp_to_df(exp=exp),
"Cfg": cfg,
"Best_param": best_parameters,
}
log.info("Printing Parameters")
log.info(exp_to_df(exp=exp))
save_log(cfg, exp, experiment_log)
fig_learn = plot_learning(exp, cfg)
fig_learn.write_image("learning" + ".png")
fig_learn.show()
plot_all(gpei, objectives, name=f"FINAL -", rend=True)
def test_best_from_model_prediction(self):
exp = get_branin_experiment()
for _ in range(3):
sobol = Models.SOBOL(search_space=exp.search_space)
generator_run = sobol.gen(n=1)
trial = exp.new_trial(generator_run=generator_run)
trial.run()
trial.mark_completed()
exp.attach_data(exp.fetch_data())
gpei = Models.BOTORCH(experiment=exp, data=exp.lookup_data())
generator_run = gpei.gen(n=1)
trial = exp.new_trial(generator_run=generator_run)
trial.run()
trial.mark_completed()
with patch.object(
ArrayModelBridge,
"model_best_point",
return_value=(
(
Arm(
name="0_0",
parameters={"x1": -4.842811906710267, "x2": 11.887089014053345},
),
(
{"branin": 34.76260622783635},
{"branin": {"branin": 0.00028306433439807734}},
),
)
),
) as mock_model_best_point, self.assertLogs(
logger="ax.service.utils.best_point", level="WARN"
) as lg:
# Test bad model fit causes function to resort back to raw data
with patch(
"ax.service.utils.best_point.assess_model_fit",
return_value=AssessModelFitResult(
good_fit_metrics_to_fisher_score={},
bad_fit_metrics_to_fisher_score={
"branin": 0,
},
),
):
self.assertIsNotNone(get_best_parameters(exp, Models))
self.assertTrue(
any("Model fit is poor" in warning for warning in lg.output),
msg=lg.output,
)
mock_model_best_point.assert_not_called()
# Test model best point is used when fit is good
with patch(
"ax.service.utils.best_point.assess_model_fit",
return_value=AssessModelFitResult(
good_fit_metrics_to_fisher_score={
"branin": 0,
},
bad_fit_metrics_to_fisher_score={},
),
):
self.assertIsNotNone(get_best_parameters(exp, Models))
mock_model_best_point.assert_called()
# Assert the non-mocked method works correctly as well
self.assertIsNotNone(get_best_parameters(exp, Models))
