def testComputeParetoFrontierByData(self):
# Experiments with batch trials must specify trial_index or data
compute_pareto_frontier(
self.experiment,
self.metrics[0],
self.metrics[1],
data=self.experiment.fetch_data(),
absolute_metrics=[m.name for m in self.metrics],
num_points=2,
)
def testComputeParetoFrontierByTrial(self):
# Experiments with batch trials must specify trial_index or data
with self.assertRaises(UnsupportedError):
compute_pareto_frontier(
self.experiment,
self.metrics[0],
self.metrics[1],
absolute_metrics=[m.name for m in self.metrics],
)
compute_pareto_frontier(
self.experiment,
self.metrics[0],
self.metrics[1],
trial_index=0,
absolute_metrics=[m.name for m in self.metrics],
num_points=2,
)
def testOODStatusQuo(self):
# An OOD status quo arm without a trial index will raise an error
experiment = get_branin_experiment()
experiment.add_tracking_metric(
BraninMetric(name="m2", param_names=["x1", "x2"]))
metrics = list(experiment.metrics.values())
sobol = Models.SOBOL(experiment.search_space)
a = sobol.gen(5)
experiment.new_batch_trial(generator_run=a).run()
# Experiments with batch trials must specify a trial index
with self.assertRaises(UnsupportedError):
compute_pareto_frontier(
experiment,
metrics[0],
metrics[1],
absolute_metrics=[m.name for m in metrics],
)
compute_pareto_frontier(
experiment,
metrics[0],
metrics[1],
trial_index=0,
absolute_metrics=[m.name for m in metrics],
)
compute_pareto_frontier(
experiment,
metrics[0],
metrics[1],
data=experiment.fetch_data(),
absolute_metrics=[m.name for m in metrics],
)
)
generatorRun = EHVImodel.gen(1)
trial = EHVIexperiment.new_trial(generator_run=generatorRun)
trial.run()
EHVIdata = Data.from_multiple_data([EHVIdata, trial.fetch_data()])
exp_df = exp_to_df(EHVIexperiment)
outcomes = np.array(exp_df[['Accuracy', 'BitCost']], dtype=np.double)
try:
hv = observed_hypervolume(modelbridge=EHVImodel)
except:
hv = 0
print("Failed to compute hv")
EHVIhvList.append(hv)
print(f"Iteration: {i}, HV: {hv}")
EHVIoutcomes = np.array(exp_to_df(EHVIexperiment)[['Accuracy', 'BitCost']],
dtype=np.double)
frontier = compute_pareto_frontier(
experiment=EHVIexperiment,
data=EHVIexperiment.fetch_data(),
primary_objective=metric_a,
secondary_objective=metric_b,
absolute_metrics=["Accuracy", "BitCost"],
num_points=25,
)
render(plot_pareto_frontier(frontier, CI_level=0.90))