def _setupBraninExperiment(self, n: int) -> Experiment:
exp = Experiment(
name="test3",
search_space=get_branin_search_space(),
tracking_metrics=[
BraninMetric(name="b", param_names=["x1", "x2"])
],
runner=SyntheticRunner(),
)
batch = exp.new_batch_trial()
batch.add_arms_and_weights(arms=get_branin_arms(n=n, seed=0))
batch.run()
(exp.new_batch_trial().add_arms_and_weights(
arms=get_branin_arms(n=3 * n, seed=1)).run())
return exp
def testExperimentWithoutName(self) -> Experiment:
exp = Experiment(
search_space=get_branin_search_space(),
tracking_metrics=[
BraninMetric(name="b", param_names=["x1", "x2"])
],
runner=SyntheticRunner(),
)
self.assertEqual("Experiment(None)", str(exp))
batch = exp.new_batch_trial()
batch.add_arms_and_weights(arms=get_branin_arms(n=5, seed=0))
batch.run()
self.assertEqual(batch.run_metadata, {"name": "0"})
def testMTExperimentFlow(self):
self.assertTrue(self.experiment.supports_trial_type("type1"))
self.assertTrue(self.experiment.supports_trial_type("type2"))
self.assertFalse(self.experiment.supports_trial_type(None))
n = 10
arms = get_branin_arms(n=n, seed=0)
b1 = self.experiment.new_batch_trial()
b1.add_arms_and_weights(arms=arms)
self.assertEqual(b1.trial_type, "type1")
b1.run()
self.assertEqual(b1.run_metadata["dummy_metadata"], "dummy1")
self.experiment.update_runner("type2",
SyntheticRunner(dummy_metadata="dummy3"))
b2 = self.experiment.new_batch_trial()
b2.trial_type = "type2"
b2.add_arms_and_weights(arms=arms)
self.assertEqual(b2.trial_type, "type2")
b2.run()
self.assertEqual(b2.run_metadata["dummy_metadata"], "dummy3")
df = self.experiment.fetch_data().df
for _, row in df.iterrows():
# Make sure proper metric present for each batch only
self.assertEqual(row["metric_name"],
"m1" if row["trial_index"] == 0 else "m2")
arm_0_slice = df.loc[df["arm_name"] == "0_0"]
self.assertNotEqual(
float(arm_0_slice[df["trial_index"] == 0]["mean"]),
float(arm_0_slice[df["trial_index"] == 1]["mean"]),
)
self.assertEqual(len(df), 2 * n)
# Set 2 metrics to be equal
self.experiment.update_tracking_metric(BraninMetric(
"m2", ["x1", "x2"]),
trial_type="type2")
df = self.experiment.fetch_data().df
arm_0_slice = df.loc[df["arm_name"] == "0_0"]
self.assertAlmostEqual(
float(arm_0_slice[df["trial_index"] == 0]["mean"]),
float(arm_0_slice[df["trial_index"] == 1]["mean"]),
places=10,
)
def testStatusQuoSetter(self):
sq_parameters = self.experiment.status_quo.parameters
self.experiment.status_quo = None
self.assertIsNone(self.experiment.status_quo)
# Verify normal update
sq_parameters["w"] = 3.5
self.experiment.status_quo = Arm(sq_parameters)
self.assertEqual(self.experiment.status_quo.parameters["w"], 3.5)
self.assertEqual(self.experiment.status_quo.name, "status_quo")
# Verify all None values
self.experiment.status_quo = Arm(
{n: None
for n in sq_parameters.keys()})
self.assertIsNone(self.experiment.status_quo.parameters["w"])
# Try extra param
sq_parameters["a"] = 4
with self.assertRaises(ValueError):
self.experiment.status_quo = Arm(sq_parameters)
# Try wrong type
sq_parameters.pop("a")
sq_parameters["w"] = "hello"
with self.assertRaises(ValueError):
self.experiment.status_quo = Arm(sq_parameters)
# Verify arms_by_signature only contains status_quo
self.assertEqual(len(self.experiment.arms_by_signature), 1)
# Change status quo, verify still just 1 arm
sq_parameters["w"] = 3.6
self.experiment.status_quo = Arm(sq_parameters)
self.assertEqual(len(self.experiment.arms_by_signature), 1)
# Make a batch, then change exp status quo, verify 2 arms
self.experiment.new_batch_trial()
sq_parameters["w"] = 3.7
self.experiment.status_quo = Arm(sq_parameters)
self.assertEqual(len(self.experiment.arms_by_signature), 2)
self.assertEqual(self.experiment.status_quo.name, "status_quo_e0")
# Try missing param
sq_parameters.pop("w")
with self.assertRaises(ValueError):
self.experiment.status_quo = Arm(sq_parameters)
# Actually name the status quo.
exp = Experiment(
name="test3",
search_space=get_branin_search_space(),
tracking_metrics=[
BraninMetric(name="b", param_names=["x1", "x2"])
],
runner=SyntheticRunner(),
)
batch = exp.new_batch_trial()
arms = get_branin_arms(n=1, seed=0)
batch.add_arms_and_weights(arms=arms)
self.assertIsNone(exp.status_quo)
exp.status_quo = arms[0]
self.assertEqual(exp.status_quo.name, "0_0")
# Try setting sq to existing arm with different name
with self.assertRaises(ValueError):
exp.status_quo = Arm(arms[0].parameters, name="new_name")