def get_factorial_experiment(
has_optimization_config: bool = True,
with_batch: bool = False,
with_status_quo: bool = False,
) -> Experiment:
exp = Experiment(
name="factorial_test_experiment",
search_space=get_factorial_search_space(),
optimization_config=OptimizationConfig(
objective=Objective(metric=get_factorial_metric())
)
if has_optimization_config
else None,
runner=SyntheticRunner(),
is_test=True,
tracking_metrics=[get_factorial_metric("secondary_metric")],
)
if with_status_quo:
exp.status_quo = Arm(
parameters={
"factor1": "level11",
"factor2": "level21",
"factor3": "level31",
}
)
if with_batch:
factorial_generator = get_factorial(search_space=exp.search_space)
factorial_run = factorial_generator.gen(n=-1)
exp.new_batch_trial(optimize_for_power=with_status_quo).add_generator_run(
factorial_run
)
return exp
def get_branin_experiment(
has_optimization_config: bool = True,
with_batch: bool = False,
with_status_quo: bool = False,
with_fidelity_parameter: bool = False,
with_choice_parameter: bool = False,
search_space: Optional[SearchSpace] = None,
) -> Experiment:
search_space = search_space or get_branin_search_space(
with_fidelity_parameter=with_fidelity_parameter,
with_choice_parameter=with_choice_parameter,
)
exp = Experiment(
name="branin_test_experiment",
search_space=search_space,
optimization_config=get_branin_optimization_config()
if has_optimization_config
else None,
runner=SyntheticRunner(),
is_test=True,
)
if with_status_quo:
exp.status_quo = Arm(parameters={"x1": 0.0, "x2": 0.0})
if with_batch:
sobol_generator = get_sobol(search_space=exp.search_space)
sobol_run = sobol_generator.gen(n=15)
exp.new_batch_trial(optimize_for_power=with_status_quo).add_generator_run(
sobol_run
)
return exp
def get_branin_experiment_with_multi_objective(
has_optimization_config: bool = True,
has_objective_thresholds: bool = False,
with_batch: bool = False,
with_status_quo: bool = False,
with_fidelity_parameter: bool = False,
num_objectives: int = 2,
) -> Experiment:
exp = Experiment(
name="branin_test_experiment",
search_space=get_branin_search_space(
with_fidelity_parameter=with_fidelity_parameter),
optimization_config=get_branin_multi_objective_optimization_config(
has_objective_thresholds=has_objective_thresholds,
num_objectives=num_objectives,
) if has_optimization_config else None,
runner=SyntheticRunner(),
is_test=True,
)
if with_status_quo:
# Experiment chooses the name "status_quo" by default
exp.status_quo = Arm(parameters={"x1": 0.0, "x2": 0.0})
if with_batch:
sobol_generator = get_sobol(search_space=exp.search_space,
seed=TEST_SOBOL_SEED)
sobol_run = sobol_generator.gen(n=5)
exp.new_batch_trial(
optimize_for_power=with_status_quo).add_generator_run(sobol_run)
return exp
def get_branin_experiment(
has_optimization_config: bool = True,
with_batch: bool = False,
with_trial: bool = False,
with_status_quo: bool = False,
with_fidelity_parameter: bool = False,
with_choice_parameter: bool = False,
with_str_choice_param: bool = False,
search_space: Optional[SearchSpace] = None,
minimize: bool = False,
named: bool = True,
with_completed_trial: bool = False,
) -> Experiment:
search_space = search_space or get_branin_search_space(
with_fidelity_parameter=with_fidelity_parameter,
with_choice_parameter=with_choice_parameter,
with_str_choice_param=with_str_choice_param,
)
exp = Experiment(
name="branin_test_experiment" if named else None,
search_space=search_space,
optimization_config=get_branin_optimization_config(
minimize=minimize) if has_optimization_config else None,
runner=SyntheticRunner(),
is_test=True,
)
if with_status_quo:
exp.status_quo = Arm(parameters={"x1": 0.0, "x2": 0.0})
if with_batch:
sobol_generator = get_sobol(search_space=exp.search_space)
sobol_run = sobol_generator.gen(n=15)
exp.new_batch_trial(
optimize_for_power=with_status_quo).add_generator_run(sobol_run)
if with_trial or with_completed_trial:
sobol_generator = get_sobol(search_space=exp.search_space)
sobol_run = sobol_generator.gen(n=1)
trial = exp.new_trial(generator_run=sobol_run)
if with_completed_trial:
trial.mark_running(no_runner_required=True)
exp.attach_data(
get_branin_data(trials=[trial])) # Add data for one trial
trial.mark_completed()
return exp
def get_branin_with_multi_task(with_multi_objective: bool = False):
exp = Experiment(
name="branin_test_experiment",
search_space=get_branin_search_space(),
optimization_config=get_branin_multi_objective_optimization_config(
has_objective_thresholds=True, )
if with_multi_objective else get_branin_optimization_config(),
runner=SyntheticRunner(),
is_test=True,
)
exp.status_quo = Arm(parameters={"x1": 0.0, "x2": 0.0}, name="status_quo")
sobol_generator = get_sobol(search_space=exp.search_space,
seed=TEST_SOBOL_SEED)
sobol_run = sobol_generator.gen(n=5)
exp.new_batch_trial(optimize_for_power=True).add_generator_run(sobol_run)
not_none(exp.trials.get(0)).run()
exp.new_batch_trial(optimize_for_power=True).add_generator_run(sobol_run)
not_none(exp.trials.get(1)).run()
return exp
def get_branin_experiment_with_timestamp_map_metric(
with_status_quo: bool = False,
rate: Optional[float] = None,
) -> Experiment:
exp = Experiment(
name="branin_with_timestamp_map_metric",
search_space=get_branin_search_space(),
optimization_config=OptimizationConfig(objective=Objective(
metric=BraninTimestampMapMetric(
name="branin_map", param_names=["x1", "x2"], rate=rate),
minimize=True,
)),
tracking_metrics=[
BraninMetric(name="branin", param_names=["x1", "x2"])
],
runner=SyntheticRunner(),
default_data_type=DataType.MAP_DATA,
)
if with_status_quo:
exp.status_quo = Arm(parameters={"x1": 0.0, "x2": 0.0})
return exp
def get_branin_experiment(
has_optimization_config: bool = True,
with_batch: bool = False,
with_status_quo: bool = False,
) -> Experiment:
exp = Experiment(
name="branin_test_experiment",
search_space=get_branin_search_space(),
optimization_config=get_branin_optimization_config()
if has_optimization_config else None,
runner=SyntheticRunner(),
is_test=True,
)
if with_status_quo:
exp.status_quo = Arm(parameters={"x1": 0, "x2": 0})
if with_batch:
sobol_generator = get_sobol(search_space=exp.search_space)
sobol_run = sobol_generator.gen(n=15)
exp.new_batch_trial().add_generator_run(sobol_run)
return exp
def testStatusQuoSetter(self):
sq_parameters = self.experiment.status_quo.parameters
self.experiment.status_quo = None
self.assertIsNone(self.experiment.status_quo)
# Verify normal update
sq_parameters["w"] = 3.5
self.experiment.status_quo = Arm(sq_parameters)
self.assertEqual(self.experiment.status_quo.parameters["w"], 3.5)
self.assertEqual(self.experiment.status_quo.name, "status_quo")
self.assertTrue("status_quo" in self.experiment.arms_by_name)
# Verify all None values
self.experiment.status_quo = Arm(
{n: None
for n in sq_parameters.keys()})
self.assertIsNone(self.experiment.status_quo.parameters["w"])
# Try extra param
sq_parameters["a"] = 4
with self.assertRaises(ValueError):
self.experiment.status_quo = Arm(sq_parameters)
# Try wrong type
sq_parameters.pop("a")
sq_parameters["w"] = "hello"
with self.assertRaises(ValueError):
self.experiment.status_quo = Arm(sq_parameters)
# Verify arms_by_signature, arms_by_name only contains status_quo
self.assertEqual(len(self.experiment.arms_by_signature), 1)
self.assertEqual(len(self.experiment.arms_by_name), 1)
# Change status quo, verify still just 1 arm
sq_parameters["w"] = 3.6
self.experiment.status_quo = Arm(sq_parameters)
self.assertEqual(len(self.experiment.arms_by_signature), 1)
self.assertEqual(len(self.experiment.arms_by_name), 1)
# Make a batch, add status quo to it, then change exp status quo, verify 2 arms
batch = self.experiment.new_batch_trial()
batch.set_status_quo_with_weight(self.experiment.status_quo, 1)
sq_parameters["w"] = 3.7
self.experiment.status_quo = Arm(sq_parameters)
self.assertEqual(len(self.experiment.arms_by_signature), 2)
self.assertEqual(len(self.experiment.arms_by_name), 2)
self.assertEqual(self.experiment.status_quo.name, "status_quo_e0")
self.assertTrue("status_quo_e0" in self.experiment.arms_by_name)
# Try missing param
sq_parameters.pop("w")
with self.assertRaises(ValueError):
self.experiment.status_quo = Arm(sq_parameters)
# Actually name the status quo.
exp = Experiment(
name="test3",
search_space=get_branin_search_space(),
tracking_metrics=[
BraninMetric(name="b", param_names=["x1", "x2"])
],
runner=SyntheticRunner(),
)
batch = exp.new_batch_trial()
arms = get_branin_arms(n=1, seed=0)
batch.add_arms_and_weights(arms=arms)
self.assertIsNone(exp.status_quo)
exp.status_quo = arms[0]
self.assertEqual(exp.status_quo.name, "0_0")
# Try setting sq to existing arm with different name
with self.assertRaises(ValueError):
exp.status_quo = Arm(arms[0].parameters, name="new_name")
