def testExperimentParameterUpdates(self):
experiment = get_experiment_with_batch_trial()
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameter).count(),
len(experiment.search_space.parameters),
)
# update a parameter
# (should perform update in place)
search_space = get_search_space()
parameter = get_choice_parameter()
parameter.add_values(["foobar"])
search_space.update_parameter(parameter)
experiment.search_space = search_space
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameter).count(),
len(experiment.search_space.parameters),
)
# add a parameter
parameter = RangeParameter(name="x1",
parameter_type=ParameterType.FLOAT,
lower=-5,
upper=10)
search_space.add_parameter(parameter)
experiment.search_space = search_space
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameter).count(),
len(experiment.search_space.parameters),
)
# remove a parameter
# (old one should be deleted)
del search_space._parameters["x1"]
experiment.search_space = search_space
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameter).count(),
len(experiment.search_space.parameters),
)
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(experiment, loaded_experiment)
def testExperimentParameterConstraintUpdates(self):
experiment = get_experiment_with_batch_trial()
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameterConstraint).count(), # 3
len(experiment.search_space.parameter_constraints), # 3
)
# add a parameter constraint
search_space = experiment.search_space
existing_constraint = experiment.search_space.parameter_constraints[0]
new_constraint = get_sum_constraint2()
search_space.add_parameter_constraints([new_constraint])
experiment.search_space = search_space
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameterConstraint).count(),
len(experiment.search_space.parameter_constraints),
)
# update a parameter constraint
# (since we don't have UIDs for these, we throw out the old one
# and create a new one)
new_constraint.bound = 5.0
search_space.set_parameter_constraints(
[existing_constraint, new_constraint])
experiment.search_space = search_space
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameterConstraint).count(),
len(experiment.search_space.parameter_constraints),
)
# remove a parameter constraint
# (old one should be deleted)
search_space.set_parameter_constraints([new_constraint])
experiment.search_space = search_space
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameterConstraint).count(),
len(experiment.search_space.parameter_constraints),
)
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(experiment, loaded_experiment)
def testExperimentOutcomeConstraintUpdates(self):
experiment = get_experiment_with_batch_trial()
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAMetric).count(), len(experiment.metrics)
)
# update outcome constraint
# (should perform update in place)
optimization_config = get_optimization_config()
outcome_constraint = get_outcome_constraint()
outcome_constraint.bound = -1.0
optimization_config.outcome_constraints = [outcome_constraint]
experiment.optimization_config = optimization_config
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAMetric).count(), len(experiment.metrics)
)
# add outcome constraint
outcome_constraint2 = OutcomeConstraint(
metric=Metric(name="outcome"), op=ComparisonOp.GEQ, bound=-0.5
)
optimization_config.outcome_constraints = [
outcome_constraint,
outcome_constraint2,
]
experiment.optimization_config = optimization_config
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAMetric).count(), len(experiment.metrics)
)
# remove outcome constraint
# (old one should become tracking metric)
optimization_config.outcome_constraints = [outcome_constraint]
experiment.optimization_config = optimization_config
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAMetric).count(), len(experiment.metrics)
)
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(experiment, loaded_experiment)
def testExperimentUpdates(self):
experiment = get_experiment_with_batch_trial()
save_experiment(experiment)
self.assertEqual(get_session().query(SQAExperiment).count(), 1)
# update experiment
# (should perform update in place)
experiment.description = "foobar"
save_experiment(experiment)
self.assertEqual(get_session().query(SQAExperiment).count(), 1)
experiment.status_quo = Arm(
parameters={"w": 0.0, "x": 1, "y": "y", "z": True}, name="new_status_quo"
)
save_experiment(experiment)
self.assertEqual(get_session().query(SQAExperiment).count(), 1)
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(experiment, loaded_experiment)
def testRegistryAdditions(self):
class MyRunner(Runner):
def run():
pass
def staging_required():
return False
class MyMetric(Metric):
pass
register_metric(MyMetric)
register_runner(MyRunner)
experiment = get_experiment_with_batch_trial()
experiment.runner = MyRunner()
experiment.add_tracking_metric(MyMetric(name="my_metric"))
save_experiment(experiment)
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(loaded_experiment, experiment)
def testExperimentGeneratorRunUpdates(self):
experiment = get_experiment_with_batch_trial()
save_experiment(experiment)
# one main generator run, one for the status quo
self.assertEqual(get_session().query(SQAGeneratorRun).count(), 2)
# add a arm
# this will create one wrapper generator run
# this will also replace the status quo generator run,
# since the weight of the status quo will have changed
trial = experiment.trials[0]
trial.add_arm(get_arm())
save_experiment(experiment)
self.assertEqual(get_session().query(SQAGeneratorRun).count(), 3)
generator_run = get_generator_run()
trial.add_generator_run(generator_run=generator_run, multiplier=0.5)
save_experiment(experiment)
self.assertEqual(get_session().query(SQAGeneratorRun).count(), 4)
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(experiment, loaded_experiment)
def test_copy_db_ids_none_search_space(self):
exp1 = get_experiment_with_batch_trial()
save_experiment(exp1)
exp2 = load_experiment(exp1.name)
self.assertEqual(exp1, exp2)
# empty search_space of exp1
exp1._search_space = None
# empty some of exp2 db_ids
exp2.trials[0].db_id = None
exp2.trials[0].generator_runs[0].arms[0].db_id = None
with self.assertWarnsRegex(
Warning,
"Encountered two objects of different types",
):
# copy db_ids from exp1 to exp2
copy_db_ids(exp1, exp2)
# empty search space of exp2 for comparison
exp2._search_space = None
self.assertEqual(exp1, exp2)
def testFailedLoad(self):
with self.assertRaises(ValueError):
load_experiment("nonexistent_experiment")
def testExperimentSaveAndLoad(self):
save_experiment(self.experiment)
loaded_experiment = load_experiment(self.experiment.name)
self.assertEqual(loaded_experiment, self.experiment)