def testValidateUpdate(self):
parameter = get_choice_parameter()
parameter_sqa = self.encoder.parameter_to_sqa(parameter)
parameter2 = get_choice_parameter()
parameter2._name = 5
parameter_sqa_2 = self.encoder.parameter_to_sqa(parameter2)
with self.assertRaises(ImmutabilityError):
parameter_sqa.update(parameter_sqa_2)
def test_validation(self):
# num_arms can be positive or -1.
with self.assertRaises(ValueError):
GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_arms=5),
GenerationStep(model=Models.GPEI, num_arms=-10),
])
# only last num_arms can be -1.
with self.assertRaises(ValueError):
GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_arms=-1),
GenerationStep(model=Models.GPEI, num_arms=10),
])
exp = Experiment(
name="test",
search_space=SearchSpace(parameters=[get_choice_parameter()]))
factorial_thompson_generation_strategy = GenerationStrategy(steps=[
GenerationStep(model=Models.FACTORIAL, num_arms=1),
GenerationStep(model=Models.THOMPSON, num_arms=2),
])
self.assertTrue(
factorial_thompson_generation_strategy._uses_registered_models)
self.assertFalse(
factorial_thompson_generation_strategy.uses_non_registered_models)
with self.assertRaises(ValueError):
factorial_thompson_generation_strategy.gen(exp)
def testExperimentParameterUpdates(self):
experiment = get_experiment_with_batch_trial()
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameter).count(),
len(experiment.search_space.parameters),
)
# update a parameter
# (should perform update in place)
search_space = get_search_space()
parameter = get_choice_parameter()
parameter.add_values(["foobar"])
search_space.update_parameter(parameter)
experiment.search_space = search_space
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameter).count(),
len(experiment.search_space.parameters),
)
# add a parameter
parameter = RangeParameter(name="x1",
parameter_type=ParameterType.FLOAT,
lower=-5,
upper=10)
search_space.add_parameter(parameter)
experiment.search_space = search_space
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameter).count(),
len(experiment.search_space.parameters),
)
# remove a parameter
# (old one should be deleted)
del search_space._parameters["x1"]
experiment.search_space = search_space
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameter).count(),
len(experiment.search_space.parameters),
)
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(experiment, loaded_experiment)
def test_validation(self):
# num_trials can be positive or -1.
with self.assertRaises(UserInputError):
GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_trials=5),
GenerationStep(model=Models.GPEI, num_trials=-10),
])
# only last num_trials can be -1.
with self.assertRaises(UserInputError):
GenerationStrategy(steps=[
GenerationStep(model=Models.SOBOL, num_trials=-1),
GenerationStep(model=Models.GPEI, num_trials=10),
])
exp = Experiment(
name="test",
search_space=SearchSpace(parameters=[get_choice_parameter()]))
factorial_thompson_generation_strategy = GenerationStrategy(steps=[
GenerationStep(model=Models.FACTORIAL, num_trials=1),
GenerationStep(model=Models.THOMPSON, num_trials=2),
])
self.assertTrue(
factorial_thompson_generation_strategy._uses_registered_models)
self.assertFalse(
factorial_thompson_generation_strategy.uses_non_registered_models)
with self.assertRaises(ValueError):
factorial_thompson_generation_strategy.gen(exp)
self.assertEqual(
GenerationStep(model=sum, num_trials=1).model_name, "sum")
with self.assertRaisesRegex(UserInputError,
"Maximum parallelism should be"):
GenerationStrategy(steps=[
GenerationStep(
model=Models.SOBOL, num_trials=5, max_parallelism=-1),
GenerationStep(model=Models.GPEI, num_trials=-1),
])
