def testExperimentParameterUpdates(self):
experiment = get_experiment_with_batch_trial()
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameter).count(),
len(experiment.search_space.parameters),
)
# update a parameter
# (should perform update in place)
search_space = get_search_space()
parameter = get_choice_parameter()
parameter.add_values(["foobar"])
search_space.update_parameter(parameter)
experiment.search_space = search_space
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameter).count(),
len(experiment.search_space.parameters),
)
# add a parameter
parameter = RangeParameter(name="x1",
parameter_type=ParameterType.FLOAT,
lower=-5,
upper=10)
search_space.add_parameter(parameter)
experiment.search_space = search_space
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameter).count(),
len(experiment.search_space.parameters),
)
# remove a parameter
# (old one should be deleted)
del search_space._parameters["x1"]
experiment.search_space = search_space
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameter).count(),
len(experiment.search_space.parameters),
)
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(experiment, loaded_experiment)
def setUp(self):
self.search_space = SearchSpace(
parameters=[
RangeParameter(
"a", lower=1, upper=3, parameter_type=ParameterType.FLOAT
),
ChoiceParameter(
"b", parameter_type=ParameterType.STRING, values=["a", "b", "c"]
),
FixedParameter("c", parameter_type=ParameterType.STRING, value="a"),
]
)
self.t = RemoveFixed(
search_space=self.search_space,
observation_features=None,
observation_data=None,
)
def get_large_ordinal_search_space(
n_ordinal_choice_parameters,
n_continuous_range_parameters) -> SearchSpace:
return SearchSpace(parameters=[ # pyre-ignore[6]
RangeParameter(
name=f"x{i}",
parameter_type=ParameterType.FLOAT,
lower=0.0,
upper=1.0,
) for i in range(n_continuous_range_parameters)
] + [ # pyre-ignore[58]
ChoiceParameter(
name=f"y{i}",
parameter_type=ParameterType.INT,
values=[2, 4, 8, 16],
) for i in range(n_ordinal_choice_parameters)
])
def parameter_from_sqa(self, parameter_sqa: SQAParameter) -> Parameter:
"""Convert SQLAlchemy Parameter to Ax Parameter."""
if parameter_sqa.domain_type == DomainType.RANGE:
if parameter_sqa.lower is None or parameter_sqa.upper is None:
raise SQADecodeError( # pragma: no cover
"`lower` and `upper` must be set for RangeParameter.")
return RangeParameter(
name=parameter_sqa.name,
parameter_type=parameter_sqa.parameter_type,
# pyre-fixme[6]: Expected `float` for 3rd param but got
# `Optional[float]`.
lower=parameter_sqa.lower,
upper=parameter_sqa.upper,
log_scale=parameter_sqa.log_scale or False,
digits=parameter_sqa.digits,
is_fidelity=parameter_sqa.is_fidelity or False,
target_value=parameter_sqa.target_value,
)
elif parameter_sqa.domain_type == DomainType.CHOICE:
if parameter_sqa.choice_values is None:
raise SQADecodeError( # pragma: no cover
"`values` must be set for ChoiceParameter.")
return ChoiceParameter(
name=parameter_sqa.name,
parameter_type=parameter_sqa.parameter_type,
# pyre-fixme[6]: Expected `List[Optional[Union[bool, float, int,
# str]]]` for 3rd param but got `Optional[List[Optional[Union[bool,
# float, int, str]]]]`.
values=parameter_sqa.choice_values,
is_fidelity=parameter_sqa.is_fidelity or False,
target_value=parameter_sqa.target_value,
)
elif parameter_sqa.domain_type == DomainType.FIXED:
# Don't throw an error if parameter_sqa.fixed_value is None;
# that might be the actual value!
return FixedParameter(
name=parameter_sqa.name,
parameter_type=parameter_sqa.parameter_type,
value=parameter_sqa.fixed_value,
is_fidelity=parameter_sqa.is_fidelity or False,
target_value=parameter_sqa.target_value,
)
else:
raise SQADecodeError(
f"Cannot decode SQAParameter because {parameter_sqa.domain_type} "
"is an invalid domain type.")
def _make_range_param(name: str, representation: TParameterRepresentation,
parameter_type: Optional[str]) -> RangeParameter:
assert "bounds" in representation, "Bounds are required for range parameters."
bounds = representation["bounds"]
assert isinstance(bounds, list) and len(bounds) == 2, (
f"Cannot parse parameter {name}: for range parameters, json representation "
"should include a list of two values, lower and upper bounds of the bounds."
)
return RangeParameter(
name=name,
parameter_type=_to_parameter_type(bounds, parameter_type, name,
"bounds"),
lower=checked_cast_to_tuple((float, int), bounds[0]),
upper=checked_cast_to_tuple((float, int), bounds[1]),
log_scale=checked_cast(bool, representation.get("log_scale", False)),
is_fidelity=checked_cast(bool,
representation.get("is_fidelity", False)),
)
def _make_range_param(name: str, representation: TParameterRepresentation,
parameter_type: Optional[str]) -> RangeParameter:
assert "bounds" in representation, "Bounds are required for range parameters."
bounds = representation["bounds"]
assert isinstance(bounds, list) and len(bounds) == 2, (
f"Cannot parse parameter {name}: for range parameters, json representation "
"should include a list of two values, lower and upper bounds of the bounds."
)
return RangeParameter(
name=name,
parameter_type=_to_parameter_type(bounds, parameter_type, name,
"bounds"),
# pyre-fixme[6]: Expected `float` for 3rd param but got
# `Optional[Union[bool, float, int, str]]`.
lower=bounds[0],
upper=bounds[1],
log_scale=representation.get("log_scale", False),
)
def testSetter(self):
new_c = SumConstraint(
parameters=[self.a, self.b], is_upper_bound=True, bound=10
)
self.ss2.add_parameter_constraints([new_c])
self.assertEqual(len(self.ss2.parameter_constraints), 2)
self.ss2.set_parameter_constraints([])
self.assertEqual(len(self.ss2.parameter_constraints), 0)
update_p = RangeParameter(
name="b", parameter_type=ParameterType.INT, lower=10, upper=20
)
self.ss2.add_parameter(self.g)
self.assertEqual(len(self.ss2.parameters), TOTAL_PARAMS + 1)
self.ss2.update_parameter(update_p)
self.assertEqual(self.ss2.parameters["b"].lower, 10)
def setUp(self):
self.search_space = SearchSpace(
parameters=[
RangeParameter("a",
lower=1,
upper=5,
parameter_type=ParameterType.INT),
ChoiceParameter("b",
parameter_type=ParameterType.STRING,
values=["a", "b", "c"]),
],
parameter_constraints=[],
)
self.t = IntRangeToChoice(
search_space=self.search_space,
observation_features=None,
observation_data=None,
)
def transform_search_space(self, search_space: SearchSpace) -> SearchSpace:
transformed_parameters: Dict[str, Parameter] = {}
for p in search_space.parameters.values():
if p.name in self.encoded_parameters:
for new_p_name in self.encoded_parameters[p.name]:
transformed_parameters[new_p_name] = RangeParameter(
name=new_p_name,
parameter_type=ParameterType.FLOAT,
lower=0,
upper=1,
)
else:
transformed_parameters[p.name] = p
return SearchSpace(
parameters=list(transformed_parameters.values()),
parameter_constraints=[
pc.clone() for pc in search_space.parameter_constraints
],
)
def testErrors(self):
t = Derelativize(search_space=None,
observation_features=None,
observation_data=None)
oc = OptimizationConfig(
objective=Objective(Metric("c")),
outcome_constraints=[
OutcomeConstraint(Metric("a"),
ComparisonOp.LEQ,
bound=2,
relative=True)
],
)
search_space = SearchSpace(
parameters=[RangeParameter("x", ParameterType.FLOAT, 0, 20)])
g = ModelBridge(search_space, None, [])
with self.assertRaises(ValueError):
t.transform_optimization_config(oc, None, None)
with self.assertRaises(ValueError):
t.transform_optimization_config(oc, g, None)
def setUp(self):
self.search_space = SearchSpace(parameters=[
RangeParameter(
"x", lower=1, upper=4, parameter_type=ParameterType.FLOAT)
])
self.training_feats = [
ObservationFeatures({"x": 1}, trial_index=0),
ObservationFeatures({"x": 2}, trial_index=0),
ObservationFeatures({"x": 3}, trial_index=1),
ObservationFeatures({"x": 4}, trial_index=2),
]
self.t = TrialAsTask(
search_space=self.search_space,
observation_features=self.training_feats,
observation_data=None,
)
self.bm = {
"bp1": {
0: "v1",
1: "v2",
2: "v3"
},
"bp2": {
0: "u1",
1: "u1",
2: "u2"
},
}
self.t2 = TrialAsTask(
search_space=self.search_space,
observation_features=self.training_feats,
observation_data=None,
config={"trial_level_map": self.bm},
)
self.t3 = TrialAsTask(
search_space=self.search_space,
observation_features=self.training_feats,
observation_data=None,
config={"trial_level_map": {}},
)
def setUp(self):
self.search_space = SearchSpace(
parameters=[
RangeParameter(
"a", lower=1, upper=3, parameter_type=ParameterType.FLOAT
),
ChoiceParameter(
"b", parameter_type=ParameterType.STRING, values=["a", "b", "c"]
),
]
)
self.observation_features = [
ObservationFeatures(parameters={"a": 2, "b": "a"}),
ObservationFeatures(parameters={"a": 3, "b": "b"}),
ObservationFeatures(parameters={"a": 3, "b": "c"}),
]
self.signature_to_parameterization = {
Arm(parameters=obsf.parameters).signature: obsf.parameters
for obsf in self.observation_features
}
self.transformed_features = [
ObservationFeatures(
parameters={"arms": Arm(parameters={"a": 2, "b": "a"}).signature}
),
ObservationFeatures(
parameters={"arms": Arm(parameters={"a": 3, "b": "b"}).signature}
),
ObservationFeatures(
parameters={"arms": Arm(parameters={"a": 3, "b": "c"}).signature}
),
]
self.t = SearchSpaceToChoice(
search_space=self.search_space,
observation_features=self.observation_features,
observation_data=None,
)
self.t2 = SearchSpaceToChoice(
search_space=self.search_space,
observation_features=[self.observation_features[0]],
observation_data=None,
)
def testTransformSearchSpace(self):
ss2 = self.search_space.clone()
ss2 = self.t.transform_search_space(ss2)
self.assertEqual(len(ss2.parameters), 1)
expected_parameter = ChoiceParameter(
name="arms",
parameter_type=ParameterType.STRING,
values=list(self.t.signature_to_parameterization.keys()),
)
self.assertEqual(ss2.parameters.get("arms"), expected_parameter)
# With use_ordered
ss2 = self.search_space.clone()
ss2 = self.t3.transform_search_space(ss2)
self.assertEqual(len(ss2.parameters), 1)
expected_parameter = ChoiceParameter(
name="arms",
parameter_type=ParameterType.STRING,
values=list(self.t.signature_to_parameterization.keys()),
is_ordered=True,
)
self.assertEqual(ss2.parameters.get("arms"), expected_parameter)
# Test error if there are fidelities
ss3 = SearchSpace(parameters=[
RangeParameter(
"a",
lower=1,
upper=3,
parameter_type=ParameterType.FLOAT,
is_fidelity=True,
target_value=3,
)
])
with self.assertRaises(ValueError):
SearchSpaceToChoice(
search_space=ss3,
observation_features=self.observation_features,
observation_data=None,
)
def _make_range_param(
name: str, representation: TParameterRepresentation, parameter_type: Optional[str]
) -> RangeParameter:
assert "bounds" in representation, "Bounds are required for range parameters."
bounds = representation["bounds"]
assert isinstance(bounds, list) and len(bounds) == 2, (
f"Cannot parse parameter {name}: for range parameters, json representation "
"should include a list of two values, lower and upper bounds of the range."
)
return RangeParameter(
name=name,
parameter_type=_to_parameter_type(bounds, parameter_type, name, "bounds"),
lower=checked_cast_to_tuple((float, int), bounds[0]),
upper=checked_cast_to_tuple((float, int), bounds[1]),
log_scale=checked_cast(bool, representation.get("log_scale", False)),
digits=representation.get("digits", None), # pyre-ignore[6]
is_fidelity=checked_cast(bool, representation.get("is_fidelity", False)),
# pyre-fixme[6]: Expected `Union[None, bool, float, int, str]` for 8th param
# but got `Union[None, List[Union[None, bool, float, int, str]], bool, float,
# int, str]`.
target_value=representation.get("target_value", None),
)
def test_GPMES(self):
"""Tests GPMES instantiation."""
exp = get_branin_experiment(with_batch=True)
with self.assertRaises(ValueError):
get_GPMES(experiment=exp, data=exp.fetch_data())
exp.trials[0].run()
gpmes = get_GPMES(experiment=exp, data=exp.fetch_data())
self.assertIsInstance(gpmes, TorchModelBridge)
# Check that .gen returns without failure
gr = gpmes.gen(n=1)
self.assertEqual(len(gr.arms), 1)
# test transform_configs with winsorization
configs = {
"Winsorize": {
"winsorization_config": WinsorizationConfig(
lower_quantile_margin=0.1,
upper_quantile_margin=0.1,
)
}
}
gpmes_win = get_GPMES(
experiment=exp, data=exp.fetch_data(), transform_configs=configs
)
self.assertIsInstance(gpmes_win, TorchModelBridge)
self.assertEqual(gpmes_win._transform_configs, configs)
# test multi-fidelity optimization
exp.parameters["x2"] = RangeParameter(
name="x2",
parameter_type=exp.parameters["x2"].parameter_type,
lower=-5.0,
upper=10.0,
is_fidelity=True,
target_value=10.0,
)
gpmes_mf = get_GPMES(experiment=exp, data=exp.fetch_data())
self.assertIsInstance(gpmes_mf, TorchModelBridge)
def setUp(self):
self.search_space = SearchSpace(parameters=[
RangeParameter(
"x", lower=1, upper=3, parameter_type=ParameterType.FLOAT),
ChoiceParameter(
"b",
parameter_type=ParameterType.FLOAT,
values=[1.0, 10.0, 100.0],
is_ordered=True,
),
ChoiceParameter(
"c",
parameter_type=ParameterType.STRING,
values=["online", "offline"],
is_task=True,
),
])
self.t = TaskEncode(
search_space=self.search_space,
observation_features=None,
observation_data=None,
)
def parameter_from_sqa(self, parameter_sqa: SQAParameter) -> Parameter:
"""Convert SQLAlchemy Parameter to Ax Parameter."""
if parameter_sqa.domain_type == DomainType.RANGE:
if parameter_sqa.lower is None or parameter_sqa.upper is None:
raise SQADecodeError( # pragma: no cover
"`lower` and `upper` must be set for RangeParameter."
)
return RangeParameter(
name=parameter_sqa.name,
parameter_type=parameter_sqa.parameter_type,
lower=parameter_sqa.lower,
upper=parameter_sqa.upper,
log_scale=parameter_sqa.log_scale or False,
digits=parameter_sqa.digits,
)
elif parameter_sqa.domain_type == DomainType.CHOICE:
if parameter_sqa.choice_values is None:
raise SQADecodeError( # pragma: no cover
"`values` must be set for ChoiceParameter."
)
return ChoiceParameter(
name=parameter_sqa.name,
parameter_type=parameter_sqa.parameter_type,
values=parameter_sqa.choice_values,
)
elif parameter_sqa.domain_type == DomainType.FIXED:
# Don't throw an error if parameter_sqa.fixed_value is None;
# that might be the actual value!
return FixedParameter(
name=parameter_sqa.name,
parameter_type=parameter_sqa.parameter_type,
value=parameter_sqa.fixed_value,
)
else:
raise SQADecodeError(
f"Cannot decode SQAParameter because {parameter_sqa.domain_type} "
"is an invalid domain type."
)
def test_ood_gen(self, _):
# Test fit_out_of_design by returning OOD candidats
exp = get_experiment_for_value()
ss = SearchSpace([RangeParameter("x", ParameterType.FLOAT, 0.0, 1.0)])
modelbridge = ModelBridge(
search_space=ss,
model=Model(),
transforms=[],
experiment=exp,
data=0,
fit_out_of_design=True,
)
obs = ObservationFeatures(parameters={"x": 3.0})
modelbridge._gen = mock.MagicMock(
"ax.modelbridge.base.ModelBridge._gen",
autospec=True,
return_value=([obs], [2], None, {}),
)
gr = modelbridge.gen(n=1)
self.assertEqual(gr.arms[0].parameters, obs.parameters)
# Test clamping arms by setting fit_out_of_design=False
modelbridge = ModelBridge(
search_space=ss,
model=Model(),
transforms=[],
experiment=exp,
data=0,
fit_out_of_design=False,
)
obs = ObservationFeatures(parameters={"x": 3.0})
modelbridge._gen = mock.MagicMock(
"ax.modelbridge.base.ModelBridge._gen",
autospec=True,
return_value=([obs], [2], None, {}),
)
gr = modelbridge.gen(n=1)
self.assertEqual(gr.arms[0].parameters, {"x": 1.0})
def transform_search_space(self, search_space: SearchSpace) -> SearchSpace:
transformed_parameters: Dict[str, Parameter] = {}
for p in search_space.parameters.values():
if p.name in self.encoded_parameters:
# TypeAssert. Only ChoiceParameters present here.
# pyre: p_ is declared to have type `ChoiceParameter` but is
# pyre-fixme[9]: used as type `Parameter`.
p_: ChoiceParameter = p
# Choice(|K|) => Range(0, K-1)
transformed_parameters[p.name] = RangeParameter(
name=p_.name,
parameter_type=ParameterType.INT,
lower=0,
upper=len(p_.values) - 1,
)
else:
transformed_parameters[p.name] = p
return SearchSpace(
parameters=list(transformed_parameters.values()),
parameter_constraints=[
pc.clone() for pc in search_space.parameter_constraints
],
)
def setUp(self):
self.obsd1 = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([1.0, 2.0, 8.0]),
covariance=np.array([[1.0, 0.2, 0.4], [0.2, 2.0, 0.8], [0.4, 0.8, 3.0]]),
)
self.obsd2 = ObservationData(
metric_names=["m1", "m1", "m2", "m2"],
means=np.array([1.0, 5.0, 2.0, 1.0]),
covariance=np.array(
[
[1.0, 0.0, 0.0, 0.0],
[0.0, 1.0, 0.2, 0.4],
[0.0, 0.2, 2.0, 0.8],
[0.0, 0.4, 0.8, 3.0],
]
),
)
self.search_space = SearchSpace(
parameters=[
RangeParameter(
name="x", parameter_type=ParameterType.FLOAT, lower=0, upper=10
),
ChoiceParameter(
name="z", parameter_type=ParameterType.STRING, values=["a", "b"]
),
]
)
self.obsf1 = ObservationFeatures({"x": 2, "z": "a"})
self.obsf2 = ObservationFeatures({"x": 5, "z": "b"})
self.t = StratifiedStandardizeY(
search_space=self.search_space,
observation_features=[self.obsf1, self.obsf2],
observation_data=[self.obsd1, self.obsd2],
config={"parameter_name": "z"},
)
def transform_search_space(self, search_space: SearchSpace) -> SearchSpace:
transformed_parameters: Dict[str, Parameter] = {}
for p_name, p in search_space.parameters.items():
if p_name in self.encoded_parameters:
if p.is_fidelity:
raise ValueError(
f"Cannot one-hot-encode fidelity parameter {p_name}")
for new_p_name in self.encoded_parameters[p_name]:
transformed_parameters[new_p_name] = RangeParameter(
name=new_p_name,
parameter_type=ParameterType.FLOAT,
lower=0,
upper=1,
)
else:
transformed_parameters[p_name] = p
return SearchSpace(
parameters=list(transformed_parameters.values()),
parameter_constraints=[
pc.clone_with_transformed_parameters(
transformed_parameters=transformed_parameters)
for pc in search_space.parameter_constraints
],
)
def test_create_experiment(self) -> None:
"""Test basic experiment creation."""
ax_client = AxClient(
GenerationStrategy(
steps=[GenerationStep(model=Models.SOBOL, num_trials=30)]))
with self.assertRaisesRegex(ValueError,
"Experiment not set on Ax client"):
ax_client.experiment
ax_client.create_experiment(
name="test_experiment",
parameters=[
{
"name": "x",
"type": "range",
"bounds": [0.001, 0.1],
"value_type": "float",
"log_scale": True,
},
{
"name": "y",
"type": "choice",
"values": [1, 2, 3],
"value_type": "int",
"is_ordered": True,
},
{
"name": "x3",
"type": "fixed",
"value": 2,
"value_type": "int"
},
{
"name": "x4",
"type": "range",
"bounds": [1.0, 3.0],
"value_type": "int",
},
{
"name": "x5",
"type": "choice",
"values": ["one", "two", "three"],
"value_type": "str",
},
{
"name": "x6",
"type": "range",
"bounds": [1.0, 3.0],
"value_type": "int",
},
],
objective_name="test_objective",
minimize=True,
outcome_constraints=["some_metric >= 3", "some_metric <= 4.0"],
parameter_constraints=["x4 <= x6"],
)
assert ax_client._experiment is not None
self.assertEqual(ax_client._experiment, ax_client.experiment)
self.assertEqual(
ax_client._experiment.search_space.parameters["x"],
RangeParameter(
name="x",
parameter_type=ParameterType.FLOAT,
lower=0.001,
upper=0.1,
log_scale=True,
),
)
self.assertEqual(
ax_client._experiment.search_space.parameters["y"],
ChoiceParameter(
name="y",
parameter_type=ParameterType.INT,
values=[1, 2, 3],
is_ordered=True,
),
)
self.assertEqual(
ax_client._experiment.search_space.parameters["x3"],
FixedParameter(name="x3",
parameter_type=ParameterType.INT,
value=2),
)
self.assertEqual(
ax_client._experiment.search_space.parameters["x4"],
RangeParameter(name="x4",
parameter_type=ParameterType.INT,
lower=1.0,
upper=3.0),
)
self.assertEqual(
ax_client._experiment.search_space.parameters["x5"],
ChoiceParameter(
name="x5",
parameter_type=ParameterType.STRING,
values=["one", "two", "three"],
),
)
self.assertEqual(
ax_client._experiment.optimization_config.outcome_constraints[0],
OutcomeConstraint(
metric=Metric(name="some_metric"),
op=ComparisonOp.GEQ,
bound=3.0,
relative=False,
),
)
self.assertEqual(
ax_client._experiment.optimization_config.outcome_constraints[1],
OutcomeConstraint(
metric=Metric(name="some_metric"),
op=ComparisonOp.LEQ,
bound=4.0,
relative=False,
),
)
self.assertTrue(
ax_client._experiment.optimization_config.objective.minimize)
def get_range_parameter2() -> RangeParameter:
return RangeParameter(name="x", parameter_type=ParameterType.INT, lower=1, upper=10)
def get_search_space_for_range_value(min: float = 3.0, max: float = 6.0) -> SearchSpace:
return SearchSpace([RangeParameter("x", ParameterType.FLOAT, min, max)])
def get_small_discrete_search_space() -> SearchSpace:
return SearchSpace([
RangeParameter("x", ParameterType.INT, 0, 1),
ChoiceParameter("y", ParameterType.STRING, ["red", "panda"]),
])
def test_constraint_from_str(self):
with self.assertRaisesRegex(ValueError, "Bound for the constraint"):
constraint_from_str("x1 + x2 <= not_numerical_bound", {
"x1": None,
"x2": None
})
with self.assertRaisesRegex(ValueError, "Outcome constraint bound"):
outcome_constraint_from_str("m1 <= not_numerical_bound")
three_val_constaint = constraint_from_str(
"x1 + x2 + x3 <= 3",
{
"x1":
RangeParameter(name="x1",
parameter_type=ParameterType.FLOAT,
lower=0.1,
upper=2.0),
"x2":
RangeParameter(name="x2",
parameter_type=ParameterType.FLOAT,
lower=0.1,
upper=2.0),
"x3":
RangeParameter(name="x3",
parameter_type=ParameterType.FLOAT,
lower=0.1,
upper=2.0),
},
)
self.assertEqual(three_val_constaint.bound, 3.0)
with self.assertRaisesRegex(ValueError, "Parameter constraint should"):
constraint_from_str("x1 + x2 + <= 3", {
"x1": None,
"x2": None,
"x3": None
})
with self.assertRaisesRegex(ValueError, "Parameter constraint should"):
constraint_from_str("x1 + x2 + x3 = 3", {
"x1": None,
"x2": None,
"x3": None
})
three_val_constaint2 = constraint_from_str(
"-x1 + 2.1*x2 - 4*x3 <= 3",
{
"x1":
RangeParameter(name="x1",
parameter_type=ParameterType.FLOAT,
lower=0.1,
upper=4.0),
"x2":
RangeParameter(name="x2",
parameter_type=ParameterType.FLOAT,
lower=0.1,
upper=4.0),
"x3":
RangeParameter(name="x3",
parameter_type=ParameterType.FLOAT,
lower=0.1,
upper=4.0),
},
)
self.assertEqual(three_val_constaint2.bound, 3.0)
self.assertEqual(three_val_constaint2.constraint_dict, {
"x1": -1.0,
"x2": 2.1,
"x3": -4.0
})
with self.assertRaisesRegex(ValueError, "Multiplier should be float"):
constraint_from_str("x1 - e*x2 + x3 <= 3", {
"x1": None,
"x2": None,
"x3": None
})
with self.assertRaisesRegex(ValueError,
"A linear constraint should be"):
constraint_from_str("x1 - 2 *x2 + 3 *x3 <= 3", {
"x1": None,
"x2": None,
"x3": None
})
with self.assertRaisesRegex(ValueError,
"A linear constraint should be"):
constraint_from_str("x1 - 2* x2 + 3* x3 <= 3", {
"x1": None,
"x2": None,
"x3": None
})
with self.assertRaisesRegex(ValueError,
"A linear constraint should be"):
constraint_from_str("x1 - 2 * x2 + 3*x3 <= 3", {
"x1": None,
"x2": None,
"x3": None
})
def setUp(self):
self.a = RangeParameter(name="a",
parameter_type=ParameterType.FLOAT,
lower=0.5,
upper=5.5)
self.b = RangeParameter(name="b",
parameter_type=ParameterType.INT,
lower=2,
upper=10)
self.c = ChoiceParameter(name="c",
parameter_type=ParameterType.STRING,
values=["foo", "bar", "baz"])
self.d = FixedParameter(name="d",
parameter_type=ParameterType.BOOL,
value=True)
self.e = ChoiceParameter(name="e",
parameter_type=ParameterType.FLOAT,
values=[0.0, 0.1, 0.2, 0.5])
self.f = RangeParameter(
name="f",
parameter_type=ParameterType.INT,
lower=2,
upper=10,
log_scale=True,
)
self.g = RangeParameter(name="g",
parameter_type=ParameterType.FLOAT,
lower=0.0,
upper=1.0)
self.parameters = [self.a, self.b, self.c, self.d, self.e, self.f]
self.ss1 = SearchSpace(parameters=self.parameters)
self.ss2 = SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=self.a, upper_parameter=self.b)
],
)
self.ss1_repr = (
"SearchSpace("
"parameters=["
"RangeParameter(name='a', parameter_type=FLOAT, range=[0.5, 5.5]), "
"RangeParameter(name='b', parameter_type=INT, range=[2, 10]), "
"ChoiceParameter(name='c', parameter_type=STRING, "
"values=['foo', 'bar', 'baz']), "
"FixedParameter(name='d', parameter_type=BOOL, value=True), "
"ChoiceParameter(name='e', parameter_type=FLOAT, "
"values=[0.0, 0.1, 0.2, 0.5]), "
"RangeParameter(name='f', parameter_type=INT, range=[2, 10], "
"log_scale=True)], "
"parameter_constraints=[])")
self.ss2_repr = (
"SearchSpace("
"parameters=["
"RangeParameter(name='a', parameter_type=FLOAT, range=[0.5, 5.5]), "
"RangeParameter(name='b', parameter_type=INT, range=[2, 10]), "
"ChoiceParameter(name='c', parameter_type=STRING, "
"values=['foo', 'bar', 'baz']), "
"FixedParameter(name='d', parameter_type=BOOL, value=True), "
"ChoiceParameter(name='e', parameter_type=FLOAT, "
"values=[0.0, 0.1, 0.2, 0.5]), "
"RangeParameter(name='f', parameter_type=INT, range=[2, 10], "
"log_scale=True)], "
"parameter_constraints=[OrderConstraint(a <= b)])")
class SearchSpaceTest(TestCase):
def setUp(self):
self.a = RangeParameter(name="a",
parameter_type=ParameterType.FLOAT,
lower=0.5,
upper=5.5)
self.b = RangeParameter(name="b",
parameter_type=ParameterType.INT,
lower=2,
upper=10)
self.c = ChoiceParameter(name="c",
parameter_type=ParameterType.STRING,
values=["foo", "bar", "baz"])
self.d = FixedParameter(name="d",
parameter_type=ParameterType.BOOL,
value=True)
self.e = ChoiceParameter(name="e",
parameter_type=ParameterType.FLOAT,
values=[0.0, 0.1, 0.2, 0.5])
self.f = RangeParameter(
name="f",
parameter_type=ParameterType.INT,
lower=2,
upper=10,
log_scale=True,
)
self.g = RangeParameter(name="g",
parameter_type=ParameterType.FLOAT,
lower=0.0,
upper=1.0)
self.parameters = [self.a, self.b, self.c, self.d, self.e, self.f]
self.ss1 = SearchSpace(parameters=self.parameters)
self.ss2 = SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=self.a, upper_parameter=self.b)
],
)
self.ss1_repr = (
"SearchSpace("
"parameters=["
"RangeParameter(name='a', parameter_type=FLOAT, range=[0.5, 5.5]), "
"RangeParameter(name='b', parameter_type=INT, range=[2, 10]), "
"ChoiceParameter(name='c', parameter_type=STRING, "
"values=['foo', 'bar', 'baz']), "
"FixedParameter(name='d', parameter_type=BOOL, value=True), "
"ChoiceParameter(name='e', parameter_type=FLOAT, "
"values=[0.0, 0.1, 0.2, 0.5]), "
"RangeParameter(name='f', parameter_type=INT, range=[2, 10], "
"log_scale=True)], "
"parameter_constraints=[])")
self.ss2_repr = (
"SearchSpace("
"parameters=["
"RangeParameter(name='a', parameter_type=FLOAT, range=[0.5, 5.5]), "
"RangeParameter(name='b', parameter_type=INT, range=[2, 10]), "
"ChoiceParameter(name='c', parameter_type=STRING, "
"values=['foo', 'bar', 'baz']), "
"FixedParameter(name='d', parameter_type=BOOL, value=True), "
"ChoiceParameter(name='e', parameter_type=FLOAT, "
"values=[0.0, 0.1, 0.2, 0.5]), "
"RangeParameter(name='f', parameter_type=INT, range=[2, 10], "
"log_scale=True)], "
"parameter_constraints=[OrderConstraint(a <= b)])")
def testEq(self):
ss2 = SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=self.a, upper_parameter=self.b)
],
)
self.assertEqual(self.ss2, ss2)
self.assertNotEqual(self.ss1, self.ss2)
def testProperties(self):
self.assertEqual(len(self.ss1.parameters), TOTAL_PARAMS)
self.assertTrue("a" in self.ss1.parameters)
self.assertTrue(len(self.ss1.tunable_parameters), TUNABLE_PARAMS)
self.assertFalse("d" in self.ss1.tunable_parameters)
self.assertTrue(len(self.ss1.range_parameters), RANGE_PARAMS)
self.assertFalse("c" in self.ss1.range_parameters)
self.assertTrue(len(self.ss1.parameter_constraints) == 0)
self.assertTrue(len(self.ss2.parameter_constraints) == 1)
def testRepr(self):
self.assertEqual(str(self.ss2), self.ss2_repr)
self.assertEqual(str(self.ss1), self.ss1_repr)
def testSetter(self):
new_c = SumConstraint(parameters=[self.a, self.b],
is_upper_bound=True,
bound=10)
self.ss2.add_parameter_constraints([new_c])
self.assertEqual(len(self.ss2.parameter_constraints), 2)
self.ss2.set_parameter_constraints([])
self.assertEqual(len(self.ss2.parameter_constraints), 0)
update_p = RangeParameter(name="b",
parameter_type=ParameterType.INT,
lower=10,
upper=20)
self.ss2.add_parameter(self.g)
self.assertEqual(len(self.ss2.parameters), TOTAL_PARAMS + 1)
self.ss2.update_parameter(update_p)
self.assertEqual(self.ss2.parameters["b"].lower, 10)
def testBadConstruction(self):
# Duplicate parameter
with self.assertRaises(ValueError):
p1 = self.parameters + [self.parameters[0]]
SearchSpace(parameters=p1, parameter_constraints=[])
# Constraint on non-existent parameter
with self.assertRaises(ValueError):
SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=self.a,
upper_parameter=self.g)
],
)
# Vanilla Constraint on non-existent parameter
with self.assertRaises(ValueError):
SearchSpace(
parameters=self.parameters,
parameter_constraints=[
ParameterConstraint(constraint_dict={"g": 1}, bound=0)
],
)
# Constraint on non-numeric parameter
with self.assertRaises(ValueError):
SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=self.a,
upper_parameter=self.d)
],
)
# Constraint on choice parameter
with self.assertRaises(ValueError):
SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=self.a,
upper_parameter=self.e)
],
)
# Constraint on logscale parameter
with self.assertRaises(ValueError):
SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=self.a,
upper_parameter=self.f)
],
)
# Constraint on mismatched parameter
with self.assertRaises(ValueError):
wrong_a = self.a.clone()
wrong_a.update_range(upper=10)
SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=wrong_a,
upper_parameter=self.b)
],
)
def testBadSetter(self):
new_p = RangeParameter(name="b",
parameter_type=ParameterType.FLOAT,
lower=0.0,
upper=1.0)
# Add duplicate parameter
with self.assertRaises(ValueError):
self.ss1.add_parameter(new_p)
# Update parameter to different type
with self.assertRaises(ValueError):
self.ss1.update_parameter(new_p)
# Update non-existent parameter
new_p = RangeParameter(name="g",
parameter_type=ParameterType.FLOAT,
lower=0.0,
upper=1.0)
with self.assertRaises(ValueError):
self.ss1.update_parameter(new_p)
def testCheckMembership(self):
p_dict = {"a": 1.0, "b": 5, "c": "foo", "d": True, "e": 0.2, "f": 5}
# Valid
self.assertTrue(self.ss2.check_membership(p_dict))
# Value out of range
p_dict["a"] = 20.0
self.assertFalse(self.ss2.check_membership(p_dict))
with self.assertRaises(ValueError):
self.ss2.check_membership(p_dict, raise_error=True)
# Violate constraints
p_dict["a"] = 5.3
self.assertFalse(self.ss2.check_membership(p_dict))
with self.assertRaises(ValueError):
self.ss2.check_membership(p_dict, raise_error=True)
# Incomplete param dict
p_dict.pop("a")
self.assertFalse(self.ss2.check_membership(p_dict))
with self.assertRaises(ValueError):
self.ss2.check_membership(p_dict, raise_error=True)
# Unknown parameter
p_dict["q"] = 40
self.assertFalse(self.ss2.check_membership(p_dict))
with self.assertRaises(ValueError):
self.ss2.check_membership(p_dict, raise_error=True)
def testCheckTypes(self):
p_dict = {"a": 1.0, "b": 5, "c": "foo", "d": True, "e": 0.2, "f": 5}
# Valid
self.assertTrue(self.ss2.check_membership(p_dict))
# Invalid type
p_dict["b"] = 5.2
self.assertFalse(self.ss2.check_types(p_dict))
with self.assertRaises(ValueError):
self.ss2.check_types(p_dict, raise_error=True)
p_dict["b"] = 5
# Incomplete param dict
p_dict.pop("a")
self.assertFalse(self.ss2.check_types(p_dict))
with self.assertRaises(ValueError):
self.ss2.check_types(p_dict, raise_error=True)
# Unknown parameter
p_dict["q"] = 40
self.assertFalse(self.ss2.check_types(p_dict))
with self.assertRaises(ValueError):
self.ss2.check_types(p_dict, raise_error=True)
def testCastArm(self):
p_dict = {"a": 1.0, "b": 5.0, "c": "foo", "d": True, "e": 0.2, "f": 5}
# Check "b" parameter goes from float to int
self.assertTrue(isinstance(p_dict["b"], float))
new_arm = self.ss2.cast_arm(Arm(p_dict))
self.assertTrue(isinstance(new_arm.parameters["b"], int))
# Unknown parameter should be unchanged
p_dict["q"] = 40
new_arm = self.ss2.cast_arm(Arm(p_dict))
self.assertTrue(isinstance(new_arm.parameters["q"], int))
def testCopy(self):
a = RangeParameter("a", ParameterType.FLOAT, 1.0, 5.5)
b = RangeParameter("b", ParameterType.FLOAT, 2.0, 5.5)
c = ChoiceParameter("c", ParameterType.INT, [2, 3])
ss = SearchSpace(
parameters=[a, b, c],
parameter_constraints=[
OrderConstraint(lower_parameter=a, upper_parameter=b)
],
)
ss_copy = ss.clone()
self.assertEqual(len(ss_copy.parameters), len(ss_copy.parameters))
self.assertEqual(len(ss_copy.parameter_constraints),
len(ss_copy.parameter_constraints))
ss_copy.add_parameter(FixedParameter("d", ParameterType.STRING, "h"))
self.assertNotEqual(len(ss_copy.parameters), len(ss.parameters))
def testOutOfDesignArm(self):
arm1 = self.ss1.out_of_design_arm()
arm2 = self.ss2.out_of_design_arm()
arm1_nones = [p is None for p in arm1.parameters.values()]
self.assertTrue(all(arm1_nones))
self.assertTrue(arm1 == arm2)
def testConstructArm(self):
# Test constructing an arm of default values
arm = self.ss1.construct_arm(name="test")
self.assertEqual(arm.name, "test")
for p_name in self.ss1.parameters.keys():
self.assertTrue(p_name in arm.parameters)
self.assertEqual(arm.parameters[p_name], None)
# Test constructing an arm with a custom value
arm = self.ss1.construct_arm({"a": 1.0})
for p_name in self.ss1.parameters.keys():
self.assertTrue(p_name in arm.parameters)
if p_name == "a":
self.assertEqual(arm.parameters[p_name], 1.0)
else:
self.assertEqual(arm.parameters[p_name], None)
# Test constructing an arm with a bad param name
with self.assertRaises(ValueError):
self.ss1.construct_arm({"IDONTEXIST_a": 1.0})
# Test constructing an arm with a bad param name
with self.assertRaises(ValueError):
self.ss1.construct_arm({"a": "notafloat"})
def setUp(self):
x = RangeParameter("x", ParameterType.FLOAT, lower=0, upper=1)
y = RangeParameter("y",
ParameterType.FLOAT,
lower=1,
upper=2,
is_fidelity=True,
target_value=2)
z = RangeParameter("z", ParameterType.FLOAT, lower=0, upper=5)
self.parameters = [x, y, z]
parameter_constraints = [
OrderConstraint(x, y),
SumConstraint([x, z], False, 3.5),
]
self.search_space = SearchSpace(self.parameters, parameter_constraints)
self.observation_features = [
ObservationFeatures(parameters={
"x": 0.2,
"y": 1.2,
"z": 3
}),
ObservationFeatures(parameters={
"x": 0.4,
"y": 1.4,
"z": 3
}),
ObservationFeatures(parameters={
"x": 0.6,
"y": 1.6,
"z": 3
}),
]
self.observation_data = [
ObservationData(
metric_names=["a", "b"],
means=np.array([1.0, -1.0]),
covariance=np.array([[1.0, 4.0], [4.0, 6.0]]),
),
ObservationData(
metric_names=["a", "b"],
means=np.array([2.0, -2.0]),
covariance=np.array([[2.0, 5.0], [5.0, 7.0]]),
),
ObservationData(metric_names=["a"],
means=np.array([3.0]),
covariance=np.array([[3.0]])),
]
self.observations = [
Observation(
features=self.observation_features[i],
data=self.observation_data[i],
arm_name=str(i),
) for i in range(3)
]
self.pending_observations = {
"b":
[ObservationFeatures(parameters={
"x": 0.6,
"y": 1.6,
"z": 3
})]
}
self.model_gen_options = {"option": "yes"}
class RangeParameterTest(TestCase):
def setUp(self):
self.param1 = RangeParameter(
name="x",
parameter_type=ParameterType.FLOAT,
lower=1,
upper=3,
log_scale=True,
digits=5,
is_fidelity=True,
target_value=2,
)
self.param1_repr = (
"RangeParameter(name='x', parameter_type=FLOAT, "
"range=[1.0, 3.0], log_scale=True, digits=5, fidelity=True, target_"
"value=2.0)")
self.param2 = RangeParameter(name="y",
parameter_type=ParameterType.INT,
lower=10,
upper=15)
self.param2_repr = (
"RangeParameter(name='y', parameter_type=INT, range=[10, 15])")
def testEq(self):
param2 = RangeParameter(
name="x",
parameter_type=ParameterType.FLOAT,
lower=1,
upper=3,
log_scale=True,
digits=5,
is_fidelity=True,
target_value=2,
)
self.assertEqual(self.param1, param2)
self.assertNotEqual(self.param1, self.param2)
def testProperties(self):
self.assertEqual(self.param1.name, "x")
self.assertEqual(self.param1.parameter_type, ParameterType.FLOAT)
self.assertEqual(self.param1.lower, 1)
self.assertEqual(self.param1.upper, 3)
self.assertEqual(self.param1.digits, 5)
self.assertTrue(self.param1.log_scale)
self.assertFalse(self.param2.log_scale)
self.assertTrue(self.param1.is_numeric)
self.assertTrue(self.param1.is_fidelity)
self.assertIsNotNone(self.param1.target_value)
self.assertFalse(self.param2.is_fidelity)
self.assertIsNone(self.param2.target_value)
def testValidate(self):
self.assertFalse(self.param1.validate(None))
self.assertFalse(self.param1.validate("foo"))
self.assertTrue(self.param1.validate(1))
self.assertTrue(self.param1.validate(1.3))
def testRepr(self):
self.assertEqual(str(self.param1), self.param1_repr)
self.assertEqual(str(self.param2), self.param2_repr)
def testBadCreations(self):
with self.assertRaises(UserInputError):
RangeParameter("x", ParameterType.STRING, 1, 3)
with self.assertRaises(UserInputError):
RangeParameter("x", ParameterType.FLOAT, 3, 1)
with self.assertRaises(UserInputError):
RangeParameter("x", ParameterType.INT, 0, 1, log_scale=True)
with self.assertRaises(UserInputError):
RangeParameter("x", ParameterType.INT, 0.5, 1)
with self.assertRaises(UserInputError):
RangeParameter("x", ParameterType.INT, 0.5, 1, is_fidelity=True)
def testBadSetter(self):
with self.assertRaises(ValueError):
self.param1.update_range(upper="foo")
with self.assertRaises(ValueError):
self.param1.update_range(lower="foo")
with self.assertRaises(UserInputError):
self.param1.update_range(lower=4)
with self.assertRaises(UserInputError):
self.param1.update_range(upper=0.5)
with self.assertRaises(UserInputError):
self.param1.update_range(lower=1.0, upper=0.9)
def testGoodSetter(self):
self.param1.update_range(lower=1.0)
self.param1.update_range(upper=1.0011)
self.param1.set_log_scale(False)
self.param1.set_digits(3)
self.assertEqual(self.param1.digits, 3)
self.assertEqual(self.param1.upper, 1.001)
# This would cast Upper = Lower = 1, which is not allowed
with self.assertRaises(UserInputError):
self.param1.set_digits(1)
self.param1.update_range(lower=2.0, upper=3.0)
self.assertEqual(self.param1.lower, 2.0)
self.assertEqual(self.param1.upper, 3.0)
def testCast(self):
self.assertEqual(self.param2.cast(2.5), 2)
self.assertEqual(self.param2.cast(3), 3)
self.assertEqual(self.param2.cast(None), None)
def testClone(self):
param_clone = self.param1.clone()
self.assertEqual(self.param1.lower, param_clone.lower)
param_clone._lower = 2.0
self.assertNotEqual(self.param1.lower, param_clone.lower)
