def testBadConstruction(self):
# Duplicate parameter
with self.assertRaises(ValueError):
p1 = self.parameters + [self.parameters[0]]
SearchSpace(parameters=p1, parameter_constraints=[])
# Constraint on non-existent parameter
with self.assertRaises(ValueError):
SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=self.a,
upper_parameter=self.g)
],
)
# Vanilla Constraint on non-existent parameter
with self.assertRaises(ValueError):
SearchSpace(
parameters=self.parameters,
parameter_constraints=[
ParameterConstraint(constraint_dict={"g": 1}, bound=0)
],
)
# Constraint on non-numeric parameter
with self.assertRaises(ValueError):
SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=self.a,
upper_parameter=self.d)
],
)
# Constraint on choice parameter
with self.assertRaises(ValueError):
SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=self.a,
upper_parameter=self.e)
],
)
# Constraint on logscale parameter
with self.assertRaises(ValueError):
SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=self.a,
upper_parameter=self.f)
],
)
# Constraint on mismatched parameter
with self.assertRaises(ValueError):
wrong_a = self.a.clone()
wrong_a.update_range(upper=10)
SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=wrong_a,
upper_parameter=self.b)
],
)
def setUp(self):
x = RangeParameter("x", ParameterType.FLOAT, lower=0, upper=1)
y = RangeParameter("y",
ParameterType.FLOAT,
lower=1,
upper=2,
is_fidelity=True,
target_value=2)
z = RangeParameter("z", ParameterType.FLOAT, lower=0, upper=5)
self.parameters = [x, y, z]
parameter_constraints = [
OrderConstraint(x, y),
SumConstraint([x, z], False, 3.5),
]
self.search_space = SearchSpace(self.parameters, parameter_constraints)
self.observation_features = [
ObservationFeatures(parameters={
"x": 0.2,
"y": 1.2,
"z": 3
}),
ObservationFeatures(parameters={
"x": 0.4,
"y": 1.4,
"z": 3
}),
ObservationFeatures(parameters={
"x": 0.6,
"y": 1.6,
"z": 3
}),
]
self.observation_data = [
ObservationData(
metric_names=["a", "b"],
means=np.array([1.0, -1.0]),
covariance=np.array([[1.0, 4.0], [4.0, 6.0]]),
),
ObservationData(
metric_names=["a", "b"],
means=np.array([2.0, -2.0]),
covariance=np.array([[2.0, 5.0], [5.0, 7.0]]),
),
ObservationData(metric_names=["a"],
means=np.array([3.0]),
covariance=np.array([[3.0]])),
]
self.observations = [
Observation(
features=self.observation_features[i],
data=self.observation_data[i],
arm_name=str(i),
) for i in range(3)
]
self.pending_observations = {
"b":
[ObservationFeatures(parameters={
"x": 0.6,
"y": 1.6,
"z": 3
})]
}
self.model_gen_options = {"option": "yes"}
class SearchSpaceToChoiceTest(TestCase):
def setUp(self):
self.search_space = SearchSpace(parameters=[
RangeParameter(
"a", lower=1, upper=3, parameter_type=ParameterType.FLOAT),
ChoiceParameter("b",
parameter_type=ParameterType.STRING,
values=["a", "b", "c"]),
])
self.observation_features = [
ObservationFeatures(parameters={
"a": 2,
"b": "a"
}),
ObservationFeatures(parameters={
"a": 3,
"b": "b"
}),
ObservationFeatures(parameters={
"a": 3,
"b": "c"
}),
]
self.signature_to_parameterization = {
Arm(parameters=obsf.parameters).signature: obsf.parameters
for obsf in self.observation_features
}
self.transformed_features = [
ObservationFeatures(
parameters={
"arms": Arm(parameters={
"a": 2,
"b": "a"
}).signature
}),
ObservationFeatures(
parameters={
"arms": Arm(parameters={
"a": 3,
"b": "b"
}).signature
}),
ObservationFeatures(
parameters={
"arms": Arm(parameters={
"a": 3,
"b": "c"
}).signature
}),
]
self.t = SearchSpaceToChoice(
search_space=self.search_space,
observation_features=self.observation_features,
observation_data=None,
)
self.t2 = SearchSpaceToChoice(
search_space=self.search_space,
observation_features=[self.observation_features[0]],
observation_data=None,
)
def testTransformSearchSpace(self):
ss2 = self.search_space.clone()
ss2 = self.t.transform_search_space(ss2)
self.assertEqual(len(ss2.parameters), 1)
expected_parameter = ChoiceParameter(
name="arms",
parameter_type=ParameterType.STRING,
values=list(self.t.signature_to_parameterization.keys()),
)
self.assertEqual(ss2.parameters.get("arms"), expected_parameter)
# Test error if there are fidelities
ss3 = SearchSpace(parameters=[
RangeParameter(
"a",
lower=1,
upper=3,
parameter_type=ParameterType.FLOAT,
is_fidelity=True,
target_value=3,
)
])
with self.assertRaises(ValueError):
SearchSpaceToChoice(
search_space=ss3,
observation_features=self.observation_features,
observation_data=None,
)
def testTransformSearchSpaceWithFixedParam(self):
ss2 = self.search_space.clone()
ss2 = self.t2.transform_search_space(ss2)
self.assertEqual(len(ss2.parameters), 1)
expected_parameter = FixedParameter(
name="arms",
parameter_type=ParameterType.STRING,
value=list(self.t2.signature_to_parameterization.keys())[0],
)
self.assertEqual(ss2.parameters.get("arms"), expected_parameter)
def testTransformObservationFeatures(self):
obs_ft2 = deepcopy(self.observation_features)
obs_ft2 = self.t.transform_observation_features(obs_ft2)
self.assertEqual(obs_ft2, self.transformed_features)
obs_ft2 = self.t.untransform_observation_features(obs_ft2)
self.assertEqual(obs_ft2, self.observation_features)
def get_search_space_for_value(val: float = 3.0) -> SearchSpace:
return SearchSpace([FixedParameter("x", ParameterType.FLOAT, val)])
name="Hartmann6, D=100",
optimal_value=-3.32237,
optimization_config=OptimizationConfig(
objective=Objective(
metric=Hartmann6Metric(
name="objective",
param_names=["x19", "x14", "x43", "x37", "x66", "x3"],
noise_sd=0.0,
),
minimize=True,
)
),
search_space=SearchSpace(
parameters=[
RangeParameter(
name=f"x{i}", parameter_type=ParameterType.FLOAT, lower=0.0, upper=1.0
)
for i in range(100)
]
),
)
hartmann6_1000 = BenchmarkProblem(
name="Hartmann6, D=1000",
optimal_value=-3.32237,
optimization_config=OptimizationConfig(
objective=Objective(
metric=Hartmann6Metric(
name="objective",
param_names=["x190", "x140", "x430", "x370", "x660", "x30"],
noise_sd=0.0,
def transform_search_space(self, search_space: SearchSpace) -> SearchSpace:
new_ss = search_space.clone()
new_ss.parameters["x"]._value *= 2.0 # pyre-ignore[16]: testing hack.
return new_ss
def get_small_discrete_search_space() -> SearchSpace:
return SearchSpace([
RangeParameter("x", ParameterType.INT, 0, 1),
ChoiceParameter("y", ParameterType.STRING, ["red", "panda"]),
])
def define_problem(function_name='exp_sin_squared',
n_dimensions=None,
noise_std=0.001,
observer=None,
visualise=False,
vis_density=250,
log_dir=Path('./')):
def get_botorch_test_function(override_name=None):
"""Gets function defined in botorch test_functions.
All parameters are obtained from the define_problems scope.
"""
if override_name is None:
function_class = getattr(test_functions,
function_name.capitalize())
else:
function_class = getattr(test_functions, override_name)
if n_dimensions is not None:
function = function_class(dim=n_dimensions,
noise_std=noise_std,
negate=True)
else:
function = function_class(noise_std=noise_std, negate=True)
return function
if function_name == 'exp_sin_squared':
def function(x):
return torch.exp(-(torch.sin(3 * x)**2 + x**2))
function.dim = 1
function._bounds = [(-5, 5)]
function._optimizers = [(0, )]
function._optimal_value = 1
elif (function_name == 'ackley' or function_name == 'griewank'
or function_name == 'levy' or function_name == 'michalewicz'
or function_name == 'rastrigin' or function_name == 'rosenbrock'):
function = get_botorch_test_function()
elif function_name == 'dixonprice':
function = get_botorch_test_function(override_name='DixonPrice')
elif function_name == 'threehumpcamel':
function = get_botorch_test_function(override_name='ThreeHumpCamel')
if observer is not None:
observer.current['optimum']['x'] = torch.tensor(
function._optimizers[0])
observer.current['optimum']['function'] = function._optimal_value
if visualise:
observer.current['function'] = function
vis.function(function,
vis_start=function._bounds[0][0],
vis_end=function._bounds[0][1],
vis_density=vis_density,
function_name=function_name,
log_dir=log_dir.parents[2])
def evaluation_function(parameterisation):
return {
'function':
(function(torch.tensor(list(parameterisation.values()))).item(),
noise_std)
}
search_space = SearchSpace(parameters=[
RangeParameter(name=f'x{i}',
parameter_type=ParameterType.FLOAT,
lower=function._bounds[i][0],
upper=function._bounds[i][1])
for i in range(function.dim)
])
return evaluation_function, search_space
def testDerelativizeTransform(self, mock_predict, mock_fit,
mock_observations_from_data):
t = Derelativize(search_space=None,
observation_features=None,
observation_data=None)
# ModelBridge with in-design status quo
search_space = SearchSpace(parameters=[
RangeParameter("x", ParameterType.FLOAT, 0, 20),
RangeParameter("y", ParameterType.FLOAT, 0, 20),
])
g = ModelBridge(
search_space=search_space,
model=None,
transforms=[],
experiment=Experiment(search_space, "test"),
data=Data(),
status_quo_name="1_1",
)
# Test with no relative constraints
objective = Objective(Metric("c"))
oc = OptimizationConfig(
objective=objective,
outcome_constraints=[
OutcomeConstraint(Metric("a"),
ComparisonOp.LEQ,
bound=2,
relative=False)
],
)
oc2 = t.transform_optimization_config(oc, g, None)
self.assertTrue(oc == oc2)
# Test with relative constraint, in-design status quo
oc = OptimizationConfig(
objective=objective,
outcome_constraints=[
OutcomeConstraint(Metric("a"),
ComparisonOp.LEQ,
bound=2,
relative=False),
OutcomeConstraint(Metric("b"),
ComparisonOp.LEQ,
bound=-10,
relative=True),
],
)
oc = t.transform_optimization_config(oc, g, None)
self.assertTrue(oc.outcome_constraints == [
OutcomeConstraint(
Metric("a"), ComparisonOp.LEQ, bound=2, relative=False),
OutcomeConstraint(
Metric("b"), ComparisonOp.LEQ, bound=4.5, relative=False),
])
obsf = mock_predict.mock_calls[0][1][1][0]
obsf2 = ObservationFeatures(parameters={"x": 2.0, "y": 10.0})
self.assertTrue(obsf == obsf2)
# Test with relative constraint, out-of-design status quo
mock_predict.side_effect = Exception()
g = ModelBridge(
search_space=search_space,
model=None,
transforms=[],
experiment=Experiment(search_space, "test"),
data=Data(),
status_quo_name="1_2",
)
oc = OptimizationConfig(
objective=objective,
outcome_constraints=[
OutcomeConstraint(Metric("a"),
ComparisonOp.LEQ,
bound=2,
relative=False),
OutcomeConstraint(Metric("b"),
ComparisonOp.LEQ,
bound=-10,
relative=True),
],
)
oc = t.transform_optimization_config(oc, g, None)
self.assertTrue(oc.outcome_constraints == [
OutcomeConstraint(
Metric("a"), ComparisonOp.LEQ, bound=2, relative=False),
OutcomeConstraint(
Metric("b"), ComparisonOp.LEQ, bound=3.6, relative=False),
])
self.assertEqual(mock_predict.call_count, 2)
# Raises error if predict fails with in-design status quo
g = ModelBridge(search_space, None, [], status_quo_name="1_1")
oc = OptimizationConfig(
objective=objective,
outcome_constraints=[
OutcomeConstraint(Metric("a"),
ComparisonOp.LEQ,
bound=2,
relative=False),
OutcomeConstraint(Metric("b"),
ComparisonOp.LEQ,
bound=-10,
relative=True),
],
)
with self.assertRaises(Exception):
oc = t.transform_optimization_config(oc, g, None)
# Raises error with relative constraint, no status quo
exp = Experiment(search_space, "name")
g = ModelBridge(search_space, None, [], exp)
with self.assertRaises(ValueError):
oc = t.transform_optimization_config(oc, g, None)
# Raises error with relative constraint, no modelbridge
with self.assertRaises(ValueError):
oc = t.transform_optimization_config(oc, None, None)
def setUp(self):
self.search_space = SearchSpace(parameters=[
RangeParameter(
"a", lower=1, upper=3, parameter_type=ParameterType.FLOAT),
ChoiceParameter("b",
parameter_type=ParameterType.STRING,
values=["a", "b", "c"]),
])
self.observation_features = [
ObservationFeatures(parameters={
"a": 2,
"b": "a"
}),
ObservationFeatures(parameters={
"a": 3,
"b": "b"
}),
ObservationFeatures(parameters={
"a": 3,
"b": "c"
}),
]
self.signature_to_parameterization = {
Arm(parameters=obsf.parameters).signature: obsf.parameters
for obsf in self.observation_features
}
self.transformed_features = [
ObservationFeatures(
parameters={
"arms": Arm(parameters={
"a": 2,
"b": "a"
}).signature
}),
ObservationFeatures(
parameters={
"arms": Arm(parameters={
"a": 3,
"b": "b"
}).signature
}),
ObservationFeatures(
parameters={
"arms": Arm(parameters={
"a": 3,
"b": "c"
}).signature
}),
]
self.t = SearchSpaceToChoice(
search_space=self.search_space,
observation_features=self.observation_features,
observation_data=None,
)
self.t2 = SearchSpaceToChoice(
search_space=self.search_space,
observation_features=[self.observation_features[0]],
observation_data=None,
)
self.t3 = SearchSpaceToChoice(
search_space=self.search_space,
observation_features=self.observation_features,
observation_data=None,
config={"use_ordered": True},
)
def setUp(self):
self.search_space = SearchSpace(
parameters=[
RangeParameter("x",
lower=1,
upper=3,
parameter_type=ParameterType.FLOAT),
RangeParameter("a",
lower=1,
upper=2,
parameter_type=ParameterType.INT),
ChoiceParameter("b",
parameter_type=ParameterType.STRING,
values=["a", "b", "c"]),
ChoiceParameter("c",
parameter_type=ParameterType.BOOL,
values=[True, False]),
ChoiceParameter(
"d",
parameter_type=ParameterType.FLOAT,
values=[1.0, 10.0, 100.0],
is_ordered=True,
),
],
parameter_constraints=[
ParameterConstraint(constraint_dict={
"x": -0.5,
"a": 1
},
bound=0.5)
],
)
self.t = OneHot(
search_space=self.search_space,
observation_features=None,
observation_data=None,
)
self.t2 = OneHot(
search_space=self.search_space,
observation_features=None,
observation_data=None,
config={"rounding": "randomized"},
)
self.transformed_features = ObservationFeatures(
parameters={
"x": 2.2,
"a": 2,
"b" + OH_PARAM_INFIX + "_0": 0,
"b" + OH_PARAM_INFIX + "_1": 1,
"b" + OH_PARAM_INFIX + "_2": 0,
# Only two choices => one parameter.
"c" + OH_PARAM_INFIX: 0,
"d": 10.0,
})
self.observation_features = ObservationFeatures(parameters={
"x": 2.2,
"a": 2,
"b": "b",
"c": False,
"d": 10.0
})
hartmann6 = BenchmarkProblem(
name=hartmann6_function.name,
fbest=hartmann6_function.fmin,
optimization_config=OptimizationConfig(objective=Objective(
metric=Hartmann6Metric(
name=hartmann6_function.name,
param_names=[f"x{i}" for i in range(6)],
noise_sd=0.01,
),
minimize=True,
)),
search_space=SearchSpace(parameters=[
RangeParameter(
name=f"x{i}",
parameter_type=ParameterType.FLOAT,
lower=param_domain[0],
upper=param_domain[1],
) for i, param_domain in enumerate(hartmann6_function.domain)
]),
)
hartmann6_constrained = BenchmarkProblem(
name=hartmann6_function.name,
fbest=hartmann6_function.fmin,
optimization_config=OptimizationConfig(
objective=Objective(
metric=Hartmann6Metric(name="hartmann6",
param_names=[f"x{i}" for i in range(6)],
noise_sd=0.2),
minimize=True,
),
def testModelBridge(self, mock_fit, mock_gen_arms, mock_observations_from_data):
# Test that on init transforms are stored and applied in the correct order
transforms = [t1, t2]
exp = get_experiment()
modelbridge = ModelBridge(search_space_for_value(), 0, transforms, exp, 0)
self.assertEqual(list(modelbridge.transforms.keys()), ["t1", "t2"])
fit_args = mock_fit.mock_calls[0][2]
self.assertTrue(fit_args["search_space"] == search_space_for_value(8.0))
self.assertTrue(
fit_args["observation_features"]
== [observation1trans().features, observation2trans().features]
)
self.assertTrue(
fit_args["observation_data"]
== [observation1trans().data, observation2trans().data]
)
self.assertTrue(mock_observations_from_data.called)
# Test that transforms are applied correctly on predict
modelbridge._predict = mock.MagicMock(
"ax.modelbridge.base.ModelBridge._predict",
autospec=True,
return_value=[observation2trans().data],
)
modelbridge.predict([observation2().features])
# Observation features sent to _predict are un-transformed afterwards
modelbridge._predict.assert_called_with([observation2().features])
# Test transforms applied on gen
modelbridge._gen = mock.MagicMock(
"ax.modelbridge.base.ModelBridge._gen",
autospec=True,
return_value=([observation1trans().features], [2], None),
)
oc = OptimizationConfig(objective=Objective(metric=Metric(name="test_metric")))
modelbridge._set_kwargs_to_save(
model_key="TestModel", model_kwargs={}, bridge_kwargs={}
)
gr = modelbridge.gen(
n=1,
search_space=search_space_for_value(),
optimization_config=oc,
pending_observations={"a": [observation2().features]},
fixed_features=ObservationFeatures({"x": 5}),
)
self.assertEqual(gr._model_key, "TestModel")
modelbridge._gen.assert_called_with(
n=1,
search_space=SearchSpace([FixedParameter("x", ParameterType.FLOAT, 8.0)]),
optimization_config=oc,
pending_observations={"a": [observation2trans().features]},
fixed_features=ObservationFeatures({"x": 36}),
model_gen_options=None,
)
mock_gen_arms.assert_called_with(
arms_by_signature={}, observation_features=[observation1().features]
)
# Gen with no pending observations and no fixed features
modelbridge.gen(
n=1, search_space=search_space_for_value(), optimization_config=None
)
modelbridge._gen.assert_called_with(
n=1,
search_space=SearchSpace([FixedParameter("x", ParameterType.FLOAT, 8.0)]),
optimization_config=None,
pending_observations={},
fixed_features=ObservationFeatures({}),
model_gen_options=None,
)
# Gen with multi-objective optimization config.
oc2 = OptimizationConfig(
objective=ScalarizedObjective(
metrics=[Metric(name="test_metric"), Metric(name="test_metric_2")]
)
)
modelbridge.gen(
n=1, search_space=search_space_for_value(), optimization_config=oc2
)
modelbridge._gen.assert_called_with(
n=1,
search_space=SearchSpace([FixedParameter("x", ParameterType.FLOAT, 8.0)]),
optimization_config=oc2,
pending_observations={},
fixed_features=ObservationFeatures({}),
model_gen_options=None,
)
# Test transforms applied on cross_validate
modelbridge._cross_validate = mock.MagicMock(
"ax.modelbridge.base.ModelBridge._cross_validate",
autospec=True,
return_value=[observation1trans().data],
)
cv_training_data = [observation2()]
cv_test_points = [observation1().features]
cv_predictions = modelbridge.cross_validate(
cv_training_data=cv_training_data, cv_test_points=cv_test_points
)
modelbridge._cross_validate.assert_called_with(
obs_feats=[observation2trans().features],
obs_data=[observation2trans().data],
cv_test_points=[observation1().features], # untransformed after
)
self.assertTrue(cv_predictions == [observation1().data])
# Test stored training data
obs = modelbridge.get_training_data()
self.assertTrue(obs == [observation1(), observation2()])
self.assertEqual(modelbridge.metric_names, {"a", "b"})
self.assertIsNone(modelbridge.status_quo)
self.assertTrue(modelbridge.model_space == search_space_for_value())
self.assertEqual(modelbridge.training_in_design, [True, True])
modelbridge.training_in_design = [True, False]
with self.assertRaises(ValueError):
modelbridge.training_in_design = [True, True, False]
ood_obs = modelbridge.out_of_design_data()
self.assertTrue(ood_obs == unwrap_observation_data([observation2().data]))
def test_REMBOStrategy(self, mock_fit_gpytorch_model, mock_optimize_acqf):
# Construct a high-D test experiment with multiple metrics
hartmann_search_space = SearchSpace(parameters=[
RangeParameter(
name=f"x{i}",
parameter_type=ParameterType.FLOAT,
lower=0.0,
upper=1.0,
) for i in range(20)
])
exp = SimpleExperiment(
name="test",
search_space=hartmann_search_space,
evaluation_function=hartmann_evaluation_function,
objective_name="hartmann6",
minimize=True,
outcome_constraints=[
OutcomeConstraint(
metric=L2NormMetric(
name="l2norm",
param_names=[f"x{i}" for i in range(6)],
noise_sd=0.2,
),
op=ComparisonOp.LEQ,
bound=1.25,
relative=False,
)
],
)
# Instantiate the strategy
gs = REMBOStrategy(D=20, d=6, k=4, init_per_proj=4)
# Check that arms and data are correctly segmented by projection
exp.new_batch_trial(generator_run=gs.gen(experiment=exp, n=2))
self.assertEqual(len(gs.arms_by_proj[0]), 2)
self.assertEqual(len(gs.arms_by_proj[1]), 0)
exp.new_batch_trial(generator_run=gs.gen(experiment=exp, n=2))
self.assertEqual(len(gs.arms_by_proj[0]), 2)
self.assertEqual(len(gs.arms_by_proj[1]), 2)
# Iterate until the first projection fits a GP
for _ in range(4):
exp.new_batch_trial(generator_run=gs.gen(experiment=exp, n=2))
mock_fit_gpytorch_model.assert_not_called()
self.assertEqual(len(gs.arms_by_proj[0]), 4)
self.assertEqual(len(gs.arms_by_proj[1]), 4)
self.assertEqual(len(gs.arms_by_proj[2]), 2)
self.assertEqual(len(gs.arms_by_proj[3]), 2)
# Keep iterating until GP is used for gen
for i in range(4):
# First two trials will go towards 3rd and 4th proj. getting enough
if i < 1: # data for GP.
self.assertLess(len(gs.arms_by_proj[2]), 4)
if i < 2:
self.assertLess(len(gs.arms_by_proj[3]), 4)
exp.new_batch_trial(generator_run=gs.gen(experiment=exp, n=2))
if i < 2:
mock_fit_gpytorch_model.assert_not_called()
else:
# After all proj. have > 4 arms' worth of data, GP can be fit.
self.assertFalse(
any(len(x) < 4 for x in gs.arms_by_proj.values()))
mock_fit_gpytorch_model.assert_called()
self.assertTrue(len(gs.model_transitions) > 0)
gs2 = gs.clone_reset()
self.assertEqual(gs2.D, 20)
self.assertEqual(gs2.d, 6)
class SearchSpaceTest(TestCase):
def setUp(self):
self.a = RangeParameter(name="a",
parameter_type=ParameterType.FLOAT,
lower=0.5,
upper=5.5)
self.b = RangeParameter(name="b",
parameter_type=ParameterType.INT,
lower=2,
upper=10)
self.c = ChoiceParameter(name="c",
parameter_type=ParameterType.STRING,
values=["foo", "bar", "baz"])
self.d = FixedParameter(name="d",
parameter_type=ParameterType.BOOL,
value=True)
self.e = ChoiceParameter(name="e",
parameter_type=ParameterType.FLOAT,
values=[0.0, 0.1, 0.2, 0.5])
self.f = RangeParameter(
name="f",
parameter_type=ParameterType.INT,
lower=2,
upper=10,
log_scale=True,
)
self.g = RangeParameter(name="g",
parameter_type=ParameterType.FLOAT,
lower=0.0,
upper=1.0)
self.parameters = [self.a, self.b, self.c, self.d, self.e, self.f]
self.ss1 = SearchSpace(parameters=self.parameters)
self.ss2 = SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=self.a, upper_parameter=self.b)
],
)
self.ss1_repr = (
"SearchSpace("
"parameters=["
"RangeParameter(name='a', parameter_type=FLOAT, range=[0.5, 5.5]), "
"RangeParameter(name='b', parameter_type=INT, range=[2, 10]), "
"ChoiceParameter(name='c', parameter_type=STRING, "
"values=['foo', 'bar', 'baz']), "
"FixedParameter(name='d', parameter_type=BOOL, value=True), "
"ChoiceParameter(name='e', parameter_type=FLOAT, "
"values=[0.0, 0.1, 0.2, 0.5]), "
"RangeParameter(name='f', parameter_type=INT, range=[2, 10], "
"log_scale=True)], "
"parameter_constraints=[])")
self.ss2_repr = (
"SearchSpace("
"parameters=["
"RangeParameter(name='a', parameter_type=FLOAT, range=[0.5, 5.5]), "
"RangeParameter(name='b', parameter_type=INT, range=[2, 10]), "
"ChoiceParameter(name='c', parameter_type=STRING, "
"values=['foo', 'bar', 'baz']), "
"FixedParameter(name='d', parameter_type=BOOL, value=True), "
"ChoiceParameter(name='e', parameter_type=FLOAT, "
"values=[0.0, 0.1, 0.2, 0.5]), "
"RangeParameter(name='f', parameter_type=INT, range=[2, 10], "
"log_scale=True)], "
"parameter_constraints=[OrderConstraint(a <= b)])")
def testEq(self):
ss2 = SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=self.a, upper_parameter=self.b)
],
)
self.assertEqual(self.ss2, ss2)
self.assertNotEqual(self.ss1, self.ss2)
def testProperties(self):
self.assertEqual(len(self.ss1.parameters), TOTAL_PARAMS)
self.assertTrue("a" in self.ss1.parameters)
self.assertTrue(len(self.ss1.tunable_parameters), TUNABLE_PARAMS)
self.assertFalse("d" in self.ss1.tunable_parameters)
self.assertTrue(len(self.ss1.range_parameters), RANGE_PARAMS)
self.assertFalse("c" in self.ss1.range_parameters)
self.assertTrue(len(self.ss1.parameter_constraints) == 0)
self.assertTrue(len(self.ss2.parameter_constraints) == 1)
def testRepr(self):
self.assertEqual(str(self.ss2), self.ss2_repr)
self.assertEqual(str(self.ss1), self.ss1_repr)
def testSetter(self):
new_c = SumConstraint(parameters=[self.a, self.b],
is_upper_bound=True,
bound=10)
self.ss2.add_parameter_constraints([new_c])
self.assertEqual(len(self.ss2.parameter_constraints), 2)
self.ss2.set_parameter_constraints([])
self.assertEqual(len(self.ss2.parameter_constraints), 0)
update_p = RangeParameter(name="b",
parameter_type=ParameterType.INT,
lower=10,
upper=20)
self.ss2.add_parameter(self.g)
self.assertEqual(len(self.ss2.parameters), TOTAL_PARAMS + 1)
self.ss2.update_parameter(update_p)
self.assertEqual(self.ss2.parameters["b"].lower, 10)
def testBadConstruction(self):
# Duplicate parameter
with self.assertRaises(ValueError):
p1 = self.parameters + [self.parameters[0]]
SearchSpace(parameters=p1, parameter_constraints=[])
# Constraint on non-existent parameter
with self.assertRaises(ValueError):
SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=self.a,
upper_parameter=self.g)
],
)
# Vanilla Constraint on non-existent parameter
with self.assertRaises(ValueError):
SearchSpace(
parameters=self.parameters,
parameter_constraints=[
ParameterConstraint(constraint_dict={"g": 1}, bound=0)
],
)
# Constraint on non-numeric parameter
with self.assertRaises(ValueError):
SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=self.a,
upper_parameter=self.d)
],
)
# Constraint on choice parameter
with self.assertRaises(ValueError):
SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=self.a,
upper_parameter=self.e)
],
)
# Constraint on logscale parameter
with self.assertRaises(ValueError):
SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=self.a,
upper_parameter=self.f)
],
)
# Constraint on mismatched parameter
with self.assertRaises(ValueError):
wrong_a = self.a.clone()
wrong_a.update_range(upper=10)
SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=wrong_a,
upper_parameter=self.b)
],
)
def testBadSetter(self):
new_p = RangeParameter(name="b",
parameter_type=ParameterType.FLOAT,
lower=0.0,
upper=1.0)
# Add duplicate parameter
with self.assertRaises(ValueError):
self.ss1.add_parameter(new_p)
# Update parameter to different type
with self.assertRaises(ValueError):
self.ss1.update_parameter(new_p)
# Update non-existent parameter
new_p = RangeParameter(name="g",
parameter_type=ParameterType.FLOAT,
lower=0.0,
upper=1.0)
with self.assertRaises(ValueError):
self.ss1.update_parameter(new_p)
def testCheckMembership(self):
p_dict = {"a": 1.0, "b": 5, "c": "foo", "d": True, "e": 0.2, "f": 5}
# Valid
self.assertTrue(self.ss2.check_membership(p_dict))
# Value out of range
p_dict["a"] = 20.0
self.assertFalse(self.ss2.check_membership(p_dict))
with self.assertRaises(ValueError):
self.ss2.check_membership(p_dict, raise_error=True)
# Violate constraints
p_dict["a"] = 5.3
self.assertFalse(self.ss2.check_membership(p_dict))
with self.assertRaises(ValueError):
self.ss2.check_membership(p_dict, raise_error=True)
# Incomplete param dict
p_dict.pop("a")
self.assertFalse(self.ss2.check_membership(p_dict))
with self.assertRaises(ValueError):
self.ss2.check_membership(p_dict, raise_error=True)
# Unknown parameter
p_dict["q"] = 40
self.assertFalse(self.ss2.check_membership(p_dict))
with self.assertRaises(ValueError):
self.ss2.check_membership(p_dict, raise_error=True)
def testCheckTypes(self):
p_dict = {"a": 1.0, "b": 5, "c": "foo", "d": True, "e": 0.2, "f": 5}
# Valid
self.assertTrue(self.ss2.check_membership(p_dict))
# Invalid type
p_dict["b"] = 5.2
self.assertFalse(self.ss2.check_types(p_dict))
with self.assertRaises(ValueError):
self.ss2.check_types(p_dict, raise_error=True)
p_dict["b"] = 5
# Incomplete param dict
p_dict.pop("a")
self.assertFalse(self.ss2.check_types(p_dict))
with self.assertRaises(ValueError):
self.ss2.check_types(p_dict, raise_error=True)
# Unknown parameter
p_dict["q"] = 40
self.assertFalse(self.ss2.check_types(p_dict))
with self.assertRaises(ValueError):
self.ss2.check_types(p_dict, raise_error=True)
def testCastArm(self):
p_dict = {"a": 1.0, "b": 5.0, "c": "foo", "d": True, "e": 0.2, "f": 5}
# Check "b" parameter goes from float to int
self.assertTrue(isinstance(p_dict["b"], float))
new_arm = self.ss2.cast_arm(Arm(p_dict))
self.assertTrue(isinstance(new_arm.parameters["b"], int))
# Unknown parameter should be unchanged
p_dict["q"] = 40
new_arm = self.ss2.cast_arm(Arm(p_dict))
self.assertTrue(isinstance(new_arm.parameters["q"], int))
def testCopy(self):
a = RangeParameter("a", ParameterType.FLOAT, 1.0, 5.5)
b = RangeParameter("b", ParameterType.FLOAT, 2.0, 5.5)
c = ChoiceParameter("c", ParameterType.INT, [2, 3])
ss = SearchSpace(
parameters=[a, b, c],
parameter_constraints=[
OrderConstraint(lower_parameter=a, upper_parameter=b)
],
)
ss_copy = ss.clone()
self.assertEqual(len(ss_copy.parameters), len(ss_copy.parameters))
self.assertEqual(len(ss_copy.parameter_constraints),
len(ss_copy.parameter_constraints))
ss_copy.add_parameter(FixedParameter("d", ParameterType.STRING, "h"))
self.assertNotEqual(len(ss_copy.parameters), len(ss.parameters))
def testOutOfDesignArm(self):
arm1 = self.ss1.out_of_design_arm()
arm2 = self.ss2.out_of_design_arm()
arm1_nones = [p is None for p in arm1.parameters.values()]
self.assertTrue(all(arm1_nones))
self.assertTrue(arm1 == arm2)
def testConstructArm(self):
# Test constructing an arm of default values
arm = self.ss1.construct_arm(name="test")
self.assertEqual(arm.name, "test")
for p_name in self.ss1.parameters.keys():
self.assertTrue(p_name in arm.parameters)
self.assertEqual(arm.parameters[p_name], None)
# Test constructing an arm with a custom value
arm = self.ss1.construct_arm({"a": 1.0})
for p_name in self.ss1.parameters.keys():
self.assertTrue(p_name in arm.parameters)
if p_name == "a":
self.assertEqual(arm.parameters[p_name], 1.0)
else:
self.assertEqual(arm.parameters[p_name], None)
# Test constructing an arm with a bad param name
with self.assertRaises(ValueError):
self.ss1.construct_arm({"IDONTEXIST_a": 1.0})
# Test constructing an arm with a bad param name
with self.assertRaises(ValueError):
self.ss1.construct_arm({"a": "notafloat"})
def testModelBridge(self, mock_fit, mock_gen_arms,
mock_observations_from_data):
# Test that on init transforms are stored and applied in the correct order
transforms = [transform_1, transform_2]
exp = get_experiment_for_value()
ss = get_search_space_for_value()
modelbridge = ModelBridge(
search_space=ss,
model=Model(),
transforms=transforms,
experiment=exp,
data=0,
)
self.assertFalse(
modelbridge._experiment_has_immutable_search_space_and_opt_config)
self.assertEqual(list(modelbridge.transforms.keys()),
["Cast", "transform_1", "transform_2"])
fit_args = mock_fit.mock_calls[0][2]
self.assertTrue(
fit_args["search_space"] == get_search_space_for_value(8.0))
self.assertTrue(fit_args["observation_features"] == [])
self.assertTrue(fit_args["observation_data"] == [])
self.assertTrue(mock_observations_from_data.called)
# Test prediction on out of design features.
modelbridge._predict = mock.MagicMock(
"ax.modelbridge.base.ModelBridge._predict",
autospec=True,
side_effect=ValueError("Out of Design"),
)
# This point is in design, and thus failures in predict are legitimate.
with mock.patch.object(ModelBridge,
"model_space",
return_value=get_search_space_for_range_values):
with self.assertRaises(ValueError):
modelbridge.predict([get_observation2().features])
# This point is out of design, and not in training data.
with self.assertRaises(ValueError):
modelbridge.predict([get_observation_status_quo0().features])
# Now it's in the training data.
with mock.patch.object(
ModelBridge,
"get_training_data",
return_value=[get_observation_status_quo0()],
):
# Return raw training value.
self.assertEqual(
modelbridge.predict([get_observation_status_quo0().features]),
unwrap_observation_data([get_observation_status_quo0().data]),
)
# Test that transforms are applied correctly on predict
modelbridge._predict = mock.MagicMock(
"ax.modelbridge.base.ModelBridge._predict",
autospec=True,
return_value=[get_observation2trans().data],
)
modelbridge.predict([get_observation2().features])
# Observation features sent to _predict are un-transformed afterwards
modelbridge._predict.assert_called_with([get_observation2().features])
# Check that _single_predict is equivalent here.
modelbridge._single_predict([get_observation2().features])
# Observation features sent to _predict are un-transformed afterwards
modelbridge._predict.assert_called_with([get_observation2().features])
# Test transforms applied on gen
modelbridge._gen = mock.MagicMock(
"ax.modelbridge.base.ModelBridge._gen",
autospec=True,
return_value=([get_observation1trans().features], [2], None, {}),
)
oc = OptimizationConfig(objective=Objective(metric=Metric(
name="test_metric")))
modelbridge._set_kwargs_to_save(model_key="TestModel",
model_kwargs={},
bridge_kwargs={})
gr = modelbridge.gen(
n=1,
search_space=get_search_space_for_value(),
optimization_config=oc,
pending_observations={"a": [get_observation2().features]},
fixed_features=ObservationFeatures({"x": 5}),
)
self.assertEqual(gr._model_key, "TestModel")
modelbridge._gen.assert_called_with(
n=1,
search_space=SearchSpace(
[FixedParameter("x", ParameterType.FLOAT, 8.0)]),
optimization_config=oc,
pending_observations={"a": [get_observation2trans().features]},
fixed_features=ObservationFeatures({"x": 36}),
model_gen_options=None,
)
mock_gen_arms.assert_called_with(
arms_by_signature={},
observation_features=[get_observation1().features])
# Gen with no pending observations and no fixed features
modelbridge.gen(n=1,
search_space=get_search_space_for_value(),
optimization_config=None)
modelbridge._gen.assert_called_with(
n=1,
search_space=SearchSpace(
[FixedParameter("x", ParameterType.FLOAT, 8.0)]),
optimization_config=None,
pending_observations={},
fixed_features=ObservationFeatures({}),
model_gen_options=None,
)
# Gen with multi-objective optimization config.
oc2 = OptimizationConfig(objective=ScalarizedObjective(
metrics=[Metric(name="test_metric"),
Metric(name="test_metric_2")]))
modelbridge.gen(n=1,
search_space=get_search_space_for_value(),
optimization_config=oc2)
modelbridge._gen.assert_called_with(
n=1,
search_space=SearchSpace(
[FixedParameter("x", ParameterType.FLOAT, 8.0)]),
optimization_config=oc2,
pending_observations={},
fixed_features=ObservationFeatures({}),
model_gen_options=None,
)
# Test transforms applied on cross_validate
modelbridge._cross_validate = mock.MagicMock(
"ax.modelbridge.base.ModelBridge._cross_validate",
autospec=True,
return_value=[get_observation1trans().data],
)
cv_training_data = [get_observation2()]
cv_test_points = [get_observation1().features]
cv_predictions = modelbridge.cross_validate(
cv_training_data=cv_training_data, cv_test_points=cv_test_points)
modelbridge._cross_validate.assert_called_with(
search_space=SearchSpace(
[FixedParameter("x", ParameterType.FLOAT, 8.0)]),
obs_feats=[get_observation2trans().features],
obs_data=[get_observation2trans().data],
cv_test_points=[get_observation1().features
], # untransformed after
)
self.assertTrue(cv_predictions == [get_observation1().data])
# Test stored training data
obs = modelbridge.get_training_data()
self.assertTrue(obs == [get_observation1(), get_observation2()])
self.assertEqual(modelbridge.metric_names, {"a", "b"})
self.assertIsNone(modelbridge.status_quo)
self.assertTrue(
modelbridge.model_space == get_search_space_for_value())
self.assertEqual(modelbridge.training_in_design, [False, False])
with self.assertRaises(ValueError):
modelbridge.training_in_design = [True, True, False]
with self.assertRaises(ValueError):
modelbridge.training_in_design = [True, True, False]
# Test feature_importances
with self.assertRaises(NotImplementedError):
modelbridge.feature_importances("a")
# Test transform observation features
with mock.patch(
"ax.modelbridge.base.ModelBridge._transform_observation_features",
autospec=True,
) as mock_tr:
modelbridge.transform_observation_features(
[get_observation2().features])
mock_tr.assert_called_with(modelbridge,
[get_observation2trans().features])
def setUp(self):
self.a = RangeParameter(name="a",
parameter_type=ParameterType.FLOAT,
lower=0.5,
upper=5.5)
self.b = RangeParameter(name="b",
parameter_type=ParameterType.INT,
lower=2,
upper=10)
self.c = ChoiceParameter(name="c",
parameter_type=ParameterType.STRING,
values=["foo", "bar", "baz"])
self.d = FixedParameter(name="d",
parameter_type=ParameterType.BOOL,
value=True)
self.e = ChoiceParameter(name="e",
parameter_type=ParameterType.FLOAT,
values=[0.0, 0.1, 0.2, 0.5])
self.f = RangeParameter(
name="f",
parameter_type=ParameterType.INT,
lower=2,
upper=10,
log_scale=True,
)
self.g = RangeParameter(name="g",
parameter_type=ParameterType.FLOAT,
lower=0.0,
upper=1.0)
self.parameters = [self.a, self.b, self.c, self.d, self.e, self.f]
self.ss1 = SearchSpace(parameters=self.parameters)
self.ss2 = SearchSpace(
parameters=self.parameters,
parameter_constraints=[
OrderConstraint(lower_parameter=self.a, upper_parameter=self.b)
],
)
self.ss1_repr = (
"SearchSpace("
"parameters=["
"RangeParameter(name='a', parameter_type=FLOAT, range=[0.5, 5.5]), "
"RangeParameter(name='b', parameter_type=INT, range=[2, 10]), "
"ChoiceParameter(name='c', parameter_type=STRING, "
"values=['foo', 'bar', 'baz']), "
"FixedParameter(name='d', parameter_type=BOOL, value=True), "
"ChoiceParameter(name='e', parameter_type=FLOAT, "
"values=[0.0, 0.1, 0.2, 0.5]), "
"RangeParameter(name='f', parameter_type=INT, range=[2, 10], "
"log_scale=True)], "
"parameter_constraints=[])")
self.ss2_repr = (
"SearchSpace("
"parameters=["
"RangeParameter(name='a', parameter_type=FLOAT, range=[0.5, 5.5]), "
"RangeParameter(name='b', parameter_type=INT, range=[2, 10]), "
"ChoiceParameter(name='c', parameter_type=STRING, "
"values=['foo', 'bar', 'baz']), "
"FixedParameter(name='d', parameter_type=BOOL, value=True), "
"ChoiceParameter(name='e', parameter_type=FLOAT, "
"values=[0.0, 0.1, 0.2, 0.5]), "
"RangeParameter(name='f', parameter_type=INT, range=[2, 10], "
"log_scale=True)], "
"parameter_constraints=[OrderConstraint(a <= b)])")
def testGen(self, mock_init):
# Test with constraints
optimization_config = OptimizationConfig(
objective=Objective(Metric("a"), minimize=True),
outcome_constraints=[
OutcomeConstraint(Metric("b"), ComparisonOp.GEQ, 2, False)
],
)
ma = DiscreteModelBridge()
model = mock.MagicMock(DiscreteModel, autospec=True, instance=True)
model.gen.return_value = ([[0.0, 2.0, 3.0], [1.0, 1.0,
3.0]], [1.0, 2.0])
ma.model = model
ma.parameters = ["x", "y", "z"]
ma.outcomes = ["a", "b"]
observation_features, weights, best_observation = ma._gen(
n=3,
search_space=self.search_space,
optimization_config=optimization_config,
pending_observations=self.pending_observations,
fixed_features=ObservationFeatures({}),
model_gen_options=self.model_gen_options,
)
gen_args = model.gen.mock_calls[0][2]
self.assertEqual(gen_args["n"], 3)
self.assertEqual(gen_args["parameter_values"],
[[0.0, 1.0], ["foo", "bar"], [True]])
self.assertTrue(
np.array_equal(gen_args["objective_weights"], np.array([-1.0,
0.0])))
self.assertTrue(
np.array_equal(gen_args["outcome_constraints"][0],
np.array([[0.0, -1.0]])))
self.assertTrue(
np.array_equal(gen_args["outcome_constraints"][1],
np.array([[-2]])))
self.assertEqual(gen_args["pending_observations"][0], [])
self.assertEqual(gen_args["pending_observations"][1],
[[0, "foo", True]])
self.assertEqual(gen_args["model_gen_options"], {"option": "yes"})
self.assertEqual(observation_features[0].parameters, {
"x": 0.0,
"y": 2.0,
"z": 3.0
})
self.assertEqual(observation_features[1].parameters, {
"x": 1.0,
"y": 1.0,
"z": 3.0
})
self.assertEqual(weights, [1.0, 2.0])
# Test with no constraints, no fixed feature, no pending observations
search_space = SearchSpace(self.parameters[:2])
optimization_config.outcome_constraints = []
ma.parameters = ["x", "y"]
ma._gen(
n=3,
search_space=search_space,
optimization_config=optimization_config,
pending_observations={},
fixed_features=ObservationFeatures({}),
model_gen_options={},
)
gen_args = model.gen.mock_calls[1][2]
self.assertEqual(gen_args["parameter_values"],
[[0.0, 1.0], ["foo", "bar"]])
self.assertIsNone(gen_args["outcome_constraints"])
self.assertIsNone(gen_args["pending_observations"])
# Test validation
optimization_config = OptimizationConfig(
objective=Objective(Metric("a"), minimize=False),
outcome_constraints=[
OutcomeConstraint(Metric("b"), ComparisonOp.GEQ, 2, True)
],
)
with self.assertRaises(ValueError):
ma._gen(
n=3,
search_space=search_space,
optimization_config=optimization_config,
pending_observations={},
fixed_features=ObservationFeatures({}),
model_gen_options={},
)
def __init__(
self,
search_space: SearchSpace,
model: Any,
transforms: Optional[List[Type[Transform]]] = None,
experiment: Optional[Experiment] = None,
data: Optional[Data] = None,
transform_configs: Optional[Dict[str, TConfig]] = None,
status_quo_name: Optional[str] = None,
status_quo_features: Optional[ObservationFeatures] = None,
optimization_config: Optional[OptimizationConfig] = None,
fit_out_of_design: bool = False,
) -> None:
"""
Applies transforms and fits model.
Args:
experiment: Is used to get arm parameters. Is not mutated.
search_space: Search space for fitting the model. Constraints need
not be the same ones used in gen.
data: Ax Data.
model: Interface will be specified in subclass. If model requires
initialization, that should be done prior to its use here.
transforms: List of uninitialized transform classes. Forward
transforms will be applied in this order, and untransforms in
the reverse order.
transform_configs: A dictionary from transform name to the
transform config dictionary.
status_quo_name: Name of the status quo arm. Can only be used if
Data has a single set of ObservationFeatures corresponding to
that arm.
status_quo_features: ObservationFeatures to use as status quo.
Either this or status_quo_name should be specified, not both.
optimization_config: Optimization config defining how to optimize
the model.
"""
t_fit_start = time.time()
self._metric_names: Set[str] = set()
self._training_data: List[Observation] = []
self._optimization_config: Optional[OptimizationConfig] = optimization_config
self._training_in_design: List[bool] = []
self._status_quo: Optional[Observation] = None
self._arms_by_signature: Optional[Dict[str, Arm]] = None
self.transforms: MutableMapping[str, Transform] = OrderedDict()
self._model_key: Optional[str] = None
self._model_kwargs: Optional[Dict[str, Any]] = None
self._bridge_kwargs: Optional[Dict[str, Any]] = None
self._model_space = search_space.clone()
self._raw_transforms = transforms
self._transform_configs: Optional[Dict[str, TConfig]] = transform_configs
self._fit_out_of_design = fit_out_of_design
if experiment is not None:
if self._optimization_config is None:
self._optimization_config = experiment.optimization_config
self._arms_by_signature = experiment.arms_by_signature
observations = (
# pyre-fixme[6]: Expected `Experiment` for 1st param but got `None`.
observations_from_data(experiment, data)
if experiment is not None and data is not None
else []
)
obs_feats_raw, obs_data_raw = self._set_training_data(
observations=observations, search_space=search_space
)
# Set model status quo
# NOTE: training data must be set before setting the status quo.
self._set_status_quo(
experiment=experiment,
status_quo_name=status_quo_name,
status_quo_features=status_quo_features,
)
obs_feats, obs_data, search_space = self._transform_data(
obs_feats=obs_feats_raw,
obs_data=obs_data_raw,
search_space=search_space,
transforms=transforms,
transform_configs=transform_configs,
)
# Apply terminal transform and fit
try:
self._fit(
model=model,
search_space=search_space,
observation_features=obs_feats,
observation_data=obs_data,
)
self.fit_time = time.time() - t_fit_start
self.fit_time_since_gen = float(self.fit_time)
except NotImplementedError:
self.fit_time = 0.0
self.fit_time_since_gen = 0.0
def testIsFactorial(self):
self.assertFalse(self.batch.is_factorial)
# Insufficient factors
small_experiment = Experiment(
name="small_test",
search_space=SearchSpace(
[FixedParameter("a", ParameterType.INT, 4)]),
)
small_trial = small_experiment.new_batch_trial().add_arm(Arm({"a": 4}))
self.assertFalse(small_trial.is_factorial)
new_batch_trial = self.experiment.new_batch_trial()
new_batch_trial.add_arms_and_weights(arms=[
Arm(parameters={
"w": 0.75,
"x": 1,
"y": "foo",
"z": True
}),
Arm(parameters={
"w": 0.75,
"x": 2,
"y": "foo",
"z": True
}),
Arm(parameters={
"w": 0.77,
"x": 1,
"y": "foo",
"z": True
}),
])
self.assertFalse(new_batch_trial.is_factorial)
new_batch_trial = self.experiment.new_batch_trial()
new_batch_trial.add_arms_and_weights(arms=[
Arm(parameters={
"w": 0.77,
"x": 1,
"y": "foo",
"z": True
}),
Arm(parameters={
"w": 0.77,
"x": 2,
"y": "foo",
"z": True
}),
Arm(parameters={
"w": 0.75,
"x": 1,
"y": "foo",
"z": True
}),
Arm(parameters={
"w": 0.75,
"x": 2,
"y": "foo",
"z": True
}),
])
self.assertTrue(new_batch_trial.is_factorial)
def __init__(
self,
search_space: SearchSpace,
model: Any,
transforms: Optional[List[Type[Transform]]] = None,
experiment: Optional[Experiment] = None,
data: Optional[Data] = None,
transform_configs: Optional[Dict[str, TConfig]] = None,
status_quo_name: Optional[str] = None,
status_quo_features: Optional[ObservationFeatures] = None,
optimization_config: Optional[OptimizationConfig] = None,
) -> None:
"""
Applies transforms and fits model.
Args:
experiment: Is used to get arm parameters. Is not mutated.
search_space: Search space for fitting the model. Constraints need
not be the same ones used in gen.
data: Ax Data.
model: Interface will be specified in subclass. If model requires
initialization, that should be done prior to its use here.
transforms: List of uninitialized transform classes. Forward
transforms will be applied in this order, and untransforms in
the reverse order.
transform_configs: A dictionary from transform name to the
transform config dictionary.
status_quo_name: Name of the status quo arm. Can only be used if
Data has a single set of ObservationFeatures corresponding to
that arm.
status_quo_features: ObservationFeatures to use as status quo.
Either this or status_quo_name should be specified, not both.
optimization_config: Optimization config defining how to optimize
the model.
"""
t_fit_start = time.time()
self._metric_names: Set[str] = set()
self._training_data: List[Observation] = []
self._optimization_config: Optional[
OptimizationConfig] = optimization_config
self._training_in_design: List[bool] = []
self._status_quo: Optional[Observation] = None
self._arms_by_signature: Optional[Dict[str, Arm]] = None
self.transforms: MutableMapping[str, Transform] = OrderedDict()
self._model_space = search_space.clone()
if experiment is not None:
if self._optimization_config is None:
self._optimization_config = experiment.optimization_config
self._arms_by_signature = experiment.arms_by_signature
# Get observation features and data
obs_feats: List[ObservationFeatures] = []
obs_data: List[ObservationData] = []
observations = (observations_from_data(experiment, data)
if experiment is not None and data is not None else [])
obs_feats, obs_data = self._set_training_data(observations)
# Set model status quo
if any(x is not None
for x in [experiment, status_quo_name, status_quo_features]):
self._set_status_quo(
experiment=experiment,
status_quo_name=status_quo_name,
status_quo_features=status_quo_features,
)
# Initialize transforms
if transform_configs is None:
transform_configs = {}
search_space = search_space.clone()
if transforms is not None:
for t in transforms:
t_instance = t(
search_space=search_space,
observation_features=obs_feats,
observation_data=obs_data,
config=transform_configs.get(t.__name__, None),
)
search_space = t_instance.transform_search_space(search_space)
obs_feats = t_instance.transform_observation_features(
obs_feats)
obs_data = t_instance.transform_observation_data(
obs_data, obs_feats)
self.transforms[t.__name__] = t_instance
# Apply terminal transform and fit
try:
self._fit(
model=model,
search_space=search_space,
observation_features=obs_feats,
observation_data=obs_data,
)
self.fit_time = time.time() - t_fit_start
self.fit_time_since_gen = float(self.fit_time)
except NotImplementedError:
self.fit_time = 0.0
self.fit_time_since_gen = 0.0
def testGen(self, mock_init, mock_best_point, mock_gen):
# Test with constraints
optimization_config = OptimizationConfig(
objective=Objective(Metric("a"), minimize=True),
outcome_constraints=[
OutcomeConstraint(Metric("b"), ComparisonOp.GEQ, 2, False)
],
)
ma = NumpyModelBridge()
ma.parameters = ["x", "y", "z"]
ma.outcomes = ["a", "b"]
ma.transforms = OrderedDict()
observation_features, weights, best_obsf, _ = ma._gen(
n=3,
search_space=self.search_space,
optimization_config=optimization_config,
pending_observations=self.pending_observations,
fixed_features=ObservationFeatures({"z": 3.0}),
model_gen_options=self.model_gen_options,
)
gen_args = mock_gen.mock_calls[0][2]
self.assertEqual(gen_args["n"], 3)
self.assertEqual(gen_args["bounds"], [(0.0, 1.0), (1.0, 2.0),
(0.0, 5.0)])
self.assertTrue(
np.array_equal(gen_args["objective_weights"], np.array([-1.0,
0.0])))
self.assertTrue(
np.array_equal(gen_args["outcome_constraints"][0],
np.array([[0.0, -1.0]])))
self.assertTrue(
np.array_equal(gen_args["outcome_constraints"][1],
np.array([[-2]])))
self.assertTrue(
np.array_equal(
gen_args["linear_constraints"][0],
np.array([[1.0, -1, 0.0], [-1.0, 0.0, -1.0]]),
))
self.assertTrue(
np.array_equal(gen_args["linear_constraints"][1],
np.array([[0.0], [-3.5]])))
self.assertEqual(gen_args["fixed_features"], {2: 3.0})
self.assertTrue(
np.array_equal(gen_args["pending_observations"][0], np.array([])))
self.assertTrue(
np.array_equal(gen_args["pending_observations"][1],
np.array([[0.6, 1.6, 3.0]])))
self.assertEqual(gen_args["model_gen_options"], {"option": "yes"})
self.assertEqual(observation_features[0].parameters, {
"x": 1.0,
"y": 2.0,
"z": 3.0
})
self.assertEqual(observation_features[1].parameters, {
"x": 3.0,
"y": 4.0,
"z": 3.0
})
self.assertTrue(np.array_equal(weights, np.array([1.0, 2.0])))
# Test with multiple objectives.
oc2 = OptimizationConfig(objective=ScalarizedObjective(
metrics=[Metric(name="a"), Metric(name="b")], minimize=True))
observation_features, weights, best_obsf, _ = ma._gen(
n=3,
search_space=self.search_space,
optimization_config=oc2,
pending_observations=self.pending_observations,
fixed_features=ObservationFeatures({"z": 3.0}),
model_gen_options=self.model_gen_options,
)
gen_args = mock_gen.mock_calls[1][2]
self.assertEqual(gen_args["bounds"], [(0.0, 1.0), (1.0, 2.0),
(0.0, 5.0)])
self.assertIsNone(gen_args["outcome_constraints"])
self.assertTrue(
np.array_equal(gen_args["objective_weights"],
np.array([-1.0, -1.0])))
# Test with MultiObjective (unweighted multiple objectives)
oc3 = MultiObjectiveOptimizationConfig(objective=MultiObjective(
metrics=[Metric(name="a"),
Metric(name="b", lower_is_better=True)],
minimize=True,
))
search_space = SearchSpace(self.parameters) # Unconstrained
observation_features, weights, best_obsf, _ = ma._gen(
n=3,
search_space=search_space,
optimization_config=oc3,
pending_observations=self.pending_observations,
fixed_features=ObservationFeatures({"z": 3.0}),
model_gen_options=self.model_gen_options,
)
gen_args = mock_gen.mock_calls[2][2]
self.assertEqual(gen_args["bounds"], [(0.0, 1.0), (1.0, 2.0),
(0.0, 5.0)])
self.assertIsNone(gen_args["outcome_constraints"])
self.assertTrue(
np.array_equal(gen_args["objective_weights"], np.array([1.0,
-1.0])))
# Test with no constraints, no fixed feature, no pending observations
search_space = SearchSpace(self.parameters[:2])
optimization_config.outcome_constraints = []
ma.parameters = ["x", "y"]
ma._gen(3, search_space, {}, ObservationFeatures({}), None,
optimization_config)
gen_args = mock_gen.mock_calls[3][2]
self.assertEqual(gen_args["bounds"], [(0.0, 1.0), (1.0, 2.0)])
self.assertIsNone(gen_args["outcome_constraints"])
self.assertIsNone(gen_args["linear_constraints"])
self.assertIsNone(gen_args["fixed_features"])
self.assertIsNone(gen_args["pending_observations"])
# Test validation
optimization_config = OptimizationConfig(
objective=Objective(Metric("a"), minimize=False),
outcome_constraints=[
OutcomeConstraint(Metric("b"), ComparisonOp.GEQ, 2, False)
],
)
with self.assertRaises(ValueError):
ma._gen(
n=3,
search_space=self.search_space,
optimization_config=optimization_config,
pending_observations={},
fixed_features=ObservationFeatures({}),
)
optimization_config.objective.minimize = True
optimization_config.outcome_constraints[0].relative = True
with self.assertRaises(ValueError):
ma._gen(
n=3,
search_space=self.search_space,
optimization_config=optimization_config,
pending_observations={},
fixed_features=ObservationFeatures({}),
)
def get_search_space_for_range_value(min: float = 3.0, max: float = 6.0) -> SearchSpace:
return SearchSpace([RangeParameter("x", ParameterType.FLOAT, min, max)])
def testInit(self):
self.assertEqual(
self.t.Ymean,
{
("m1", "a"): 1.0,
("m1", "b"): 3.0,
("m2", "a"): 5.0,
("m2", "b"): 1.5
},
)
self.assertEqual(
self.t.Ystd,
{
("m1", "a"): 1.0,
("m1", "b"): 2.0,
("m2", "a"): 3.0,
("m2", "b"): 0.5
},
)
with self.assertRaises(ValueError):
# No parameter specified
StratifiedStandardizeY(
search_space=self.search_space,
observation_features=[self.obsf1, self.obsf2],
observation_data=[self.obsd1, self.obsd2],
)
with self.assertRaises(ValueError):
# Wrong parameter type
StratifiedStandardizeY(
search_space=self.search_space,
observation_features=[self.obsf1, self.obsf2],
observation_data=[self.obsd1, self.obsd2],
config={"parameter_name": "x"},
)
# Multiple tasks parameters
ss3 = SearchSpace(parameters=[
RangeParameter(name="x",
parameter_type=ParameterType.FLOAT,
lower=0,
upper=10),
ChoiceParameter(
name="z",
parameter_type=ParameterType.STRING,
values=["a", "b"],
is_task=True,
),
ChoiceParameter(
name="z2",
parameter_type=ParameterType.STRING,
values=["a", "b"],
is_task=True,
),
])
with self.assertRaises(ValueError):
StratifiedStandardizeY(
search_space=ss3,
observation_features=[self.obsf1, self.obsf2],
observation_data=[self.obsd1, self.obsd2],
)
# Grab from task feature
ss2 = SearchSpace(parameters=[
RangeParameter(name="x",
parameter_type=ParameterType.FLOAT,
lower=0,
upper=10),
ChoiceParameter(
name="z",
parameter_type=ParameterType.STRING,
values=["a", "b"],
is_task=True,
),
])
t2 = StratifiedStandardizeY(
search_space=ss2,
observation_features=[self.obsf1, self.obsf2],
observation_data=[self.obsd1, self.obsd2],
)
self.assertEqual(
t2.Ymean,
{
("m1", "a"): 1.0,
("m1", "b"): 3.0,
("m2", "a"): 5.0,
("m2", "b"): 1.5
},
)
self.assertEqual(
t2.Ystd,
{
("m1", "a"): 1.0,
("m1", "b"): 2.0,
("m2", "a"): 3.0,
("m2", "b"): 0.5
},
)
