def testExperimentParameterUpdates(self): experiment = get_experiment_with_batch_trial() save_experiment(experiment) self.assertEqual( get_session().query(SQAParameter).count(), len(experiment.search_space.parameters), ) # update a parameter # (should perform update in place) search_space = get_search_space() parameter = get_choice_parameter() parameter.add_values(["foobar"]) search_space.update_parameter(parameter) experiment.search_space = search_space save_experiment(experiment) self.assertEqual( get_session().query(SQAParameter).count(), len(experiment.search_space.parameters), ) # add a parameter parameter = RangeParameter(name="x1", parameter_type=ParameterType.FLOAT, lower=-5, upper=10) search_space.add_parameter(parameter) experiment.search_space = search_space save_experiment(experiment) self.assertEqual( get_session().query(SQAParameter).count(), len(experiment.search_space.parameters), ) # remove a parameter # (old one should be deleted) del search_space._parameters["x1"] experiment.search_space = search_space save_experiment(experiment) self.assertEqual( get_session().query(SQAParameter).count(), len(experiment.search_space.parameters), ) loaded_experiment = load_experiment(experiment.name) self.assertEqual(experiment, loaded_experiment)
def setUp(self): self.search_space = SearchSpace( parameters=[ RangeParameter( "a", lower=1, upper=3, parameter_type=ParameterType.FLOAT ), ChoiceParameter( "b", parameter_type=ParameterType.STRING, values=["a", "b", "c"] ), FixedParameter("c", parameter_type=ParameterType.STRING, value="a"), ] ) self.t = RemoveFixed( search_space=self.search_space, observation_features=None, observation_data=None, )
def get_large_ordinal_search_space( n_ordinal_choice_parameters, n_continuous_range_parameters) -> SearchSpace: return SearchSpace(parameters=[ # pyre-ignore[6] RangeParameter( name=f"x{i}", parameter_type=ParameterType.FLOAT, lower=0.0, upper=1.0, ) for i in range(n_continuous_range_parameters) ] + [ # pyre-ignore[58] ChoiceParameter( name=f"y{i}", parameter_type=ParameterType.INT, values=[2, 4, 8, 16], ) for i in range(n_ordinal_choice_parameters) ])
def parameter_from_sqa(self, parameter_sqa: SQAParameter) -> Parameter: """Convert SQLAlchemy Parameter to Ax Parameter.""" if parameter_sqa.domain_type == DomainType.RANGE: if parameter_sqa.lower is None or parameter_sqa.upper is None: raise SQADecodeError( # pragma: no cover "`lower` and `upper` must be set for RangeParameter.") return RangeParameter( name=parameter_sqa.name, parameter_type=parameter_sqa.parameter_type, # pyre-fixme[6]: Expected `float` for 3rd param but got # `Optional[float]`. lower=parameter_sqa.lower, upper=parameter_sqa.upper, log_scale=parameter_sqa.log_scale or False, digits=parameter_sqa.digits, is_fidelity=parameter_sqa.is_fidelity or False, target_value=parameter_sqa.target_value, ) elif parameter_sqa.domain_type == DomainType.CHOICE: if parameter_sqa.choice_values is None: raise SQADecodeError( # pragma: no cover "`values` must be set for ChoiceParameter.") return ChoiceParameter( name=parameter_sqa.name, parameter_type=parameter_sqa.parameter_type, # pyre-fixme[6]: Expected `List[Optional[Union[bool, float, int, # str]]]` for 3rd param but got `Optional[List[Optional[Union[bool, # float, int, str]]]]`. values=parameter_sqa.choice_values, is_fidelity=parameter_sqa.is_fidelity or False, target_value=parameter_sqa.target_value, ) elif parameter_sqa.domain_type == DomainType.FIXED: # Don't throw an error if parameter_sqa.fixed_value is None; # that might be the actual value! return FixedParameter( name=parameter_sqa.name, parameter_type=parameter_sqa.parameter_type, value=parameter_sqa.fixed_value, is_fidelity=parameter_sqa.is_fidelity or False, target_value=parameter_sqa.target_value, ) else: raise SQADecodeError( f"Cannot decode SQAParameter because {parameter_sqa.domain_type} " "is an invalid domain type.")
def _make_range_param(name: str, representation: TParameterRepresentation, parameter_type: Optional[str]) -> RangeParameter: assert "bounds" in representation, "Bounds are required for range parameters." bounds = representation["bounds"] assert isinstance(bounds, list) and len(bounds) == 2, ( f"Cannot parse parameter {name}: for range parameters, json representation " "should include a list of two values, lower and upper bounds of the bounds." ) return RangeParameter( name=name, parameter_type=_to_parameter_type(bounds, parameter_type, name, "bounds"), lower=checked_cast_to_tuple((float, int), bounds[0]), upper=checked_cast_to_tuple((float, int), bounds[1]), log_scale=checked_cast(bool, representation.get("log_scale", False)), is_fidelity=checked_cast(bool, representation.get("is_fidelity", False)), )
def _make_range_param(name: str, representation: TParameterRepresentation, parameter_type: Optional[str]) -> RangeParameter: assert "bounds" in representation, "Bounds are required for range parameters." bounds = representation["bounds"] assert isinstance(bounds, list) and len(bounds) == 2, ( f"Cannot parse parameter {name}: for range parameters, json representation " "should include a list of two values, lower and upper bounds of the bounds." ) return RangeParameter( name=name, parameter_type=_to_parameter_type(bounds, parameter_type, name, "bounds"), # pyre-fixme[6]: Expected `float` for 3rd param but got # `Optional[Union[bool, float, int, str]]`. lower=bounds[0], upper=bounds[1], log_scale=representation.get("log_scale", False), )
def testSetter(self): new_c = SumConstraint( parameters=[self.a, self.b], is_upper_bound=True, bound=10 ) self.ss2.add_parameter_constraints([new_c]) self.assertEqual(len(self.ss2.parameter_constraints), 2) self.ss2.set_parameter_constraints([]) self.assertEqual(len(self.ss2.parameter_constraints), 0) update_p = RangeParameter( name="b", parameter_type=ParameterType.INT, lower=10, upper=20 ) self.ss2.add_parameter(self.g) self.assertEqual(len(self.ss2.parameters), TOTAL_PARAMS + 1) self.ss2.update_parameter(update_p) self.assertEqual(self.ss2.parameters["b"].lower, 10)
def setUp(self): self.search_space = SearchSpace( parameters=[ RangeParameter("a", lower=1, upper=5, parameter_type=ParameterType.INT), ChoiceParameter("b", parameter_type=ParameterType.STRING, values=["a", "b", "c"]), ], parameter_constraints=[], ) self.t = IntRangeToChoice( search_space=self.search_space, observation_features=None, observation_data=None, )
def transform_search_space(self, search_space: SearchSpace) -> SearchSpace: transformed_parameters: Dict[str, Parameter] = {} for p in search_space.parameters.values(): if p.name in self.encoded_parameters: for new_p_name in self.encoded_parameters[p.name]: transformed_parameters[new_p_name] = RangeParameter( name=new_p_name, parameter_type=ParameterType.FLOAT, lower=0, upper=1, ) else: transformed_parameters[p.name] = p return SearchSpace( parameters=list(transformed_parameters.values()), parameter_constraints=[ pc.clone() for pc in search_space.parameter_constraints ], )
def testErrors(self): t = Derelativize(search_space=None, observation_features=None, observation_data=None) oc = OptimizationConfig( objective=Objective(Metric("c")), outcome_constraints=[ OutcomeConstraint(Metric("a"), ComparisonOp.LEQ, bound=2, relative=True) ], ) search_space = SearchSpace( parameters=[RangeParameter("x", ParameterType.FLOAT, 0, 20)]) g = ModelBridge(search_space, None, []) with self.assertRaises(ValueError): t.transform_optimization_config(oc, None, None) with self.assertRaises(ValueError): t.transform_optimization_config(oc, g, None)
def setUp(self): self.search_space = SearchSpace(parameters=[ RangeParameter( "x", lower=1, upper=4, parameter_type=ParameterType.FLOAT) ]) self.training_feats = [ ObservationFeatures({"x": 1}, trial_index=0), ObservationFeatures({"x": 2}, trial_index=0), ObservationFeatures({"x": 3}, trial_index=1), ObservationFeatures({"x": 4}, trial_index=2), ] self.t = TrialAsTask( search_space=self.search_space, observation_features=self.training_feats, observation_data=None, ) self.bm = { "bp1": { 0: "v1", 1: "v2", 2: "v3" }, "bp2": { 0: "u1", 1: "u1", 2: "u2" }, } self.t2 = TrialAsTask( search_space=self.search_space, observation_features=self.training_feats, observation_data=None, config={"trial_level_map": self.bm}, ) self.t3 = TrialAsTask( search_space=self.search_space, observation_features=self.training_feats, observation_data=None, config={"trial_level_map": {}}, )
def setUp(self): self.search_space = SearchSpace( parameters=[ RangeParameter( "a", lower=1, upper=3, parameter_type=ParameterType.FLOAT ), ChoiceParameter( "b", parameter_type=ParameterType.STRING, values=["a", "b", "c"] ), ] ) self.observation_features = [ ObservationFeatures(parameters={"a": 2, "b": "a"}), ObservationFeatures(parameters={"a": 3, "b": "b"}), ObservationFeatures(parameters={"a": 3, "b": "c"}), ] self.signature_to_parameterization = { Arm(parameters=obsf.parameters).signature: obsf.parameters for obsf in self.observation_features } self.transformed_features = [ ObservationFeatures( parameters={"arms": Arm(parameters={"a": 2, "b": "a"}).signature} ), ObservationFeatures( parameters={"arms": Arm(parameters={"a": 3, "b": "b"}).signature} ), ObservationFeatures( parameters={"arms": Arm(parameters={"a": 3, "b": "c"}).signature} ), ] self.t = SearchSpaceToChoice( search_space=self.search_space, observation_features=self.observation_features, observation_data=None, ) self.t2 = SearchSpaceToChoice( search_space=self.search_space, observation_features=[self.observation_features[0]], observation_data=None, )
def testTransformSearchSpace(self): ss2 = self.search_space.clone() ss2 = self.t.transform_search_space(ss2) self.assertEqual(len(ss2.parameters), 1) expected_parameter = ChoiceParameter( name="arms", parameter_type=ParameterType.STRING, values=list(self.t.signature_to_parameterization.keys()), ) self.assertEqual(ss2.parameters.get("arms"), expected_parameter) # With use_ordered ss2 = self.search_space.clone() ss2 = self.t3.transform_search_space(ss2) self.assertEqual(len(ss2.parameters), 1) expected_parameter = ChoiceParameter( name="arms", parameter_type=ParameterType.STRING, values=list(self.t.signature_to_parameterization.keys()), is_ordered=True, ) self.assertEqual(ss2.parameters.get("arms"), expected_parameter) # Test error if there are fidelities ss3 = SearchSpace(parameters=[ RangeParameter( "a", lower=1, upper=3, parameter_type=ParameterType.FLOAT, is_fidelity=True, target_value=3, ) ]) with self.assertRaises(ValueError): SearchSpaceToChoice( search_space=ss3, observation_features=self.observation_features, observation_data=None, )
def _make_range_param( name: str, representation: TParameterRepresentation, parameter_type: Optional[str] ) -> RangeParameter: assert "bounds" in representation, "Bounds are required for range parameters." bounds = representation["bounds"] assert isinstance(bounds, list) and len(bounds) == 2, ( f"Cannot parse parameter {name}: for range parameters, json representation " "should include a list of two values, lower and upper bounds of the range." ) return RangeParameter( name=name, parameter_type=_to_parameter_type(bounds, parameter_type, name, "bounds"), lower=checked_cast_to_tuple((float, int), bounds[0]), upper=checked_cast_to_tuple((float, int), bounds[1]), log_scale=checked_cast(bool, representation.get("log_scale", False)), digits=representation.get("digits", None), # pyre-ignore[6] is_fidelity=checked_cast(bool, representation.get("is_fidelity", False)), # pyre-fixme[6]: Expected `Union[None, bool, float, int, str]` for 8th param # but got `Union[None, List[Union[None, bool, float, int, str]], bool, float, # int, str]`. target_value=representation.get("target_value", None), )
def test_GPMES(self): """Tests GPMES instantiation.""" exp = get_branin_experiment(with_batch=True) with self.assertRaises(ValueError): get_GPMES(experiment=exp, data=exp.fetch_data()) exp.trials[0].run() gpmes = get_GPMES(experiment=exp, data=exp.fetch_data()) self.assertIsInstance(gpmes, TorchModelBridge) # Check that .gen returns without failure gr = gpmes.gen(n=1) self.assertEqual(len(gr.arms), 1) # test transform_configs with winsorization configs = { "Winsorize": { "winsorization_config": WinsorizationConfig( lower_quantile_margin=0.1, upper_quantile_margin=0.1, ) } } gpmes_win = get_GPMES( experiment=exp, data=exp.fetch_data(), transform_configs=configs ) self.assertIsInstance(gpmes_win, TorchModelBridge) self.assertEqual(gpmes_win._transform_configs, configs) # test multi-fidelity optimization exp.parameters["x2"] = RangeParameter( name="x2", parameter_type=exp.parameters["x2"].parameter_type, lower=-5.0, upper=10.0, is_fidelity=True, target_value=10.0, ) gpmes_mf = get_GPMES(experiment=exp, data=exp.fetch_data()) self.assertIsInstance(gpmes_mf, TorchModelBridge)
def setUp(self): self.search_space = SearchSpace(parameters=[ RangeParameter( "x", lower=1, upper=3, parameter_type=ParameterType.FLOAT), ChoiceParameter( "b", parameter_type=ParameterType.FLOAT, values=[1.0, 10.0, 100.0], is_ordered=True, ), ChoiceParameter( "c", parameter_type=ParameterType.STRING, values=["online", "offline"], is_task=True, ), ]) self.t = TaskEncode( search_space=self.search_space, observation_features=None, observation_data=None, )
def parameter_from_sqa(self, parameter_sqa: SQAParameter) -> Parameter: """Convert SQLAlchemy Parameter to Ax Parameter.""" if parameter_sqa.domain_type == DomainType.RANGE: if parameter_sqa.lower is None or parameter_sqa.upper is None: raise SQADecodeError( # pragma: no cover "`lower` and `upper` must be set for RangeParameter." ) return RangeParameter( name=parameter_sqa.name, parameter_type=parameter_sqa.parameter_type, lower=parameter_sqa.lower, upper=parameter_sqa.upper, log_scale=parameter_sqa.log_scale or False, digits=parameter_sqa.digits, ) elif parameter_sqa.domain_type == DomainType.CHOICE: if parameter_sqa.choice_values is None: raise SQADecodeError( # pragma: no cover "`values` must be set for ChoiceParameter." ) return ChoiceParameter( name=parameter_sqa.name, parameter_type=parameter_sqa.parameter_type, values=parameter_sqa.choice_values, ) elif parameter_sqa.domain_type == DomainType.FIXED: # Don't throw an error if parameter_sqa.fixed_value is None; # that might be the actual value! return FixedParameter( name=parameter_sqa.name, parameter_type=parameter_sqa.parameter_type, value=parameter_sqa.fixed_value, ) else: raise SQADecodeError( f"Cannot decode SQAParameter because {parameter_sqa.domain_type} " "is an invalid domain type." )
def test_ood_gen(self, _): # Test fit_out_of_design by returning OOD candidats exp = get_experiment_for_value() ss = SearchSpace([RangeParameter("x", ParameterType.FLOAT, 0.0, 1.0)]) modelbridge = ModelBridge( search_space=ss, model=Model(), transforms=[], experiment=exp, data=0, fit_out_of_design=True, ) obs = ObservationFeatures(parameters={"x": 3.0}) modelbridge._gen = mock.MagicMock( "ax.modelbridge.base.ModelBridge._gen", autospec=True, return_value=([obs], [2], None, {}), ) gr = modelbridge.gen(n=1) self.assertEqual(gr.arms[0].parameters, obs.parameters) # Test clamping arms by setting fit_out_of_design=False modelbridge = ModelBridge( search_space=ss, model=Model(), transforms=[], experiment=exp, data=0, fit_out_of_design=False, ) obs = ObservationFeatures(parameters={"x": 3.0}) modelbridge._gen = mock.MagicMock( "ax.modelbridge.base.ModelBridge._gen", autospec=True, return_value=([obs], [2], None, {}), ) gr = modelbridge.gen(n=1) self.assertEqual(gr.arms[0].parameters, {"x": 1.0})
def transform_search_space(self, search_space: SearchSpace) -> SearchSpace: transformed_parameters: Dict[str, Parameter] = {} for p in search_space.parameters.values(): if p.name in self.encoded_parameters: # TypeAssert. Only ChoiceParameters present here. # pyre: p_ is declared to have type `ChoiceParameter` but is # pyre-fixme[9]: used as type `Parameter`. p_: ChoiceParameter = p # Choice(|K|) => Range(0, K-1) transformed_parameters[p.name] = RangeParameter( name=p_.name, parameter_type=ParameterType.INT, lower=0, upper=len(p_.values) - 1, ) else: transformed_parameters[p.name] = p return SearchSpace( parameters=list(transformed_parameters.values()), parameter_constraints=[ pc.clone() for pc in search_space.parameter_constraints ], )
def setUp(self): self.obsd1 = ObservationData( metric_names=["m1", "m2", "m2"], means=np.array([1.0, 2.0, 8.0]), covariance=np.array([[1.0, 0.2, 0.4], [0.2, 2.0, 0.8], [0.4, 0.8, 3.0]]), ) self.obsd2 = ObservationData( metric_names=["m1", "m1", "m2", "m2"], means=np.array([1.0, 5.0, 2.0, 1.0]), covariance=np.array( [ [1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.2, 0.4], [0.0, 0.2, 2.0, 0.8], [0.0, 0.4, 0.8, 3.0], ] ), ) self.search_space = SearchSpace( parameters=[ RangeParameter( name="x", parameter_type=ParameterType.FLOAT, lower=0, upper=10 ), ChoiceParameter( name="z", parameter_type=ParameterType.STRING, values=["a", "b"] ), ] ) self.obsf1 = ObservationFeatures({"x": 2, "z": "a"}) self.obsf2 = ObservationFeatures({"x": 5, "z": "b"}) self.t = StratifiedStandardizeY( search_space=self.search_space, observation_features=[self.obsf1, self.obsf2], observation_data=[self.obsd1, self.obsd2], config={"parameter_name": "z"}, )
def transform_search_space(self, search_space: SearchSpace) -> SearchSpace: transformed_parameters: Dict[str, Parameter] = {} for p_name, p in search_space.parameters.items(): if p_name in self.encoded_parameters: if p.is_fidelity: raise ValueError( f"Cannot one-hot-encode fidelity parameter {p_name}") for new_p_name in self.encoded_parameters[p_name]: transformed_parameters[new_p_name] = RangeParameter( name=new_p_name, parameter_type=ParameterType.FLOAT, lower=0, upper=1, ) else: transformed_parameters[p_name] = p return SearchSpace( parameters=list(transformed_parameters.values()), parameter_constraints=[ pc.clone_with_transformed_parameters( transformed_parameters=transformed_parameters) for pc in search_space.parameter_constraints ], )
def test_create_experiment(self) -> None: """Test basic experiment creation.""" ax_client = AxClient( GenerationStrategy( steps=[GenerationStep(model=Models.SOBOL, num_trials=30)])) with self.assertRaisesRegex(ValueError, "Experiment not set on Ax client"): ax_client.experiment ax_client.create_experiment( name="test_experiment", parameters=[ { "name": "x", "type": "range", "bounds": [0.001, 0.1], "value_type": "float", "log_scale": True, }, { "name": "y", "type": "choice", "values": [1, 2, 3], "value_type": "int", "is_ordered": True, }, { "name": "x3", "type": "fixed", "value": 2, "value_type": "int" }, { "name": "x4", "type": "range", "bounds": [1.0, 3.0], "value_type": "int", }, { "name": "x5", "type": "choice", "values": ["one", "two", "three"], "value_type": "str", }, { "name": "x6", "type": "range", "bounds": [1.0, 3.0], "value_type": "int", }, ], objective_name="test_objective", minimize=True, outcome_constraints=["some_metric >= 3", "some_metric <= 4.0"], parameter_constraints=["x4 <= x6"], ) assert ax_client._experiment is not None self.assertEqual(ax_client._experiment, ax_client.experiment) self.assertEqual( ax_client._experiment.search_space.parameters["x"], RangeParameter( name="x", parameter_type=ParameterType.FLOAT, lower=0.001, upper=0.1, log_scale=True, ), ) self.assertEqual( ax_client._experiment.search_space.parameters["y"], ChoiceParameter( name="y", parameter_type=ParameterType.INT, values=[1, 2, 3], is_ordered=True, ), ) self.assertEqual( ax_client._experiment.search_space.parameters["x3"], FixedParameter(name="x3", parameter_type=ParameterType.INT, value=2), ) self.assertEqual( ax_client._experiment.search_space.parameters["x4"], RangeParameter(name="x4", parameter_type=ParameterType.INT, lower=1.0, upper=3.0), ) self.assertEqual( ax_client._experiment.search_space.parameters["x5"], ChoiceParameter( name="x5", parameter_type=ParameterType.STRING, values=["one", "two", "three"], ), ) self.assertEqual( ax_client._experiment.optimization_config.outcome_constraints[0], OutcomeConstraint( metric=Metric(name="some_metric"), op=ComparisonOp.GEQ, bound=3.0, relative=False, ), ) self.assertEqual( ax_client._experiment.optimization_config.outcome_constraints[1], OutcomeConstraint( metric=Metric(name="some_metric"), op=ComparisonOp.LEQ, bound=4.0, relative=False, ), ) self.assertTrue( ax_client._experiment.optimization_config.objective.minimize)
def get_range_parameter2() -> RangeParameter: return RangeParameter(name="x", parameter_type=ParameterType.INT, lower=1, upper=10)
def get_search_space_for_range_value(min: float = 3.0, max: float = 6.0) -> SearchSpace: return SearchSpace([RangeParameter("x", ParameterType.FLOAT, min, max)])
def get_small_discrete_search_space() -> SearchSpace: return SearchSpace([ RangeParameter("x", ParameterType.INT, 0, 1), ChoiceParameter("y", ParameterType.STRING, ["red", "panda"]), ])
def test_constraint_from_str(self): with self.assertRaisesRegex(ValueError, "Bound for the constraint"): constraint_from_str("x1 + x2 <= not_numerical_bound", { "x1": None, "x2": None }) with self.assertRaisesRegex(ValueError, "Outcome constraint bound"): outcome_constraint_from_str("m1 <= not_numerical_bound") three_val_constaint = constraint_from_str( "x1 + x2 + x3 <= 3", { "x1": RangeParameter(name="x1", parameter_type=ParameterType.FLOAT, lower=0.1, upper=2.0), "x2": RangeParameter(name="x2", parameter_type=ParameterType.FLOAT, lower=0.1, upper=2.0), "x3": RangeParameter(name="x3", parameter_type=ParameterType.FLOAT, lower=0.1, upper=2.0), }, ) self.assertEqual(three_val_constaint.bound, 3.0) with self.assertRaisesRegex(ValueError, "Parameter constraint should"): constraint_from_str("x1 + x2 + <= 3", { "x1": None, "x2": None, "x3": None }) with self.assertRaisesRegex(ValueError, "Parameter constraint should"): constraint_from_str("x1 + x2 + x3 = 3", { "x1": None, "x2": None, "x3": None }) three_val_constaint2 = constraint_from_str( "-x1 + 2.1*x2 - 4*x3 <= 3", { "x1": RangeParameter(name="x1", parameter_type=ParameterType.FLOAT, lower=0.1, upper=4.0), "x2": RangeParameter(name="x2", parameter_type=ParameterType.FLOAT, lower=0.1, upper=4.0), "x3": RangeParameter(name="x3", parameter_type=ParameterType.FLOAT, lower=0.1, upper=4.0), }, ) self.assertEqual(three_val_constaint2.bound, 3.0) self.assertEqual(three_val_constaint2.constraint_dict, { "x1": -1.0, "x2": 2.1, "x3": -4.0 }) with self.assertRaisesRegex(ValueError, "Multiplier should be float"): constraint_from_str("x1 - e*x2 + x3 <= 3", { "x1": None, "x2": None, "x3": None }) with self.assertRaisesRegex(ValueError, "A linear constraint should be"): constraint_from_str("x1 - 2 *x2 + 3 *x3 <= 3", { "x1": None, "x2": None, "x3": None }) with self.assertRaisesRegex(ValueError, "A linear constraint should be"): constraint_from_str("x1 - 2* x2 + 3* x3 <= 3", { "x1": None, "x2": None, "x3": None }) with self.assertRaisesRegex(ValueError, "A linear constraint should be"): constraint_from_str("x1 - 2 * x2 + 3*x3 <= 3", { "x1": None, "x2": None, "x3": None })
def setUp(self): self.a = RangeParameter(name="a", parameter_type=ParameterType.FLOAT, lower=0.5, upper=5.5) self.b = RangeParameter(name="b", parameter_type=ParameterType.INT, lower=2, upper=10) self.c = ChoiceParameter(name="c", parameter_type=ParameterType.STRING, values=["foo", "bar", "baz"]) self.d = FixedParameter(name="d", parameter_type=ParameterType.BOOL, value=True) self.e = ChoiceParameter(name="e", parameter_type=ParameterType.FLOAT, values=[0.0, 0.1, 0.2, 0.5]) self.f = RangeParameter( name="f", parameter_type=ParameterType.INT, lower=2, upper=10, log_scale=True, ) self.g = RangeParameter(name="g", parameter_type=ParameterType.FLOAT, lower=0.0, upper=1.0) self.parameters = [self.a, self.b, self.c, self.d, self.e, self.f] self.ss1 = SearchSpace(parameters=self.parameters) self.ss2 = SearchSpace( parameters=self.parameters, parameter_constraints=[ OrderConstraint(lower_parameter=self.a, upper_parameter=self.b) ], ) self.ss1_repr = ( "SearchSpace(" "parameters=[" "RangeParameter(name='a', parameter_type=FLOAT, range=[0.5, 5.5]), " "RangeParameter(name='b', parameter_type=INT, range=[2, 10]), " "ChoiceParameter(name='c', parameter_type=STRING, " "values=['foo', 'bar', 'baz']), " "FixedParameter(name='d', parameter_type=BOOL, value=True), " "ChoiceParameter(name='e', parameter_type=FLOAT, " "values=[0.0, 0.1, 0.2, 0.5]), " "RangeParameter(name='f', parameter_type=INT, range=[2, 10], " "log_scale=True)], " "parameter_constraints=[])") self.ss2_repr = ( "SearchSpace(" "parameters=[" "RangeParameter(name='a', parameter_type=FLOAT, range=[0.5, 5.5]), " "RangeParameter(name='b', parameter_type=INT, range=[2, 10]), " "ChoiceParameter(name='c', parameter_type=STRING, " "values=['foo', 'bar', 'baz']), " "FixedParameter(name='d', parameter_type=BOOL, value=True), " "ChoiceParameter(name='e', parameter_type=FLOAT, " "values=[0.0, 0.1, 0.2, 0.5]), " "RangeParameter(name='f', parameter_type=INT, range=[2, 10], " "log_scale=True)], " "parameter_constraints=[OrderConstraint(a <= b)])")
class SearchSpaceTest(TestCase): def setUp(self): self.a = RangeParameter(name="a", parameter_type=ParameterType.FLOAT, lower=0.5, upper=5.5) self.b = RangeParameter(name="b", parameter_type=ParameterType.INT, lower=2, upper=10) self.c = ChoiceParameter(name="c", parameter_type=ParameterType.STRING, values=["foo", "bar", "baz"]) self.d = FixedParameter(name="d", parameter_type=ParameterType.BOOL, value=True) self.e = ChoiceParameter(name="e", parameter_type=ParameterType.FLOAT, values=[0.0, 0.1, 0.2, 0.5]) self.f = RangeParameter( name="f", parameter_type=ParameterType.INT, lower=2, upper=10, log_scale=True, ) self.g = RangeParameter(name="g", parameter_type=ParameterType.FLOAT, lower=0.0, upper=1.0) self.parameters = [self.a, self.b, self.c, self.d, self.e, self.f] self.ss1 = SearchSpace(parameters=self.parameters) self.ss2 = SearchSpace( parameters=self.parameters, parameter_constraints=[ OrderConstraint(lower_parameter=self.a, upper_parameter=self.b) ], ) self.ss1_repr = ( "SearchSpace(" "parameters=[" "RangeParameter(name='a', parameter_type=FLOAT, range=[0.5, 5.5]), " "RangeParameter(name='b', parameter_type=INT, range=[2, 10]), " "ChoiceParameter(name='c', parameter_type=STRING, " "values=['foo', 'bar', 'baz']), " "FixedParameter(name='d', parameter_type=BOOL, value=True), " "ChoiceParameter(name='e', parameter_type=FLOAT, " "values=[0.0, 0.1, 0.2, 0.5]), " "RangeParameter(name='f', parameter_type=INT, range=[2, 10], " "log_scale=True)], " "parameter_constraints=[])") self.ss2_repr = ( "SearchSpace(" "parameters=[" "RangeParameter(name='a', parameter_type=FLOAT, range=[0.5, 5.5]), " "RangeParameter(name='b', parameter_type=INT, range=[2, 10]), " "ChoiceParameter(name='c', parameter_type=STRING, " "values=['foo', 'bar', 'baz']), " "FixedParameter(name='d', parameter_type=BOOL, value=True), " "ChoiceParameter(name='e', parameter_type=FLOAT, " "values=[0.0, 0.1, 0.2, 0.5]), " "RangeParameter(name='f', parameter_type=INT, range=[2, 10], " "log_scale=True)], " "parameter_constraints=[OrderConstraint(a <= b)])") def testEq(self): ss2 = SearchSpace( parameters=self.parameters, parameter_constraints=[ OrderConstraint(lower_parameter=self.a, upper_parameter=self.b) ], ) self.assertEqual(self.ss2, ss2) self.assertNotEqual(self.ss1, self.ss2) def testProperties(self): self.assertEqual(len(self.ss1.parameters), TOTAL_PARAMS) self.assertTrue("a" in self.ss1.parameters) self.assertTrue(len(self.ss1.tunable_parameters), TUNABLE_PARAMS) self.assertFalse("d" in self.ss1.tunable_parameters) self.assertTrue(len(self.ss1.range_parameters), RANGE_PARAMS) self.assertFalse("c" in self.ss1.range_parameters) self.assertTrue(len(self.ss1.parameter_constraints) == 0) self.assertTrue(len(self.ss2.parameter_constraints) == 1) def testRepr(self): self.assertEqual(str(self.ss2), self.ss2_repr) self.assertEqual(str(self.ss1), self.ss1_repr) def testSetter(self): new_c = SumConstraint(parameters=[self.a, self.b], is_upper_bound=True, bound=10) self.ss2.add_parameter_constraints([new_c]) self.assertEqual(len(self.ss2.parameter_constraints), 2) self.ss2.set_parameter_constraints([]) self.assertEqual(len(self.ss2.parameter_constraints), 0) update_p = RangeParameter(name="b", parameter_type=ParameterType.INT, lower=10, upper=20) self.ss2.add_parameter(self.g) self.assertEqual(len(self.ss2.parameters), TOTAL_PARAMS + 1) self.ss2.update_parameter(update_p) self.assertEqual(self.ss2.parameters["b"].lower, 10) def testBadConstruction(self): # Duplicate parameter with self.assertRaises(ValueError): p1 = self.parameters + [self.parameters[0]] SearchSpace(parameters=p1, parameter_constraints=[]) # Constraint on non-existent parameter with self.assertRaises(ValueError): SearchSpace( parameters=self.parameters, parameter_constraints=[ OrderConstraint(lower_parameter=self.a, upper_parameter=self.g) ], ) # Vanilla Constraint on non-existent parameter with self.assertRaises(ValueError): SearchSpace( parameters=self.parameters, parameter_constraints=[ ParameterConstraint(constraint_dict={"g": 1}, bound=0) ], ) # Constraint on non-numeric parameter with self.assertRaises(ValueError): SearchSpace( parameters=self.parameters, parameter_constraints=[ OrderConstraint(lower_parameter=self.a, upper_parameter=self.d) ], ) # Constraint on choice parameter with self.assertRaises(ValueError): SearchSpace( parameters=self.parameters, parameter_constraints=[ OrderConstraint(lower_parameter=self.a, upper_parameter=self.e) ], ) # Constraint on logscale parameter with self.assertRaises(ValueError): SearchSpace( parameters=self.parameters, parameter_constraints=[ OrderConstraint(lower_parameter=self.a, upper_parameter=self.f) ], ) # Constraint on mismatched parameter with self.assertRaises(ValueError): wrong_a = self.a.clone() wrong_a.update_range(upper=10) SearchSpace( parameters=self.parameters, parameter_constraints=[ OrderConstraint(lower_parameter=wrong_a, upper_parameter=self.b) ], ) def testBadSetter(self): new_p = RangeParameter(name="b", parameter_type=ParameterType.FLOAT, lower=0.0, upper=1.0) # Add duplicate parameter with self.assertRaises(ValueError): self.ss1.add_parameter(new_p) # Update parameter to different type with self.assertRaises(ValueError): self.ss1.update_parameter(new_p) # Update non-existent parameter new_p = RangeParameter(name="g", parameter_type=ParameterType.FLOAT, lower=0.0, upper=1.0) with self.assertRaises(ValueError): self.ss1.update_parameter(new_p) def testCheckMembership(self): p_dict = {"a": 1.0, "b": 5, "c": "foo", "d": True, "e": 0.2, "f": 5} # Valid self.assertTrue(self.ss2.check_membership(p_dict)) # Value out of range p_dict["a"] = 20.0 self.assertFalse(self.ss2.check_membership(p_dict)) with self.assertRaises(ValueError): self.ss2.check_membership(p_dict, raise_error=True) # Violate constraints p_dict["a"] = 5.3 self.assertFalse(self.ss2.check_membership(p_dict)) with self.assertRaises(ValueError): self.ss2.check_membership(p_dict, raise_error=True) # Incomplete param dict p_dict.pop("a") self.assertFalse(self.ss2.check_membership(p_dict)) with self.assertRaises(ValueError): self.ss2.check_membership(p_dict, raise_error=True) # Unknown parameter p_dict["q"] = 40 self.assertFalse(self.ss2.check_membership(p_dict)) with self.assertRaises(ValueError): self.ss2.check_membership(p_dict, raise_error=True) def testCheckTypes(self): p_dict = {"a": 1.0, "b": 5, "c": "foo", "d": True, "e": 0.2, "f": 5} # Valid self.assertTrue(self.ss2.check_membership(p_dict)) # Invalid type p_dict["b"] = 5.2 self.assertFalse(self.ss2.check_types(p_dict)) with self.assertRaises(ValueError): self.ss2.check_types(p_dict, raise_error=True) p_dict["b"] = 5 # Incomplete param dict p_dict.pop("a") self.assertFalse(self.ss2.check_types(p_dict)) with self.assertRaises(ValueError): self.ss2.check_types(p_dict, raise_error=True) # Unknown parameter p_dict["q"] = 40 self.assertFalse(self.ss2.check_types(p_dict)) with self.assertRaises(ValueError): self.ss2.check_types(p_dict, raise_error=True) def testCastArm(self): p_dict = {"a": 1.0, "b": 5.0, "c": "foo", "d": True, "e": 0.2, "f": 5} # Check "b" parameter goes from float to int self.assertTrue(isinstance(p_dict["b"], float)) new_arm = self.ss2.cast_arm(Arm(p_dict)) self.assertTrue(isinstance(new_arm.parameters["b"], int)) # Unknown parameter should be unchanged p_dict["q"] = 40 new_arm = self.ss2.cast_arm(Arm(p_dict)) self.assertTrue(isinstance(new_arm.parameters["q"], int)) def testCopy(self): a = RangeParameter("a", ParameterType.FLOAT, 1.0, 5.5) b = RangeParameter("b", ParameterType.FLOAT, 2.0, 5.5) c = ChoiceParameter("c", ParameterType.INT, [2, 3]) ss = SearchSpace( parameters=[a, b, c], parameter_constraints=[ OrderConstraint(lower_parameter=a, upper_parameter=b) ], ) ss_copy = ss.clone() self.assertEqual(len(ss_copy.parameters), len(ss_copy.parameters)) self.assertEqual(len(ss_copy.parameter_constraints), len(ss_copy.parameter_constraints)) ss_copy.add_parameter(FixedParameter("d", ParameterType.STRING, "h")) self.assertNotEqual(len(ss_copy.parameters), len(ss.parameters)) def testOutOfDesignArm(self): arm1 = self.ss1.out_of_design_arm() arm2 = self.ss2.out_of_design_arm() arm1_nones = [p is None for p in arm1.parameters.values()] self.assertTrue(all(arm1_nones)) self.assertTrue(arm1 == arm2) def testConstructArm(self): # Test constructing an arm of default values arm = self.ss1.construct_arm(name="test") self.assertEqual(arm.name, "test") for p_name in self.ss1.parameters.keys(): self.assertTrue(p_name in arm.parameters) self.assertEqual(arm.parameters[p_name], None) # Test constructing an arm with a custom value arm = self.ss1.construct_arm({"a": 1.0}) for p_name in self.ss1.parameters.keys(): self.assertTrue(p_name in arm.parameters) if p_name == "a": self.assertEqual(arm.parameters[p_name], 1.0) else: self.assertEqual(arm.parameters[p_name], None) # Test constructing an arm with a bad param name with self.assertRaises(ValueError): self.ss1.construct_arm({"IDONTEXIST_a": 1.0}) # Test constructing an arm with a bad param name with self.assertRaises(ValueError): self.ss1.construct_arm({"a": "notafloat"})
def setUp(self): x = RangeParameter("x", ParameterType.FLOAT, lower=0, upper=1) y = RangeParameter("y", ParameterType.FLOAT, lower=1, upper=2, is_fidelity=True, target_value=2) z = RangeParameter("z", ParameterType.FLOAT, lower=0, upper=5) self.parameters = [x, y, z] parameter_constraints = [ OrderConstraint(x, y), SumConstraint([x, z], False, 3.5), ] self.search_space = SearchSpace(self.parameters, parameter_constraints) self.observation_features = [ ObservationFeatures(parameters={ "x": 0.2, "y": 1.2, "z": 3 }), ObservationFeatures(parameters={ "x": 0.4, "y": 1.4, "z": 3 }), ObservationFeatures(parameters={ "x": 0.6, "y": 1.6, "z": 3 }), ] self.observation_data = [ ObservationData( metric_names=["a", "b"], means=np.array([1.0, -1.0]), covariance=np.array([[1.0, 4.0], [4.0, 6.0]]), ), ObservationData( metric_names=["a", "b"], means=np.array([2.0, -2.0]), covariance=np.array([[2.0, 5.0], [5.0, 7.0]]), ), ObservationData(metric_names=["a"], means=np.array([3.0]), covariance=np.array([[3.0]])), ] self.observations = [ Observation( features=self.observation_features[i], data=self.observation_data[i], arm_name=str(i), ) for i in range(3) ] self.pending_observations = { "b": [ObservationFeatures(parameters={ "x": 0.6, "y": 1.6, "z": 3 })] } self.model_gen_options = {"option": "yes"}
class RangeParameterTest(TestCase): def setUp(self): self.param1 = RangeParameter( name="x", parameter_type=ParameterType.FLOAT, lower=1, upper=3, log_scale=True, digits=5, is_fidelity=True, target_value=2, ) self.param1_repr = ( "RangeParameter(name='x', parameter_type=FLOAT, " "range=[1.0, 3.0], log_scale=True, digits=5, fidelity=True, target_" "value=2.0)") self.param2 = RangeParameter(name="y", parameter_type=ParameterType.INT, lower=10, upper=15) self.param2_repr = ( "RangeParameter(name='y', parameter_type=INT, range=[10, 15])") def testEq(self): param2 = RangeParameter( name="x", parameter_type=ParameterType.FLOAT, lower=1, upper=3, log_scale=True, digits=5, is_fidelity=True, target_value=2, ) self.assertEqual(self.param1, param2) self.assertNotEqual(self.param1, self.param2) def testProperties(self): self.assertEqual(self.param1.name, "x") self.assertEqual(self.param1.parameter_type, ParameterType.FLOAT) self.assertEqual(self.param1.lower, 1) self.assertEqual(self.param1.upper, 3) self.assertEqual(self.param1.digits, 5) self.assertTrue(self.param1.log_scale) self.assertFalse(self.param2.log_scale) self.assertTrue(self.param1.is_numeric) self.assertTrue(self.param1.is_fidelity) self.assertIsNotNone(self.param1.target_value) self.assertFalse(self.param2.is_fidelity) self.assertIsNone(self.param2.target_value) def testValidate(self): self.assertFalse(self.param1.validate(None)) self.assertFalse(self.param1.validate("foo")) self.assertTrue(self.param1.validate(1)) self.assertTrue(self.param1.validate(1.3)) def testRepr(self): self.assertEqual(str(self.param1), self.param1_repr) self.assertEqual(str(self.param2), self.param2_repr) def testBadCreations(self): with self.assertRaises(UserInputError): RangeParameter("x", ParameterType.STRING, 1, 3) with self.assertRaises(UserInputError): RangeParameter("x", ParameterType.FLOAT, 3, 1) with self.assertRaises(UserInputError): RangeParameter("x", ParameterType.INT, 0, 1, log_scale=True) with self.assertRaises(UserInputError): RangeParameter("x", ParameterType.INT, 0.5, 1) with self.assertRaises(UserInputError): RangeParameter("x", ParameterType.INT, 0.5, 1, is_fidelity=True) def testBadSetter(self): with self.assertRaises(ValueError): self.param1.update_range(upper="foo") with self.assertRaises(ValueError): self.param1.update_range(lower="foo") with self.assertRaises(UserInputError): self.param1.update_range(lower=4) with self.assertRaises(UserInputError): self.param1.update_range(upper=0.5) with self.assertRaises(UserInputError): self.param1.update_range(lower=1.0, upper=0.9) def testGoodSetter(self): self.param1.update_range(lower=1.0) self.param1.update_range(upper=1.0011) self.param1.set_log_scale(False) self.param1.set_digits(3) self.assertEqual(self.param1.digits, 3) self.assertEqual(self.param1.upper, 1.001) # This would cast Upper = Lower = 1, which is not allowed with self.assertRaises(UserInputError): self.param1.set_digits(1) self.param1.update_range(lower=2.0, upper=3.0) self.assertEqual(self.param1.lower, 2.0) self.assertEqual(self.param1.upper, 3.0) def testCast(self): self.assertEqual(self.param2.cast(2.5), 2) self.assertEqual(self.param2.cast(3), 3) self.assertEqual(self.param2.cast(None), None) def testClone(self): param_clone = self.param1.clone() self.assertEqual(self.param1.lower, param_clone.lower) param_clone._lower = 2.0 self.assertNotEqual(self.param1.lower, param_clone.lower)