def testGetBoundsAndTask(self): bounds, task_features, target_fidelities = get_bounds_and_task( self.search_space, ["x", "y", "z"]) self.assertEqual(bounds, [(0.0, 1.0), (1.0, 2.0), (0.0, 5.0)]) self.assertEqual(task_features, []) self.assertEqual(target_fidelities, {1: 2.0}) bounds, task_features, target_fidelities = get_bounds_and_task( self.search_space, ["x", "z"]) self.assertEqual(target_fidelities, {}) # Test that Int param is treated as task feature search_space = SearchSpace(self.parameters) search_space._parameters["x"] = RangeParameter("x", ParameterType.INT, lower=1, upper=4) bounds, task_features, target_fidelities = get_bounds_and_task( search_space, ["x", "y", "z"]) self.assertEqual(task_features, [0]) # Test validation search_space._parameters["x"] = ChoiceParameter( "x", ParameterType.FLOAT, [0.1, 0.4]) with self.assertRaises(ValueError): get_bounds_and_task(search_space, ["x", "y", "z"]) search_space._parameters["x"] = RangeParameter("x", ParameterType.FLOAT, lower=1.0, upper=4.0, log_scale=True) with self.assertRaises(ValueError): get_bounds_and_task(search_space, ["x", "y", "z"])
def test_transform_callback_int_log(self, *_): exp = get_branin_experiment(with_batch=True) parameters = [ RangeParameter( name="x1", parameter_type=ParameterType.INT, lower=1, upper=100, log_scale=True, ), RangeParameter( name="x2", parameter_type=ParameterType.INT, lower=1, upper=100, log_scale=True, ), ] gpei = TorchModelBridge( experiment=exp, data=exp.fetch_data(), search_space=SearchSpace(parameters=parameters), model=BotorchModel(), transforms=[IntToFloat, Log], torch_dtype=torch.double, fit_out_of_design=True, ) transformed = gpei._transform_callback([0.5, 1.5]) self.assertTrue(np.allclose(transformed, [0.47712, 1.50515]))
def setUp(self): self.search_space = SearchSpace(parameters=[ RangeParameter( "x", lower=1, upper=3, parameter_type=ParameterType.FLOAT, log_scale=True, ), RangeParameter( "a", lower=1, upper=2, parameter_type=ParameterType.INT), ChoiceParameter("b", parameter_type=ParameterType.STRING, values=["a", "b", "c"]), ]) self.t = Log( search_space=self.search_space, observation_features=None, observation_data=None, ) self.search_space_with_target = SearchSpace(parameters=[ RangeParameter( "x", lower=1, upper=3, parameter_type=ParameterType.FLOAT, log_scale=True, is_fidelity=True, target_value=3, ) ])
def test_transform_callback_int(self, *_): exp = get_branin_experiment(with_batch=True) data = get_branin_data(trial_indices=exp.trials) parameters = [ RangeParameter(name="x1", parameter_type=ParameterType.INT, lower=1, upper=10), RangeParameter(name="x2", parameter_type=ParameterType.INT, lower=5, upper=15), ] gpei = TorchModelBridge( experiment=exp, data=data, search_space=SearchSpace(parameters=parameters), model=BotorchModel(), transforms=[IntToFloat], torch_dtype=torch.double, fit_out_of_design=True, ) transformed = gpei._transform_callback([5.4, 7.6]) self.assertTrue(np.allclose(transformed, [5, 8])) np_mb = ArrayModelBridge( experiment=exp, data=exp.fetch_data(), search_space=SearchSpace(parameters=parameters), model=NumpyModel(), transforms=[IntToFloat], ) transformed = np_mb._transform_callback(np.array([5.4, 7.6])) self.assertTrue(np.allclose(transformed, [5, 8]))
def setUp(self): self.search_space = SearchSpace( parameters=[ RangeParameter("x", lower=1, upper=3, parameter_type=ParameterType.FLOAT), RangeParameter("a", lower=1, upper=2, parameter_type=ParameterType.INT), ChoiceParameter( "b", parameter_type=ParameterType.FLOAT, values=[1.0, 10.0, 100.0], is_ordered=True, ), ChoiceParameter( "c", parameter_type=ParameterType.FLOAT, values=[10.0, 100.0, 1000.0], is_ordered=True, ), ChoiceParameter("d", parameter_type=ParameterType.STRING, values=["r", "q", "z"]), ], parameter_constraints=[ ParameterConstraint(constraint_dict={ "x": -0.5, "a": 1 }, bound=0.5) ], ) self.t = ChoiceEncode( search_space=self.search_space, observation_features=None, observation_data=None, ) self.observation_features = [ ObservationFeatures(parameters={ "x": 2.2, "a": 2, "b": 10.0, "c": 10.0, "d": "r" }) ] # expected parameters after transform self.expected_transformed_params = { "x": 2.2, "a": 2, # ordered float choice originally; transformed normalized value "b": normalize_values([1.0, 10.0, 100.0])[1], # ordered float choice originally; transformed normalized value "c": normalize_values([10.0, 100.0, 1000.0])[0], # string choice originally; transformed to int index. "d": 0, }
def setUp(self): self.search_space = SearchSpace( parameters=[ RangeParameter( "x", lower=1, upper=3, parameter_type=ParameterType.FLOAT ), RangeParameter( "y", lower=1, upper=2, parameter_type=ParameterType.FLOAT ), RangeParameter( "z", lower=1, upper=2, parameter_type=ParameterType.FLOAT, log_scale=True, ), RangeParameter("a", lower=1, upper=2, parameter_type=ParameterType.INT), ChoiceParameter( "b", parameter_type=ParameterType.STRING, values=["a", "b", "c"] ), ], parameter_constraints=[], ) self.t = CenteredUnitX( search_space=self.search_space, observation_features=None, observation_data=None, )
def test_transform_callback_log(self, *_): parameters = [ RangeParameter( name="x1", parameter_type=ParameterType.FLOAT, lower=1, upper=3, log_scale=True, ), RangeParameter( name="x2", parameter_type=ParameterType.FLOAT, lower=1, upper=3, log_scale=True, ), ] search_space = SearchSpace(parameters=parameters) exp = get_branin_experiment(with_batch=True, search_space=search_space) gpei = TorchModelBridge( experiment=exp, data=exp.fetch_data(), search_space=search_space, model=BotorchModel(), transforms=[Log], torch_dtype=torch.double, fit_out_of_design=True, ) transformed = gpei._transform_callback([1.2, 2.5]) self.assertTrue(np.allclose(transformed, [1.2, 2.5]))
def setUp(self): self.a = RangeParameter( name="a", parameter_type=ParameterType.FLOAT, lower=0.5, upper=5.5 ) self.b = RangeParameter( name="b", parameter_type=ParameterType.INT, lower=2, upper=10 ) self.c = ChoiceParameter( name="c", parameter_type=ParameterType.STRING, values=["foo", "bar", "baz"] ) self.d = FixedParameter(name="d", parameter_type=ParameterType.BOOL, value=True) self.e = ChoiceParameter( name="e", parameter_type=ParameterType.FLOAT, values=[0.0, 0.1, 0.2, 0.5] ) self.f = RangeParameter( name="f", parameter_type=ParameterType.INT, lower=2, upper=10, log_scale=True, ) self.g = RangeParameter( name="g", parameter_type=ParameterType.FLOAT, lower=0.0, upper=1.0 ) self.parameters = [self.a, self.b, self.c, self.d, self.e, self.f] self.ss1 = SearchSpace(parameters=self.parameters) self.ss2 = SearchSpace( parameters=self.parameters, parameter_constraints=[ OrderConstraint(lower_parameter=self.a, upper_parameter=self.b) ], ) self.ss1_repr = ( "SearchSpace(" "parameters=[" "RangeParameter(name='a', parameter_type=FLOAT, range=[0.5, 5.5]), " "RangeParameter(name='b', parameter_type=INT, range=[2, 10]), " "ChoiceParameter(name='c', parameter_type=STRING, " "values=['foo', 'bar', 'baz']), " "FixedParameter(name='d', parameter_type=BOOL, value=True), " "ChoiceParameter(name='e', parameter_type=FLOAT, " "values=[0.0, 0.1, 0.2, 0.5]), " "RangeParameter(name='f', parameter_type=INT, range=[2, 10], " "log_scale=True)], " "parameter_constraints=[])" ) self.ss2_repr = ( "SearchSpace(" "parameters=[" "RangeParameter(name='a', parameter_type=FLOAT, range=[0.5, 5.5]), " "RangeParameter(name='b', parameter_type=INT, range=[2, 10]), " "ChoiceParameter(name='c', parameter_type=STRING, " "values=['foo', 'bar', 'baz']), " "FixedParameter(name='d', parameter_type=BOOL, value=True), " "ChoiceParameter(name='e', parameter_type=FLOAT, " "values=[0.0, 0.1, 0.2, 0.5]), " "RangeParameter(name='f', parameter_type=INT, range=[2, 10], " "log_scale=True)], " "parameter_constraints=[OrderConstraint(a <= b)])" )
def search_space_for_range_values(min: float = 3.0, max: float = 6.0) -> SearchSpace: return SearchSpace( [ RangeParameter("x", ParameterType.FLOAT, min, max), RangeParameter("y", ParameterType.FLOAT, min, max), ] )
def testExtractSearchSpaceDigest(self): search_space_digest = extract_search_space_digest( self.search_space, ["x", "y", "z"]) self.assertEqual(search_space_digest.bounds, [(0.0, 1.0), (1.0, 2.0), (0.0, 5.0)]) self.assertEqual(search_space_digest.task_features, []) self.assertEqual(search_space_digest.target_fidelities, {1: 2.0}) search_space_digest = extract_search_space_digest( self.search_space, ["x", "z"]) self.assertEqual(search_space_digest.target_fidelities, {}) # Test that Int param is treated as task feature search_space = SearchSpace(self.parameters) search_space._parameters["x"] = RangeParameter("x", ParameterType.INT, lower=1, upper=4) search_space_digest = extract_search_space_digest( search_space, ["x", "y", "z"]) self.assertEqual(search_space_digest.task_features, [0]) # Test validation search_space._parameters["x"] = ChoiceParameter( "x", ParameterType.FLOAT, [0.1, 0.4]) with self.assertRaises(ValueError): extract_search_space_digest(search_space, ["x", "y", "z"]) search_space._parameters["x"] = RangeParameter("x", ParameterType.FLOAT, lower=1.0, upper=4.0, log_scale=True) with self.assertRaises(ValueError): extract_search_space_digest(search_space, ["x", "y", "z"])
def get_branin_search_space( with_fidelity_parameter: bool = False, with_choice_parameter: bool = False ) -> SearchSpace: parameters = [ RangeParameter( name="x1", parameter_type=ParameterType.FLOAT, lower=-5, upper=10 ), ChoiceParameter( name="x2", parameter_type=ParameterType.FLOAT, values=[float(x) for x in range(0, 16)], ) if with_choice_parameter else RangeParameter( name="x2", parameter_type=ParameterType.FLOAT, lower=0, upper=15 ), ] if with_fidelity_parameter: parameters.append( RangeParameter( name="fidelity", parameter_type=ParameterType.FLOAT, lower=0.0, upper=1.0, is_fidelity=True, target_value=1.0, ) ) return SearchSpace(parameters=cast(List[Parameter], parameters))
def testRoundingWithImpossiblyConstrainedIntRanges(self): parameters = [ RangeParameter("x", lower=1, upper=3, parameter_type=ParameterType.INT), RangeParameter("y", lower=1, upper=3, parameter_type=ParameterType.INT), ] constrained_int_search_space = SearchSpace( parameters=parameters, parameter_constraints=[ SumConstraint(parameters=parameters, is_upper_bound=True, bound=3) ], ) t = IntToFloat( search_space=constrained_int_search_space, observation_features=None, observation_data=None, ) self.assertEqual(t.rounding, "randomized") observation_features = [ ObservationFeatures(parameters={ "x": 2.6, "y": 2.6 }) ] self.assertFalse( constrained_int_search_space.check_membership( t.untransform_observation_features( observation_features=observation_features)[0].parameters))
def setUp(self): self.search_space = SearchSpace( parameters=[ RangeParameter( "x", lower=1, upper=3, parameter_type=ParameterType.FLOAT ), RangeParameter("a", lower=1, upper=2, parameter_type=ParameterType.INT), ChoiceParameter( "b", parameter_type=ParameterType.FLOAT, values=[1.0, 10.0, 100.0], is_ordered=True, ), ChoiceParameter( "c", parameter_type=ParameterType.FLOAT, values=[10.0, 100.0, 1000.0], is_ordered=True, ), ChoiceParameter( "d", parameter_type=ParameterType.STRING, values=["r", "q", "z"] ), ], parameter_constraints=[ ParameterConstraint(constraint_dict={"x": -0.5, "a": 1}, bound=0.5) ], ) self.t = OrderedChoiceEncode( search_space=self.search_space, observation_features=None, observation_data=None, )
def setUp(self): x = RangeParameter("x", ParameterType.FLOAT, lower=0, upper=1) y = RangeParameter("y", ParameterType.FLOAT, lower=1, upper=2) z = RangeParameter("z", ParameterType.FLOAT, lower=0, upper=5) self.parameters = [x, y, z] parameter_constraints = [ OrderConstraint(x, y), SumConstraint([x, z], False, 3.5), ] self.search_space = SearchSpace(self.parameters, parameter_constraints) self.observation_features = [ ObservationFeatures(parameters={ "x": 0.2, "y": 1.2, "z": 3 }), ObservationFeatures(parameters={ "x": 0.4, "y": 1.4, "z": 3 }), ObservationFeatures(parameters={ "x": 0.6, "y": 1.6, "z": 3 }), ] self.observation_data = [ ObservationData( metric_names=["a", "b"], means=np.array([1.0, -1.0]), covariance=np.array([[1.0, 4.0], [4.0, 6.0]]), ), ObservationData( metric_names=["a", "b"], means=np.array([2.0, -2.0]), covariance=np.array([[2.0, 5.0], [5.0, 7.0]]), ), ObservationData(metric_names=["a"], means=np.array([3.0]), covariance=np.array([[3.0]])), ] self.observations = [ Observation( features=self.observation_features[i], data=self.observation_data[i], arm_name=str(i), ) for i in range(3) ] self.pending_observations = { "b": [ObservationFeatures(parameters={ "x": 0.6, "y": 1.6, "z": 3 })] } self.model_gen_options = {"option": "yes"}
def get_discrete_search_space() -> SearchSpace: return SearchSpace( [ RangeParameter("x", ParameterType.INT, 0, 3), RangeParameter("y", ParameterType.INT, 5, 7), ChoiceParameter("z", ParameterType.STRING, ["red", "panda"]), ] )
def setUp(self): self.search_space = SearchSpace( parameters=[ RangeParameter( "x", lower=1, upper=3, parameter_type=ParameterType.FLOAT, ), RangeParameter("a", lower=1, upper=2, parameter_type=ParameterType.INT), ChoiceParameter( "b", parameter_type=ParameterType.STRING, values=["a", "b", "c"] ), ChoiceParameter( "c", parameter_type=ParameterType.BOOL, values=[True, False], is_ordered=False, ), ChoiceParameter( "d", parameter_type=ParameterType.FLOAT, values=[1.0, 10.0, 100.0], is_ordered=True, ), ], parameter_constraints=[ ParameterConstraint(constraint_dict={"x": -0.5, "a": 1}, bound=0.5) ], ) self.t = OneHot( search_space=self.search_space, observation_features=None, observation_data=None, ) self.t2 = OneHot( search_space=self.search_space, observation_features=None, observation_data=None, config={"rounding": "randomized"}, ) self.transformed_features = ObservationFeatures( parameters={ "x": 2.2, "a": 2, "b" + OH_PARAM_INFIX + "_0": 0, "b" + OH_PARAM_INFIX + "_1": 1, "b" + OH_PARAM_INFIX + "_2": 0, # Only two choices => one parameter. "c" + OH_PARAM_INFIX: 0, "d": 10.0, } ) self.observation_features = ObservationFeatures( parameters={"x": 2.2, "a": 2, "b": "b", "c": False, "d": 10.0} )
def setUp(self): self.search_space = SearchSpace( parameters=[ RangeParameter("x", lower=1, upper=3, parameter_type=ParameterType.FLOAT), RangeParameter("y", lower=1, upper=2, parameter_type=ParameterType.FLOAT), RangeParameter( "z", lower=1, upper=2, parameter_type=ParameterType.FLOAT, log_scale=True, ), RangeParameter("a", lower=1, upper=2, parameter_type=ParameterType.INT), ChoiceParameter("b", parameter_type=ParameterType.STRING, values=["a", "b", "c"]), ], parameter_constraints=[ ParameterConstraint(constraint_dict={ "x": -0.5, "y": 1 }, bound=0.5), ParameterConstraint(constraint_dict={ "x": -0.5, "a": 1 }, bound=0.5), ], ) self.t = UnitX( search_space=self.search_space, observation_features=None, observation_data=None, ) self.search_space_with_target = SearchSpace(parameters=[ RangeParameter( "x", lower=1, upper=3, parameter_type=ParameterType.FLOAT, is_fidelity=True, target_value=3, ) ])
def testBadCreations(self): with self.assertRaises(ValueError): RangeParameter("x", ParameterType.STRING, 1, 3) with self.assertRaises(ValueError): RangeParameter("x", ParameterType.FLOAT, 3, 1) with self.assertRaises(ValueError): RangeParameter("x", ParameterType.INT, 0, 1, log_scale=True) with self.assertRaises(ValueError): RangeParameter("x", ParameterType.INT, 0.5, 1)
def setUp(self): self.x = RangeParameter("x", ParameterType.INT, lower=-5, upper=5) self.y = RangeParameter("y", ParameterType.INT, lower=-5, upper=5) self.constraint1 = SumConstraint(parameters=[self.x, self.y], is_upper_bound=True, bound=5) self.constraint2 = SumConstraint(parameters=[self.x, self.y], is_upper_bound=False, bound=-5) self.constraint_repr1 = "SumConstraint(x + y <= 5.0)" self.constraint_repr2 = "SumConstraint(x + y >= -5.0)"
def setUp(self): x = RangeParameter("x", ParameterType.FLOAT, lower=0, upper=1) y = RangeParameter("y", ParameterType.FLOAT, lower=1, upper=2) z = RangeParameter("z", ParameterType.FLOAT, lower=0, upper=5) self.parameters = [x, y, z] parameter_constraints = [ OrderConstraint(x, y), SumConstraint([x, z], False, 3.5), ] self.search_space = SearchSpace(self.parameters, parameter_constraints) self.model_gen_options = {"option": "yes"}
def get_branin_search_space() -> SearchSpace: parameters = [ RangeParameter( name="x1", parameter_type=ParameterType.FLOAT, lower=-5, upper=10 ), RangeParameter( name="x2", parameter_type=ParameterType.FLOAT, lower=0, upper=15 ), ] # Expected `List[ax.core.parameter.Parameter]` for 2nd parameter # `parameters` to call `ax.core.search_space.SearchSpace.__init__` but got # `List[RangeParameter]`. # pyre-fixme[6]: return SearchSpace(parameters=parameters)
def transform_search_space(self, search_space: SearchSpace) -> SearchSpace: transformed_parameters: Dict[str, Parameter] = {} for p_name, p in search_space.parameters.items(): if p_name in self.encoded_parameters and isinstance( p, ChoiceParameter): if p.is_fidelity: raise ValueError( f"Cannot choice-encode fidelity parameter {p_name}") # Choice(|K|) => Range(0, K-1) transformed_parameters[p_name] = RangeParameter( name=p_name, parameter_type=ParameterType.INT, lower=0, upper=len(p.values) - 1, ) else: transformed_parameters[p.name] = p return SearchSpace( parameters=list(transformed_parameters.values()), parameter_constraints=[ pc.clone_with_transformed_parameters( transformed_parameters=transformed_parameters) for pc in search_space.parameter_constraints ], )
def get_num_boost_rounds_parameter() -> RangeParameter: return RangeParameter( name="num_boost_rounds", parameter_type=ParameterType.INT, lower=10, upper=20, )
def get_l2_reg_weight_parameter() -> RangeParameter: return RangeParameter( name="l2_reg_weight", parameter_type=ParameterType.FLOAT, lower=0.00001, upper=0.001, )
def get_lr_parameter() -> RangeParameter: return RangeParameter( name="learning_rate", parameter_type=ParameterType.FLOAT, lower=0.001, upper=0.1, )
def testTransformSearchSpace(self): ss2 = self.search_space.clone() ss2 = self.t.transform_search_space(ss2) self.assertEqual(len(ss2.parameters), 1) expected_parameter = ChoiceParameter( name="arms", parameter_type=ParameterType.STRING, values=list(self.t.signature_to_parameterization.keys()), ) self.assertEqual(ss2.parameters.get("arms"), expected_parameter) # Test error if there are fidelities ss3 = SearchSpace(parameters=[ RangeParameter( "a", lower=1, upper=3, parameter_type=ParameterType.FLOAT, is_fidelity=True, target_value=3, ) ]) with self.assertRaises(ValueError): SearchSpaceToChoice( search_space=ss3, observation_features=self.observation_features, observation_data=None, )
def transform_search_space(self, search_space: SearchSpace) -> SearchSpace: transformed_parameters: Dict[str, Parameter] = {} for p in search_space.parameters.values(): # Refine type, since we've only added RangeParameters above. if p.name in self.transform_parameters: # pyre: p_cast is declared to have type `RangeParameter` but # pyre-fixme[9]: is used as type `Parameter`. p_cast: RangeParameter = p transformed_parameters[p.name] = RangeParameter( name=p_cast.name, parameter_type=ParameterType.FLOAT, lower=p_cast.lower, upper=p_cast.upper, log_scale=p_cast.log_scale, digits=p_cast.digits, ) else: transformed_parameters[p.name] = p return SearchSpace( parameters=list(transformed_parameters.values()), parameter_constraints=[ pc.clone_with_transformed_parameters( transformed_parameters=transformed_parameters) for pc in search_space.parameter_constraints ], )
def test_GPKG(self): """Tests GPKG instantiation.""" exp = get_branin_experiment(with_batch=True) with self.assertRaises(ValueError): get_GPKG(experiment=exp, data=exp.fetch_data()) exp.trials[0].run() gpkg = get_GPKG(experiment=exp, data=exp.fetch_data()) self.assertIsInstance(gpkg, TorchModelBridge) # test transform_configs with winsorization configs = { "Winsorize": { "winsorization_lower": 0.1, "winsorization_upper": 0.1 } } gpkg_win = get_GPKG(experiment=exp, data=exp.fetch_data(), transform_configs=configs) self.assertIsInstance(gpkg_win, TorchModelBridge) self.assertEqual(gpkg_win._transform_configs, configs) # test multi-fidelity optimization exp.parameters["x2"] = RangeParameter( name="x2", parameter_type=exp.parameters["x2"].parameter_type, lower=-5.0, upper=10.0, is_fidelity=True, target_value=10.0, ) gpkg_mf = get_GPKG(experiment=exp, data=exp.fetch_data()) self.assertIsInstance(gpkg_mf, TorchModelBridge)
def transform_search_space(self, search_space: SearchSpace) -> SearchSpace: transformed_parameters: Dict[str, Parameter] = {} for p_name, p in search_space.parameters.items(): if p_name in self.transform_parameters and isinstance(p, RangeParameter): transformed_parameters[p_name] = RangeParameter( name=p_name, parameter_type=ParameterType.FLOAT, # +/- 0.5 ensures that sampling # 1) floating point numbers from (quasi-)Uniform(0,1) # 2) unnormalizing to the raw search space # 3) rounding # results in uniform (quasi-)random integers lower=p.lower - 0.49999, upper=p.upper + 0.49999, log_scale=p.log_scale, digits=p.digits, is_fidelity=p.is_fidelity, target_value=p.target_value, # casting happens in constructor ) else: transformed_parameters[p.name] = p return SearchSpace( parameters=list(transformed_parameters.values()), parameter_constraints=[ pc.clone_with_transformed_parameters( transformed_parameters=transformed_parameters ) for pc in search_space.parameter_constraints ], )
def transform_search_space(self, search_space: SearchSpace) -> SearchSpace: transformed_parameters: Dict[str, Parameter] = {} for p_name, p in search_space.parameters.items(): if p_name in self.transform_parameters and isinstance(p, RangeParameter): transformed_parameters[p_name] = RangeParameter( name=p_name, parameter_type=ParameterType.FLOAT, lower=p.lower, upper=p.upper, log_scale=p.log_scale, digits=p.digits, is_fidelity=p.is_fidelity, target_value=p.target_value, # casting happens in constructor ) else: transformed_parameters[p.name] = p return SearchSpace( parameters=list(transformed_parameters.values()), parameter_constraints=[ pc.clone_with_transformed_parameters( transformed_parameters=transformed_parameters ) for pc in search_space.parameter_constraints ], )