def setUp(self): self.constraint = ParameterConstraint(constraint_dict={ "x": 2.0, "y": -3.0 }, bound=6.0) self.constraint_repr = "ParameterConstraint(2.0*x + -3.0*y <= 6.0)"
def setUp(self): self.search_space = SearchSpace( parameters=[ RangeParameter("x", lower=1, upper=3, parameter_type=ParameterType.FLOAT), RangeParameter("y", lower=1, upper=2, parameter_type=ParameterType.FLOAT), RangeParameter( "z", lower=1, upper=2, parameter_type=ParameterType.FLOAT, log_scale=True, ), RangeParameter("a", lower=1, upper=2, parameter_type=ParameterType.INT), ChoiceParameter("b", parameter_type=ParameterType.STRING, values=["a", "b", "c"]), ], parameter_constraints=[ ParameterConstraint(constraint_dict={ "x": -0.5, "y": 1 }, bound=0.5), ParameterConstraint(constraint_dict={ "x": -0.5, "a": 1 }, bound=0.5), ], ) self.t = UnitX( search_space=self.search_space, observation_features=None, observation_data=None, ) self.search_space_with_target = SearchSpace(parameters=[ RangeParameter( "x", lower=1, upper=3, parameter_type=ParameterType.FLOAT, is_fidelity=True, target_value=3, ) ])
def testSortable(self): constraint1 = ParameterConstraint(constraint_dict={ "x": 2.0, "y": -3.0 }, bound=1.0) constraint2 = ParameterConstraint(constraint_dict={ "y": -3.0, "x": 2.0 }, bound=6.0) self.assertTrue(constraint1 < constraint2)
class ParameterConstraintTest(TestCase): def setUp(self): self.constraint = ParameterConstraint(constraint_dict={ "x": 2.0, "y": -3.0 }, bound=6.0) self.constraint_repr = "ParameterConstraint(2.0*x + -3.0*y <= 6.0)" def testEq(self): constraint1 = ParameterConstraint(constraint_dict={ "x": 2.0, "y": -3.0 }, bound=6.0) constraint2 = ParameterConstraint(constraint_dict={ "y": -3.0, "x": 2.0 }, bound=6.0) self.assertEqual(constraint1, constraint2) constraint3 = ParameterConstraint(constraint_dict={ "x": 2.0, "y": -5.0 }, bound=6.0) self.assertNotEqual(constraint1, constraint3) def testProperties(self): self.assertEqual(self.constraint.constraint_dict["x"], 2.0) self.assertEqual(self.constraint.bound, 6.0) def testRepr(self): self.assertEqual(str(self.constraint), self.constraint_repr) def testValidate(self): parameters = {"x": 4, "z": 3} with self.assertRaises(ValueError): self.constraint.check(parameters) parameters = {"x": 4, "y": 1} self.assertTrue(self.constraint.check(parameters)) self.constraint.bound = 4.0 self.assertFalse(self.constraint.check(parameters)) def testClone(self): constraint_clone = self.constraint.clone() self.assertEqual(self.constraint.bound, constraint_clone.bound) constraint_clone._bound = 7.0 self.assertNotEqual(self.constraint.bound, constraint_clone.bound)
def setUp(self): self.search_space = SearchSpace( parameters=[ RangeParameter("x", lower=1, upper=3, parameter_type=ParameterType.FLOAT), RangeParameter("a", lower=1, upper=2, parameter_type=ParameterType.INT), ChoiceParameter( "b", parameter_type=ParameterType.FLOAT, values=[1.0, 10.0, 100.0], is_ordered=True, ), ChoiceParameter( "c", parameter_type=ParameterType.FLOAT, values=[10.0, 100.0, 1000.0], is_ordered=True, ), ChoiceParameter("d", parameter_type=ParameterType.STRING, values=["r", "q", "z"]), ], parameter_constraints=[ ParameterConstraint(constraint_dict={ "x": -0.5, "a": 1 }, bound=0.5) ], ) self.t = ChoiceEncode( search_space=self.search_space, observation_features=None, observation_data=None, ) self.observation_features = [ ObservationFeatures(parameters={ "x": 2.2, "a": 2, "b": 10.0, "c": 10.0, "d": "r" }) ] # expected parameters after transform self.expected_transformed_params = { "x": 2.2, "a": 2, # ordered float choice originally; transformed normalized value "b": normalize_values([1.0, 10.0, 100.0])[1], # ordered float choice originally; transformed normalized value "c": normalize_values([10.0, 100.0, 1000.0])[0], # string choice originally; transformed to int index. "d": 0, }
def testTransformSearchSpace(self): ss2 = deepcopy(self.search_space) ss2 = self.t.transform_search_space(ss2) # Parameters transformed true_bounds = { "x": (-1.0, 1.0), "y": (-1.0, 1.0), "z": (1.0, 2.0), "a": (1.0, 2.0), } for p_name, (l, u) in true_bounds.items(): self.assertEqual(ss2.parameters[p_name].lower, l) self.assertEqual(ss2.parameters[p_name].upper, u) self.assertEqual(ss2.parameters["b"].values, ["a", "b", "c"]) self.assertEqual(len(ss2.parameters), 5) # Constraints error ss2 = deepcopy(self.search_space) ss2.add_parameter_constraints([ ParameterConstraint(constraint_dict={ "x": -0.5, "y": 1 }, bound=0.5) ]) with self.assertRaises(ValueError): ss2 = self.t.transform_search_space(ss2) t = CenteredUnitX( search_space=self.search_space_with_target, observation_features=None, observation_data=None, ) t.transform_search_space(self.search_space_with_target) self.assertEqual( self.search_space_with_target.parameters["x"].target_value, 1.0)
def setUp(self): self.search_space = SearchSpace( parameters=[ RangeParameter( "x", lower=1, upper=3, parameter_type=ParameterType.FLOAT ), RangeParameter("a", lower=1, upper=2, parameter_type=ParameterType.INT), ChoiceParameter( "b", parameter_type=ParameterType.FLOAT, values=[1.0, 10.0, 100.0], is_ordered=True, ), ChoiceParameter( "c", parameter_type=ParameterType.FLOAT, values=[10.0, 100.0, 1000.0], is_ordered=True, ), ChoiceParameter( "d", parameter_type=ParameterType.STRING, values=["r", "q", "z"] ), ], parameter_constraints=[ ParameterConstraint(constraint_dict={"x": -0.5, "a": 1}, bound=0.5) ], ) self.t = OrderedChoiceEncode( search_space=self.search_space, observation_features=None, observation_data=None, )
def get_parameter_constraint(param_x: str = "x", param_y: str = "w") -> ParameterConstraint: return ParameterConstraint(constraint_dict={ param_x: 1.0, param_y: -1.0 }, bound=1.0)
def transform_search_space(self, search_space: SearchSpace) -> SearchSpace: for p_name, p in search_space.parameters.items(): if p_name in self.bounds and isinstance(p, RangeParameter): p.update_range( lower=normalize_value(p.lower, self.bounds[p_name]), upper=normalize_value(p.upper, self.bounds[p_name]), ) if p.target_value is not None: p._target_value = normalize_value( p.target_value, self.bounds[p_name] # pyre-ignore[6] ) new_constraints: List[ParameterConstraint] = [] for c in search_space.parameter_constraints: constraint_dict: Dict[str, float] = {} bound = float(c.bound) for p_name, w in c.constraint_dict.items(): # p is RangeParameter, but may not be transformed (Int or log) if p_name in self.bounds: l, u = self.bounds[p_name] constraint_dict[p_name] = w * (u - l) bound -= w * l else: constraint_dict[p_name] = w new_constraints.append( ParameterConstraint(constraint_dict=constraint_dict, bound=bound)) search_space.set_parameter_constraints(new_constraints) return search_space
def parameter_constraint_from_sqa( self, parameter_constraint_sqa: SQAParameterConstraint, parameters: List[Parameter], ) -> ParameterConstraint: """Convert SQLAlchemy ParameterConstraint to Ax ParameterConstraint.""" parameter_map = {p.name: p for p in parameters} if parameter_constraint_sqa.type == ParameterConstraintType.ORDER: lower_name = None upper_name = None for k, v in parameter_constraint_sqa.constraint_dict.items(): if v == 1: lower_name = k elif v == -1: upper_name = k if not lower_name or not upper_name: raise SQADecodeError( "Cannot decode SQAParameterConstraint because `lower_name` or " "`upper_name` was not found." ) lower_parameter = parameter_map[lower_name] upper_parameter = parameter_map[upper_name] constraint = OrderConstraint( lower_parameter=lower_parameter, upper_parameter=upper_parameter ) elif parameter_constraint_sqa.type == ParameterConstraintType.SUM: # This operation is potentially very inefficient. # It is O(#constrained_parameters * #total_parameters) parameter_names = list(parameter_constraint_sqa.constraint_dict.keys()) constraint_parameters = [ next( search_space_param for search_space_param in parameters if search_space_param.name == c_p_name ) for c_p_name in parameter_names ] a_values = list(parameter_constraint_sqa.constraint_dict.values()) if len(a_values) == 0: raise SQADecodeError( "Cannot decode SQAParameterConstraint because `constraint_dict` " "is empty." ) a = a_values[0] is_upper_bound = a == 1 bound = parameter_constraint_sqa.bound * a constraint = SumConstraint( parameters=constraint_parameters, is_upper_bound=is_upper_bound, bound=bound, ) else: constraint = ParameterConstraint( constraint_dict=dict(parameter_constraint_sqa.constraint_dict), bound=parameter_constraint_sqa.bound, ) constraint.db_id = parameter_constraint_sqa.id return constraint
def setUp(self): self.search_space = SearchSpace( parameters=[ RangeParameter( "x", lower=1, upper=3, parameter_type=ParameterType.FLOAT, ), RangeParameter("a", lower=1, upper=2, parameter_type=ParameterType.INT), ChoiceParameter( "b", parameter_type=ParameterType.STRING, values=["a", "b", "c"] ), ChoiceParameter( "c", parameter_type=ParameterType.BOOL, values=[True, False], is_ordered=False, ), ChoiceParameter( "d", parameter_type=ParameterType.FLOAT, values=[1.0, 10.0, 100.0], is_ordered=True, ), ], parameter_constraints=[ ParameterConstraint(constraint_dict={"x": -0.5, "a": 1}, bound=0.5) ], ) self.t = OneHot( search_space=self.search_space, observation_features=None, observation_data=None, ) self.t2 = OneHot( search_space=self.search_space, observation_features=None, observation_data=None, config={"rounding": "randomized"}, ) self.transformed_features = ObservationFeatures( parameters={ "x": 2.2, "a": 2, "b" + OH_PARAM_INFIX + "_0": 0, "b" + OH_PARAM_INFIX + "_1": 1, "b" + OH_PARAM_INFIX + "_2": 0, # Only two choices => one parameter. "c" + OH_PARAM_INFIX: 0, "d": 10.0, } ) self.observation_features = ObservationFeatures( parameters={"x": 2.2, "a": 2, "b": "b", "c": False, "d": 10.0} )
def testEq(self): constraint1 = ParameterConstraint(constraint_dict={ "x": 2.0, "y": -3.0 }, bound=6.0) constraint2 = ParameterConstraint(constraint_dict={ "y": -3.0, "x": 2.0 }, bound=6.0) self.assertEqual(constraint1, constraint2) constraint3 = ParameterConstraint(constraint_dict={ "x": 2.0, "y": -5.0 }, bound=6.0) self.assertNotEqual(constraint1, constraint3)
def transform_search_space(self, search_space: SearchSpace) -> SearchSpace: for p_name, p in search_space.parameters.items(): if isinstance(p, RangeParameter) and p_name in self.bounds: p.update_range(lower=0.0, upper=1.0) new_constraints: List[ParameterConstraint] = [] for c in search_space.parameter_constraints: constraint_dict: Dict[str, float] = {} bound = float(c.bound) for p_name, w in c.constraint_dict.items(): # p is RangeParameter, but may not be transformed (Int or log) if p_name in self.bounds: l, u = self.bounds[p_name] constraint_dict[p_name] = w * (u - l) bound -= w * l else: constraint_dict[p_name] = w new_constraints.append( ParameterConstraint(constraint_dict=constraint_dict, bound=bound) ) search_space.set_parameter_constraints(new_constraints) return search_space
def setUp(self): self.search_space = SearchSpace( parameters=[ RangeParameter("x", lower=1, upper=3, parameter_type=ParameterType.FLOAT), RangeParameter("a", lower=1, upper=2, parameter_type=ParameterType.INT), RangeParameter("d", lower=1, upper=3, parameter_type=ParameterType.INT), ChoiceParameter("b", parameter_type=ParameterType.STRING, values=["a", "b", "c"]), ], parameter_constraints=[ ParameterConstraint(constraint_dict={ "x": -0.5, "a": 1 }, bound=0.5) ], ) self.t = IntToFloat( search_space=self.search_space, observation_features=None, observation_data=None, ) self.t2 = IntToFloat( search_space=self.search_space, observation_features=None, observation_data=None, config={"rounding": "randomized"}, )
def testTransformSearchSpace(self): ss2 = deepcopy(self.search_space) ss2 = self.t.transform_search_space(ss2) # Parameters transformed true_bounds = { "x": (-1.0, 1.0), "y": (-1.0, 1.0), "z": (1.0, 2.0), "a": (1.0, 2.0), } for p_name, (l, u) in true_bounds.items(): self.assertEqual(ss2.parameters[p_name].lower, l) self.assertEqual(ss2.parameters[p_name].upper, u) self.assertEqual(ss2.parameters["b"].values, ["a", "b", "c"]) self.assertEqual(len(ss2.parameters), 5) # Constraints error ss2 = deepcopy(self.search_space) ss2.add_parameter_constraints( [ParameterConstraint(constraint_dict={"x": -0.5, "y": 1}, bound=0.5)] ) with self.assertRaises(ValueError): ss2 = self.t.transform_search_space(ss2)
def constraint_from_str( representation: str, parameters: Dict[str, Parameter] ) -> ParameterConstraint: """Parse string representation of a parameter constraint.""" tokens = representation.split() parameter_names = parameters.keys() order_const = len(tokens) == 3 and tokens[1] in COMPARISON_OPS sum_const = ( len(tokens) >= 5 and len(tokens) % 2 == 1 and tokens[-2] in COMPARISON_OPS ) if not (order_const or sum_const): raise ValueError( "Parameter constraint should be of form <parameter_name> >= " "<other_parameter_name> for order constraints or `<parameter_name> " "+ <other_parameter_name> >= x, where any number of terms can be " "added and `x` is a float bound. Acceptable comparison operators " 'are ">=" and "<=".' ) if len(tokens) == 3: # Case "x1 >= x2" => order constraint. left, right = tokens[0], tokens[2] assert left in parameter_names, f"Parameter {left} not in {parameter_names}." assert right in parameter_names, f"Parameter {right} not in {parameter_names}." return ( OrderConstraint( lower_parameter=parameters[left], upper_parameter=parameters[right] ) if COMPARISON_OPS[tokens[1]] is ComparisonOp.LEQ else OrderConstraint( lower_parameter=parameters[right], upper_parameter=parameters[left] ) ) try: # Case "x1 - 2*x2 + x3 >= 2" => parameter constraint. bound = float(tokens[-1]) except ValueError: raise ValueError(f"Bound for the constraint must be a number; got {tokens[-1]}") if any(token[0] == "*" or token[-1] == "*" for token in tokens): raise ValueError( "A linear constraint should be the form a*x + b*y - c*z <= d" ", where a,b,c,d are float constants and x,y,z are parameters. " "There should be no space in each term around the operator * while " "there should be a single space around each operator +, -, <= and >=." ) parameter_weight = {} comparison_multiplier = ( 1.0 if COMPARISON_OPS[tokens[-2]] is ComparisonOp.LEQ else -1.0 ) operator_sign = 1.0 # Determines whether the operator is + or - for idx, token in enumerate(tokens[:-2]): if idx % 2 == 0: split_token = token.split("*") parameter = "" # Initializing the parameter multiplier = 1.0 # Initializing the multiplier if len(split_token) == 2: # There is a non-unit multiplier try: multiplier = float(split_token[0]) except ValueError: raise ValueError( f"Multiplier should be float; got {split_token[0]}" ) parameter = split_token[1] elif len(split_token) == 1: # The multiplier is either -1 or 1 parameter = split_token[0] if parameter[0] == "-": # The multiplier is -1 parameter = parameter[1:] multiplier = -1.0 else: multiplier = 1.0 assert ( parameter in parameter_names ), f"Parameter {parameter} not in {parameter_names}." parameter_weight[parameter] = operator_sign * multiplier else: assert ( token == "+" or token == "-" ), f"Expected a mixed constraint, found operator {token}." operator_sign = 1.0 if token == "+" else -1.0 return ParameterConstraint( constraint_dict={ p: comparison_multiplier * parameter_weight[p] for p in parameter_weight }, bound=comparison_multiplier * bound, )
def get_parameter_constraint() -> ParameterConstraint: return ParameterConstraint(constraint_dict={"x": 1.0, "w": -1.0}, bound=1.0)
def test_infer_objective_thresholds(self, _, cuda=False): # lightweight test exp = get_branin_experiment_with_multi_objective( has_optimization_config=True, with_batch=True, with_status_quo=True, ) for trial in exp.trials.values(): trial.mark_running(no_runner_required=True).mark_completed() exp.attach_data( get_branin_data_multi_objective(trial_indices=exp.trials.keys()) ) data = exp.fetch_data() modelbridge = TorchModelBridge( search_space=exp.search_space, model=MultiObjectiveBotorchModel(), optimization_config=exp.optimization_config, transforms=Cont_X_trans + Y_trans, torch_device=torch.device("cuda" if cuda else "cpu"), experiment=exp, data=data, ) fixed_features = ObservationFeatures(parameters={"x1": 0.0}) search_space = exp.search_space.clone() param_constraints = [ ParameterConstraint(constraint_dict={"x1": 1.0}, bound=10.0) ] search_space.add_parameter_constraints(param_constraints) oc = exp.optimization_config.clone() oc.objective._objectives[0].minimize = True expected_base_gen_args = modelbridge._get_transformed_gen_args( search_space=search_space.clone(), optimization_config=oc, fixed_features=fixed_features, ) with ExitStack() as es: mock_model_infer_obj_t = es.enter_context( patch( "ax.modelbridge.torch.infer_objective_thresholds", wraps=infer_objective_thresholds, ) ) mock_get_transformed_gen_args = es.enter_context( patch.object( modelbridge, "_get_transformed_gen_args", wraps=modelbridge._get_transformed_gen_args, ) ) mock_get_transformed_model_gen_args = es.enter_context( patch.object( modelbridge, "_get_transformed_model_gen_args", wraps=modelbridge._get_transformed_model_gen_args, ) ) mock_untransform_objective_thresholds = es.enter_context( patch.object( modelbridge, "_untransform_objective_thresholds", wraps=modelbridge._untransform_objective_thresholds, ) ) obj_thresholds = modelbridge.infer_objective_thresholds( search_space=search_space, optimization_config=oc, fixed_features=fixed_features, ) expected_obj_weights = torch.tensor([-1.0, 1.0]) ckwargs = mock_model_infer_obj_t.call_args[1] self.assertTrue( torch.equal(ckwargs["objective_weights"], expected_obj_weights) ) # check that transforms have been applied (at least UnitX) self.assertEqual(ckwargs["bounds"], [(0.0, 1.0), (0.0, 1.0)]) lc = ckwargs["linear_constraints"] self.assertTrue(torch.equal(lc[0], torch.tensor([[15.0, 0.0]]))) self.assertTrue(torch.equal(lc[1], torch.tensor([[15.0]]))) self.assertEqual(ckwargs["fixed_features"], {0: 1.0 / 3.0}) mock_get_transformed_gen_args.assert_called_once() mock_get_transformed_model_gen_args.assert_called_once_with( search_space=expected_base_gen_args.search_space, fixed_features=expected_base_gen_args.fixed_features, pending_observations=expected_base_gen_args.pending_observations, optimization_config=expected_base_gen_args.optimization_config, ) mock_untransform_objective_thresholds.assert_called_once() ckwargs = mock_untransform_objective_thresholds.call_args[1] self.assertTrue( torch.equal(ckwargs["objective_weights"], expected_obj_weights) ) self.assertEqual(ckwargs["bounds"], [(0.0, 1.0), (0.0, 1.0)]) self.assertEqual(ckwargs["fixed_features"], {0: 1.0 / 3.0}) self.assertEqual(obj_thresholds[0].metric.name, "branin_a") self.assertEqual(obj_thresholds[1].metric.name, "branin_b") self.assertEqual(obj_thresholds[0].op, ComparisonOp.LEQ) self.assertEqual(obj_thresholds[1].op, ComparisonOp.GEQ) self.assertFalse(obj_thresholds[0].relative) self.assertFalse(obj_thresholds[1].relative) df = exp_to_df(exp) Y = np.stack([df.branin_a.values, df.branin_b.values]).T Y = torch.from_numpy(Y) Y[:, 0] *= -1 pareto_Y = Y[is_non_dominated(Y)] nadir = pareto_Y.min(dim=0).values self.assertTrue( np.all( np.array([-obj_thresholds[0].bound, obj_thresholds[1].bound]) < nadir.numpy() ) ) # test using MTGP sobol_generator = get_sobol( search_space=exp.search_space, seed=TEST_SOBOL_SEED, # set initial position equal to the number of sobol arms generated # so far. This means that new sobol arms will complement the previous # arms in a space-filling fashion init_position=len(exp.arms_by_name) - 1, ) sobol_run = sobol_generator.gen(n=2) trial = exp.new_batch_trial(optimize_for_power=True) trial.add_generator_run(sobol_run) trial.mark_running(no_runner_required=True).mark_completed() data = exp.fetch_data() torch.manual_seed(0) # make model fitting deterministic modelbridge = TorchModelBridge( search_space=exp.search_space, model=MultiObjectiveBotorchModel(), optimization_config=exp.optimization_config, transforms=ST_MTGP_trans, experiment=exp, data=data, ) fixed_features = ObservationFeatures(parameters={}, trial_index=1) expected_base_gen_args = modelbridge._get_transformed_gen_args( search_space=search_space.clone(), optimization_config=exp.optimization_config, fixed_features=fixed_features, ) with ExitStack() as es: mock_model_infer_obj_t = es.enter_context( patch( "ax.modelbridge.torch.infer_objective_thresholds", wraps=infer_objective_thresholds, ) ) mock_untransform_objective_thresholds = es.enter_context( patch.object( modelbridge, "_untransform_objective_thresholds", wraps=modelbridge._untransform_objective_thresholds, ) ) obj_thresholds = modelbridge.infer_objective_thresholds( search_space=search_space, optimization_config=exp.optimization_config, fixed_features=fixed_features, ) ckwargs = mock_model_infer_obj_t.call_args[1] self.assertEqual(ckwargs["fixed_features"], {2: 1.0}) mock_untransform_objective_thresholds.assert_called_once() ckwargs = mock_untransform_objective_thresholds.call_args[1] self.assertEqual(ckwargs["fixed_features"], {2: 1.0}) self.assertEqual(obj_thresholds[0].metric.name, "branin_a") self.assertEqual(obj_thresholds[1].metric.name, "branin_b") self.assertEqual(obj_thresholds[0].op, ComparisonOp.GEQ) self.assertEqual(obj_thresholds[1].op, ComparisonOp.GEQ) self.assertFalse(obj_thresholds[0].relative) self.assertFalse(obj_thresholds[1].relative) df = exp_to_df(exp) trial_mask = df.trial_index == 1 Y = np.stack([df.branin_a.values[trial_mask], df.branin_b.values[trial_mask]]).T Y = torch.from_numpy(Y) pareto_Y = Y[is_non_dominated(Y)] nadir = pareto_Y.min(dim=0).values self.assertTrue( np.all( np.array([obj_thresholds[0].bound, obj_thresholds[1].bound]) < nadir.numpy() ) )
def testBadConstruction(self): # Duplicate parameter with self.assertRaises(ValueError): p1 = self.parameters + [self.parameters[0]] SearchSpace(parameters=p1, parameter_constraints=[]) # Constraint on non-existent parameter with self.assertRaises(ValueError): SearchSpace( parameters=self.parameters, parameter_constraints=[ OrderConstraint(lower_parameter=self.a, upper_parameter=self.g) ], ) # Vanilla Constraint on non-existent parameter with self.assertRaises(ValueError): SearchSpace( parameters=self.parameters, parameter_constraints=[ ParameterConstraint(constraint_dict={"g": 1}, bound=0) ], ) # Constraint on non-numeric parameter with self.assertRaises(ValueError): SearchSpace( parameters=self.parameters, parameter_constraints=[ OrderConstraint(lower_parameter=self.a, upper_parameter=self.d) ], ) # Constraint on choice parameter with self.assertRaises(ValueError): SearchSpace( parameters=self.parameters, parameter_constraints=[ OrderConstraint(lower_parameter=self.a, upper_parameter=self.e) ], ) # Constraint on logscale parameter with self.assertRaises(ValueError): SearchSpace( parameters=self.parameters, parameter_constraints=[ OrderConstraint(lower_parameter=self.a, upper_parameter=self.f) ], ) # Constraint on mismatched parameter with self.assertRaises(ValueError): wrong_a = self.a.clone() wrong_a.update_range(upper=10) SearchSpace( parameters=self.parameters, parameter_constraints=[ OrderConstraint(lower_parameter=wrong_a, upper_parameter=self.b) ], )
class ParameterConstraintTest(TestCase): def setUp(self): self.constraint = ParameterConstraint(constraint_dict={ "x": 2.0, "y": -3.0 }, bound=6.0) self.constraint_repr = "ParameterConstraint(2.0*x + -3.0*y <= 6.0)" def testEq(self): constraint1 = ParameterConstraint(constraint_dict={ "x": 2.0, "y": -3.0 }, bound=6.0) constraint2 = ParameterConstraint(constraint_dict={ "y": -3.0, "x": 2.0 }, bound=6.0) self.assertEqual(constraint1, constraint2) constraint3 = ParameterConstraint(constraint_dict={ "x": 2.0, "y": -5.0 }, bound=6.0) self.assertNotEqual(constraint1, constraint3) def testProperties(self): self.assertEqual(self.constraint.constraint_dict["x"], 2.0) self.assertEqual(self.constraint.bound, 6.0) def testRepr(self): self.assertEqual(str(self.constraint), self.constraint_repr) def testValidate(self): parameters = {"x": 4, "z": 3} with self.assertRaises(ValueError): self.constraint.check(parameters) # check slack constraint parameters = {"x": 4, "y": 1} self.assertTrue(self.constraint.check(parameters)) # check tight constraint (within numerical tolerance) parameters = {"x": 4, "y": (2 - 0.5e-8) / 3} self.assertTrue(self.constraint.check(parameters)) # check violated constraint parameters = {"x": 4, "y": (2 - 0.5e-6) / 3} self.assertFalse(self.constraint.check(parameters)) def testClone(self): constraint_clone = self.constraint.clone() self.assertEqual(self.constraint.bound, constraint_clone.bound) constraint_clone._bound = 7.0 self.assertNotEqual(self.constraint.bound, constraint_clone.bound) def testCloneWithTransformedParameters(self): constraint_clone = self.constraint.clone_with_transformed_parameters( transformed_parameters={}) self.assertEqual(self.constraint.bound, constraint_clone.bound) constraint_clone._bound = 7.0 self.assertNotEqual(self.constraint.bound, constraint_clone.bound) def testSortable(self): constraint1 = ParameterConstraint(constraint_dict={ "x": 2.0, "y": -3.0 }, bound=1.0) constraint2 = ParameterConstraint(constraint_dict={ "y": -3.0, "x": 2.0 }, bound=6.0) self.assertTrue(constraint1 < constraint2)
def constraint_from_str( representation: str, parameters: Dict[str, Parameter] ) -> ParameterConstraint: """Parse string representation of a parameter constraint.""" tokens = representation.split() parameter_names = parameters.keys() order_const = len(tokens) == 3 and tokens[1] in COMPARISON_OPS sum_const = ( (len(tokens) >= 5) and (len(tokens) % 2 == 1) and (tokens[-2] in COMPARISON_OPS) and ("*" not in tokens) ) parameter_const = ( (len(tokens) >= 5) and (len(tokens) % 2 == 1) and (tokens[-2] in COMPARISON_OPS) and ("*" in tokens) ) if not (order_const or sum_const or parameter_const): raise ValueError( "Parameter constraint should be of form" "order constraint: <parameter_name> >= <other_parameter_name>" "sum constraint: `<parameter_name> + <other_parameter_name> >= x`, where any number of parameters can be " "summed up and `x` is a float bound." "Acceptable comparison operators are >= and <=" "parameter constraint: `<weight> * <parameter_name> + <other_weight> * <other_parameter_name> <= x`", "where any number of parameters and weights can be" "summed up and `x` is a float bound." "Only comparison operator is <=" "<weight> can be negative", ) if order_const: # Case "x1 >= x2" => order constraint. left, right = tokens[0], tokens[2] assert left in parameter_names, f"Parameter {left} not in {parameter_names}." assert right in parameter_names, f"Parameter {right} not in {parameter_names}." return ( OrderConstraint( lower_parameter=parameters[left], upper_parameter=parameters[right] ) if COMPARISON_OPS[tokens[1]] is ComparisonOp.LEQ else OrderConstraint( lower_parameter=parameters[right], upper_parameter=parameters[left] ) ) try: bound = float(tokens[-1]) except ValueError: raise ValueError( f"Bound for sum or parameter constraint must be a number; got {tokens[-1]}" ) used_parameters = [] used_weights = [] if sum_const: for idx, token in enumerate(tokens[:-2]): if idx % 2 == 0: assert ( token in parameter_names ), f"Parameter {token} not in {parameter_names}." used_parameters.append(token) else: assert ( token == "+" ), f"Expected a sum constraint, found operator {token}." return SumConstraint( parameters=[parameters[p] for p in parameters if p in used_parameters], is_upper_bound=COMPARISON_OPS[tokens[-2]] is ComparisonOp.LEQ, bound=bound, ) if parameter_const: for idx, token in enumerate(tokens[:-2]): if idx % 4 == 0: try: weight = float(token) used_weights.append(weight) except: raise ValueError( f"Weight for parameter constraint must be a number; got {token}" ) elif idx % 4 == 1: assert ( token == "*" ), f"Expected a multiplication, found operator {token}." elif idx % 4 == 2: assert ( token in parameter_names ), f"Parameter {token} not in {parameter_names}." used_parameters.append(token) else: assert ( token == "+" ), f"Expected a sum constraint, found operator {token}." return ParameterConstraint( constraint_dict=dict(zip(used_parameters, used_weights)), bound=bound )
def test_infer_objective_thresholds(self, _, cuda=False): # lightweight test exp = get_branin_experiment_with_multi_objective( has_optimization_config=True, with_batch=True, with_status_quo=True, ) for trial in exp.trials.values(): trial.mark_running(no_runner_required=True).mark_completed() exp.attach_data( get_branin_data_multi_objective(trial_indices=exp.trials.keys())) data = exp.fetch_data() modelbridge = MultiObjectiveTorchModelBridge( search_space=exp.search_space, model=MultiObjectiveBotorchModel(), optimization_config=exp.optimization_config, transforms=Cont_X_trans + Y_trans, torch_device=torch.device("cuda" if cuda else "cpu"), experiment=exp, data=data, ) fixed_features = ObservationFeatures(parameters={"x1": 0.0}) search_space = exp.search_space.clone() param_constraints = [ ParameterConstraint(constraint_dict={"x1": 1.0}, bound=10.0) ] outcome_constraints = [ OutcomeConstraint( metric=exp.metrics["branin_a"], op=ComparisonOp.GEQ, bound=-40.0, relative=False, ) ] search_space.add_parameter_constraints(param_constraints) exp.optimization_config.outcome_constraints = outcome_constraints oc = exp.optimization_config.clone() oc.objective._objectives[0].minimize = True expected_base_gen_args = modelbridge._get_transformed_gen_args( search_space=search_space.clone(), optimization_config=oc, fixed_features=fixed_features, ) with ExitStack() as es: mock_model_infer_obj_t = es.enter_context( patch( "ax.modelbridge.multi_objective_torch.infer_objective_thresholds", wraps=infer_objective_thresholds, )) mock_get_transformed_gen_args = es.enter_context( patch.object( modelbridge, "_get_transformed_gen_args", wraps=modelbridge._get_transformed_gen_args, )) mock_get_transformed_model_gen_args = es.enter_context( patch.object( modelbridge, "_get_transformed_model_gen_args", wraps=modelbridge._get_transformed_model_gen_args, )) mock_untransform_objective_thresholds = es.enter_context( patch.object( modelbridge, "untransform_objective_thresholds", wraps=modelbridge.untransform_objective_thresholds, )) obj_thresholds = modelbridge.infer_objective_thresholds( search_space=search_space, optimization_config=oc, fixed_features=fixed_features, ) expected_obj_weights = torch.tensor([-1.0, 1.0]) ckwargs = mock_model_infer_obj_t.call_args[1] self.assertTrue( torch.equal(ckwargs["objective_weights"], expected_obj_weights)) # check that transforms have been applied (at least UnitX) self.assertEqual(ckwargs["bounds"], [(0.0, 1.0), (0.0, 1.0)]) oc = ckwargs["outcome_constraints"] self.assertTrue(torch.equal(oc[0], torch.tensor([[-1.0, 0.0]]))) self.assertTrue(torch.equal(oc[1], torch.tensor([[45.0]]))) lc = ckwargs["linear_constraints"] self.assertTrue(torch.equal(lc[0], torch.tensor([[15.0, 0.0]]))) self.assertTrue(torch.equal(lc[1], torch.tensor([[15.0]]))) self.assertEqual(ckwargs["fixed_features"], {0: 1.0 / 3.0}) mock_get_transformed_gen_args.assert_called_once() mock_get_transformed_model_gen_args.assert_called_once_with( search_space=expected_base_gen_args.search_space, fixed_features=expected_base_gen_args.fixed_features, pending_observations=expected_base_gen_args. pending_observations, optimization_config=expected_base_gen_args.optimization_config, ) mock_untransform_objective_thresholds.assert_called_once() ckwargs = mock_untransform_objective_thresholds.call_args[1] self.assertTrue( torch.equal(ckwargs["objective_weights"], expected_obj_weights)) self.assertEqual(ckwargs["bounds"], [(0.0, 1.0), (0.0, 1.0)]) self.assertEqual(ckwargs["fixed_features"], {0: 1.0 / 3.0}) self.assertEqual(obj_thresholds[0].metric.name, "branin_a") self.assertEqual(obj_thresholds[1].metric.name, "branin_b") self.assertEqual(obj_thresholds[0].op, ComparisonOp.LEQ) self.assertEqual(obj_thresholds[1].op, ComparisonOp.GEQ) self.assertFalse(obj_thresholds[0].relative) self.assertFalse(obj_thresholds[1].relative) df = exp_to_df(exp) Y = np.stack([df.branin_a.values, df.branin_b.values]).T Y = torch.from_numpy(Y) Y[:, 0] *= -1 pareto_Y = Y[is_non_dominated(Y)] nadir = pareto_Y.min(dim=0).values self.assertTrue( np.all( np.array([-obj_thresholds[0].bound, obj_thresholds[1].bound]) < nadir.numpy())) # test using MTGP sobol_generator = get_sobol(search_space=exp.search_space) sobol_run = sobol_generator.gen(n=5) trial = exp.new_batch_trial(optimize_for_power=True) trial.add_generator_run(sobol_run) trial.mark_running(no_runner_required=True).mark_completed() data = exp.fetch_data() modelbridge = MultiObjectiveTorchModelBridge( search_space=exp.search_space, model=MultiObjectiveBotorchModel(), optimization_config=exp.optimization_config, transforms=ST_MTGP_trans, experiment=exp, data=data, ) fixed_features = ObservationFeatures(parameters={}, trial_index=1) expected_base_gen_args = modelbridge._get_transformed_gen_args( search_space=search_space.clone(), optimization_config=exp.optimization_config, fixed_features=fixed_features, ) with self.assertRaises(ValueError): # Check that a ValueError is raised when MTGP is being used # and trial_index is not specified as a fixed features. # Note: this error is raised by StratifiedStandardizeY modelbridge.infer_objective_thresholds( search_space=search_space, optimization_config=exp.optimization_config, ) with ExitStack() as es: mock_model_infer_obj_t = es.enter_context( patch( "ax.modelbridge.multi_objective_torch.infer_objective_thresholds", wraps=infer_objective_thresholds, )) mock_untransform_objective_thresholds = es.enter_context( patch.object( modelbridge, "untransform_objective_thresholds", wraps=modelbridge.untransform_objective_thresholds, )) obj_thresholds = modelbridge.infer_objective_thresholds( search_space=search_space, optimization_config=exp.optimization_config, fixed_features=fixed_features, ) ckwargs = mock_model_infer_obj_t.call_args[1] self.assertEqual(ckwargs["fixed_features"], {2: 1.0}) mock_untransform_objective_thresholds.assert_called_once() ckwargs = mock_untransform_objective_thresholds.call_args[1] self.assertEqual(ckwargs["fixed_features"], {2: 1.0}) self.assertEqual(obj_thresholds[0].metric.name, "branin_a") self.assertEqual(obj_thresholds[1].metric.name, "branin_b") self.assertEqual(obj_thresholds[0].op, ComparisonOp.GEQ) self.assertEqual(obj_thresholds[1].op, ComparisonOp.GEQ) self.assertFalse(obj_thresholds[0].relative) self.assertFalse(obj_thresholds[1].relative) df = exp_to_df(exp) trial_mask = df.trial_index == 1 Y = np.stack( [df.branin_a.values[trial_mask], df.branin_b.values[trial_mask]]).T Y = torch.from_numpy(Y) pareto_Y = Y[is_non_dominated(Y)] nadir = pareto_Y.min(dim=0).values self.assertTrue( np.all( np.array([obj_thresholds[0].bound, obj_thresholds[1].bound]) < nadir.numpy()))