def test_uniformfloat_transform(self): """This checks whether a value sampled through the configuration space (it does not happend when the variable is sampled alone) stays equal when it is serialized via JSON and the deserialized again.""" cs = ConfigurationSpace() a = cs.add_hyperparameter(UniformFloatHyperparameter('a', -5, 10)) b = cs.add_hyperparameter( NormalFloatHyperparameter('b', 1, 2, log=True)) for i in range(100): config = cs.sample_configuration() value = OrderedDict(sorted(config.get_dictionary().items())) string = json.dumps(value) saved_value = json.loads(string) saved_value = OrderedDict(sorted(byteify(saved_value).items())) self.assertEqual(repr(value), repr(saved_value)) # Next, test whether the truncation also works when initializing the # Configuration with a dictionary for i in range(100): rs = np.random.RandomState(1) value_a = a.sample(rs) value_b = b.sample(rs) values_dict = {'a': value_a, 'b': value_b} config = Configuration(cs, values=values_dict) string = json.dumps(config.get_dictionary()) saved_value = json.loads(string) saved_value = byteify(saved_value) self.assertEqual(values_dict, saved_value)
def test_add_hyperparameters_with_equal_names(self): cs = ConfigurationSpace() hp = UniformIntegerHyperparameter("name", 0, 10) cs.add_hyperparameter(hp) self.assertRaisesRegexp( ValueError, "Hyperparameter 'name' is already in the " "configuration space.", cs.add_hyperparameter, hp)
def setUp(self): logging.basicConfig(level=logging.DEBUG) self.cs = ConfigurationSpace() self.cs.add_hyperparameter(CategoricalHyperparameter( name="cat_a_b", choices=["a", "b"], default_value="a")) self.cs.add_hyperparameter(UniformFloatHyperparameter( name="float_0_1", lower=0, upper=1, default_value=0.5)) self.cs.add_hyperparameter(UniformIntegerHyperparameter( name='integer_0_100', lower=-10, upper=10, default_value=0)) self.rh = runhistory.RunHistory(aggregate_func=average_cost) rs = numpy.random.RandomState(1) to_count = 0 cn_count = 0 for i in range(500): config, seed, runtime, status, instance_id = \ generate_config(cs=self.cs, rs=rs) if runtime == 40: to_count += 1 if runtime < 40 and status == StatusType.TIMEOUT: cn_count += 1 self.rh.add(config=config, cost=runtime, time=runtime, status=status, instance_id=instance_id, seed=seed, additional_info=None) print("%d TIMEOUTs, %d censored" % (to_count, cn_count)) self.scen = Scen() self.scen.run_obj = "runtime" self.scen.overall_obj = "par10" self.scen.cutoff = 40 types, bounds = get_types(self.cs, None) self.model = RandomForestWithInstances( types=types, bounds=bounds, instance_features=None, seed=1234567980)
def setUp(self): cs = ConfigurationSpace() hp1 = cs.add_hyperparameter(CategoricalHyperparameter( "parent", [0, 1])) hp2 = cs.add_hyperparameter( UniformIntegerHyperparameter("child", 0, 10)) hp3 = cs.add_hyperparameter( UniformIntegerHyperparameter("friend", 0, 5)) self.cs = cs
def test_get_conditions(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) self.assertEqual([], cs.get_conditions()) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) self.assertEqual([cond1], cs.get_conditions())
def testRandomImputation(self): rs = numpy.random.RandomState(1) for i in range(0, 150, 15): # First random imputation sanity check num_samples = max(1, i*10) num_feat = max(1, i) num_censored = int(num_samples*0.1) X = rs.rand(num_samples, num_feat) y = numpy.sin(X[:, 0:1]) cutoff = max(y) * 0.9 y[y > cutoff] = cutoff # We have some cen data cen_X = X[:num_censored, :] cen_y = y[:num_censored] uncen_X = X[num_censored:, :] uncen_y = y[num_censored:] cen_y /= 2 cs = ConfigurationSpace() for i in range(num_feat): cs.add_hyperparameter(UniformFloatHyperparameter( name="a_%d" % i, lower=0, upper=1, default_value=0.5) ) types, bounds = get_types(cs, None) print(types) print(bounds) print('#'*120) print(cen_X) print(uncen_X) print('~'*120) self.model = RandomForestWithInstances(types=types, bounds=bounds, instance_features=None, seed=1234567980) imputor = rfr_imputator.RFRImputator(rng=rs, cutoff=cutoff, threshold=cutoff*10, change_threshold=0.01, max_iter=5, model=self.model) imp_y = imputor.impute(censored_X=cen_X, censored_y=cen_y, uncensored_X=uncen_X, uncensored_y=uncen_y) if imp_y is None: continue for idx in range(cen_y.shape[0]): self.assertGreater(imp_y[idx], cen_y[idx]) self.assertTrue(numpy.isfinite(imp_y).all())
def deactivate_inactive_hyperparameters( configuration: Dict, configuration_space: ConfigurationSpace, ): hyperparameters = configuration_space.get_hyperparameters() configuration = Configuration(configuration_space=configuration_space, values=configuration, allow_inactive_with_values=True) hps = deque() unconditional_hyperparameters = configuration_space.get_all_unconditional_hyperparameters() hyperparameters_with_children = list() for uhp in unconditional_hyperparameters: children = configuration_space._children_of[uhp] if len(children) > 0: hyperparameters_with_children.append(uhp) hps.extendleft(hyperparameters_with_children) inactive = set() while len(hps) > 0: hp = hps.pop() children = configuration_space._children_of[hp] for child in children: conditions = configuration_space._parent_conditions_of[child.name] for condition in conditions: if not condition.evaluate_vector(configuration.get_array()): dic = configuration.get_dictionary() try: del dic[child.name] except KeyError: continue configuration = Configuration( configuration_space=configuration_space, values=dic, allow_inactive_with_values=True) inactive.add(child.name) hps.appendleft(child.name) for hp in hyperparameters: if hp.name in inactive: dic = configuration.get_dictionary() try: del dic[hp.name] except KeyError: continue configuration = Configuration( configuration_space=configuration_space, values=dic, allow_inactive_with_values=True) return Configuration(configuration_space, values=configuration.get_dictionary())
def setUp(self): logging.basicConfig() self.logger = logging.getLogger(self.__module__ + '.' + self.__class__.__name__) self.logger.setLevel(logging.DEBUG) base_directory = os.path.split(__file__)[0] base_directory = os.path.abspath( os.path.join(base_directory, '..', '..')) self.current_dir = os.getcwd() os.chdir(base_directory) self.cs = ConfigurationSpace() self.test_scenario_dict = { 'algo': 'echo Hello', 'paramfile': 'test/test_files/scenario_test/param.pcs', 'execdir': '.', 'deterministic': 0, 'run_obj': 'runtime', 'overall_obj': 'mean10', 'cutoff_time': 5, 'wallclock-limit': 18000, 'instance_file': 'test/test_files/scenario_test/training.txt', 'test_instance_file': 'test/test_files/scenario_test/test.txt', 'feature_file': 'test/test_files/scenario_test/features.txt', 'output_dir': 'test/test_files/scenario_test/tmp_output' }
def test_illegal_default_configuration(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("loss", ["l1", "l2"], default_value='l1') hp2 = CategoricalHyperparameter("penalty", ["l1", "l2"], default_value='l1') cs.add_hyperparameter(hp1) cs.add_hyperparameter(hp2) forb1 = ForbiddenEqualsClause(hp1, "l1") forb2 = ForbiddenEqualsClause(hp2, "l1") forb3 = ForbiddenAndConjunction(forb1, forb2) # cs.add_forbidden_clause(forb3) self.assertRaisesRegexp( ValueError, "Given vector violates forbidden clause \(Forbidden: loss == \'l1\' && " "Forbidden: penalty == \'l1\'\)", cs.add_forbidden_clause, forb3)
def test_hyperparameters_with_valid_condition(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond) self.assertEqual(len(cs._hyperparameters), 2)
def test_meta_data_stored(self): meta_data = { 'additional': 'meta-data', 'useful': 'for integrations', 'input_id': 42 } cs = ConfigurationSpace(meta=dict(meta_data)) self.assertEqual(cs.meta, meta_data)
def test_keys(self): # A regression test to make sure issue #49 does no longer pop up. By # iterating over the configuration in the for loop, it should not raise # a KeyError if the child hyperparameter is inactive. cs = ConfigurationSpace() shrinkage = CategoricalHyperparameter( "shrinkage", ["None", "auto", "manual"], default_value="None", ) shrinkage_factor = UniformFloatHyperparameter( "shrinkage_factor", 0., 1., 0.5, ) cs.add_hyperparameters([shrinkage, shrinkage_factor]) cs.add_condition(EqualsCondition(shrinkage_factor, shrinkage, "manual")) for i in range(10): config = cs.sample_configuration() d = { hp_name: config[hp_name] for hp_name in config if config[hp_name] is not None }
def test_get_hyperparameters_topological_sort_simple(self): for iteration in range(10): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) # This automatically checks the configuration! Configuration(cs, dict(parent=0, child=5))
def test_get_hyperparameter(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) retval = cs.get_hyperparameter("parent") self.assertEqual(hp1, retval) retval = cs.get_hyperparameter("child") self.assertEqual(hp2, retval) self.assertRaises(KeyError, cs.get_hyperparameter, "grandfather")
def test_condition_with_cycles(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) cond2 = EqualsCondition(hp1, hp2, 0) self.assertRaisesRegexp( ValueError, "Hyperparameter configuration " "contains a cycle \[\['child', 'parent'\]\]", cs.add_condition, cond2)
def test_add_configuration_space(self): cs = ConfigurationSpace() hp1 = cs.add_hyperparameter(CategoricalHyperparameter( "input1", [0, 1])) forb1 = cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 1)) hp2 = cs.add_hyperparameter( UniformIntegerHyperparameter("child", 0, 10)) cond = cs.add_condition(EqualsCondition(hp2, hp1, 0)) cs2 = ConfigurationSpace() cs2.add_configuration_space('prefix', cs, delimiter='__') self.assertEqual( str(cs2), '''Configuration space object: Hyperparameters: prefix__child, Type: UniformInteger, Range: [0, 10], Default: 5 prefix__input1, Type: Categorical, Choices: {0, 1}, Default: 0 Conditions: prefix__child | prefix__input1 == 0 Forbidden Clauses: Forbidden: prefix__input1 == 1 ''')
def get_config_space(): cs = ConfigurationSpace() cs.add_hyperparameter( UniformIntegerHyperparameter(name='a', lower=0, upper=100)) cs.add_hyperparameter( UniformIntegerHyperparameter(name='b', lower=0, upper=100)) return cs
def test_check_configuration_input_checking(self): cs = ConfigurationSpace() self.assertRaisesRegexp( TypeError, "The method check_configuration must be called " "with an instance of %s. " "Your input was of type %s" % (Configuration, type("String")), cs.check_configuration, "String") # For the check configuration method with vector representation self.assertRaisesRegexp( TypeError, "The method check_configuration must" " be called with an instance of " "np.ndarray Your input was of type %s" % (type("String")), cs.check_configuration_vector_representation, "String")
def _test_random_neigbor(self, hp): cs = ConfigurationSpace() if not isinstance(hp, list): hp = [hp] for hp_ in hp: cs.add_hyperparameter(hp_) cs.seed(1) config = cs.sample_configuration() for i in range(100): new_config = get_random_neighbor(config, i) self.assertNotEqual(config, new_config)
def test_add_conjunction(self): hp1 = CategoricalHyperparameter("input1", [0, 1]) hp2 = CategoricalHyperparameter("input2", [0, 1]) hp3 = CategoricalHyperparameter("input3", [0, 1]) hp4 = Constant("And", "True") cond1 = EqualsCondition(hp4, hp1, 1) cond2 = EqualsCondition(hp4, hp2, 1) cond3 = EqualsCondition(hp4, hp3, 1) andconj1 = AndConjunction(cond1, cond2, cond3) cs = ConfigurationSpace() cs.add_hyperparameter(hp1) cs.add_hyperparameter(hp2) cs.add_hyperparameter(hp3) cs.add_hyperparameter(hp4) cs.add_condition(andconj1) self.assertNotIn(hp4, cs.get_all_unconditional_hyperparameters())
def test_check_forbidden_with_sampled_vector_configuration(self): cs = ConfigurationSpace() metric = CategoricalHyperparameter("metric", ["minkowski", "other"]) cs.add_hyperparameter(metric) forbidden = ForbiddenEqualsClause(metric, "other") cs.add_forbidden_clause(forbidden) configuration = Configuration(cs, vector=np.ones(1, dtype=float)) self.assertRaisesRegexp(ValueError, "violates forbidden clause", cs._check_forbidden, configuration.get_array())
def test_condition_without_added_hyperparameters(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) hp2 = UniformIntegerHyperparameter("child", 0, 10) cond = EqualsCondition(hp2, hp1, 0) self.assertRaisesRegexp( ValueError, "Child hyperparameter 'child' not " "in configuration space.", cs.add_condition, cond) cs.add_hyperparameter(hp1) self.assertRaisesRegexp( ValueError, "Child hyperparameter 'child' not " "in configuration space.", cs.add_condition, cond) # Test also the parent hyperparameter cs2 = ConfigurationSpace() cs2.add_hyperparameter(hp2) self.assertRaisesRegexp( ValueError, "Parent hyperparameter 'parent' " "not in configuration space.", cs2.add_condition, cond)
def test_add_second_condition_wo_conjunction(self): hp1 = CategoricalHyperparameter("input1", [0, 1]) hp2 = CategoricalHyperparameter("input2", [0, 1]) hp3 = Constant("And", "True") cond1 = EqualsCondition(hp3, hp1, 1) cond2 = EqualsCondition(hp3, hp2, 1) cs = ConfigurationSpace() cs.add_hyperparameter(hp1) cs.add_hyperparameter(hp2) cs.add_hyperparameter(hp3) cs.add_condition(cond1) self.assertRaisesRegexp( ValueError, "Adding a second condition \(different\) for a " "hyperparameter is ambigouos and " "therefore forbidden. Add a conjunction " "instead!", cs.add_condition, cond2)
def test_add_forbidden_clause(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("input1", [0, 1]) cs.add_hyperparameter(hp1) forb = ForbiddenEqualsClause(hp1, 1) # TODO add checking whether a forbidden clause makes sense at all cs.add_forbidden_clause(forb) # TODO add something to properly retrieve the forbidden clauses self.assertEqual( str(cs), "Configuration space object:\n " "Hyperparameters:\n input1, " "Type: Categorical, Choices: {0, 1}, " "Default: 0\n" " Forbidden Clauses:\n" " Forbidden: input1 == 1\n")
def fix_types(configuration: dict, configuration_space: ConfigurationSpace): ''' iterates over all hyperparameters in the ConfigSpaceNNI and fixes the types of the parameter values in configuration. Arguments --------- configuration: dict param name -> param value configuration_space: ConfigurationSpace Configuration space which knows the types for all parameter values Returns ------- configuration: dict with fixed types of parameter values ''' for param in configuration_space.get_hyperparameters(): param_name = param.name if configuration.get(param_name) is not None: if isinstance(param, (CategoricalHyperparameter)): # should be unnecessary, but to be on the safe param_name: configuration[param_name] = str(configuration[param_name]) elif isinstance(param, (OrdinalHyperparameter)): # should be unnecessary, but to be on the safe side: configuration[param_name] = str(configuration[param_name]) elif isinstance(param, Constant): # should be unnecessary, but to be on the safe side: configuration[param_name] = str(configuration[param_name]) elif isinstance(param, UniformFloatHyperparameter): configuration[param_name] = float(configuration[param_name]) elif isinstance(param, UniformIntegerHyperparameter): configuration[param_name] = int(configuration[param_name]) else: raise TypeError("Unknown hyperparameter type %s" % type(param)) return configuration
def _test_get_one_exchange_neighbourhood(self, hp): cs = ConfigurationSpace() num_neighbors = 0 if not isinstance(hp, list): hp = [hp] for hp_ in hp: cs.add_hyperparameter(hp_) if np.isinf(hp_.get_num_neighbors()): num_neighbors += 4 else: num_neighbors += hp_.get_num_neighbors() cs.seed(1) config = cs.get_default_configuration() all_neighbors = [] for i in range(100): neighborhood = get_one_exchange_neighbourhood(config, i) for new_config in neighborhood: self.assertNotEqual(config, new_config) all_neighbors.append(new_config) return all_neighbors
def test_setting_illegal_value(self): cs = ConfigurationSpace() cs.add_hyperparameter(UniformFloatHyperparameter('x', 0, 1)) configuration = {'x': 2} self.assertRaises(ValueError, Configuration, cs, configuration)
def test_setitem(self): ''' Checks overriding a sampled configuration ''' pcs = ConfigurationSpace() pcs.add_hyperparameter( UniformIntegerHyperparameter('x0', 1, 5, default_value=1)) x1 = pcs.add_hyperparameter( CategoricalHyperparameter('x1', ['ab', 'bc', 'cd', 'de'], default_value='ab')) # Condition x2 = pcs.add_hyperparameter(CategoricalHyperparameter('x2', [1, 2])) pcs.add_condition(EqualsCondition(x2, x1, 'ab')) # Forbidden x3 = pcs.add_hyperparameter(CategoricalHyperparameter('x3', [1, 2])) pcs.add_forbidden_clause(ForbiddenEqualsClause(x3, 2)) conf = pcs.get_default_configuration() # failed because it's a invalid configuration with self.assertRaisesRegex(ValueError, "Illegal value '0' for hyperparameter x0"): conf['x0'] = 0 # failed because the variable didn't exists with self.assertRaisesRegex( KeyError, "Hyperparameter 'x_0' does not exist in this configuration space." ): conf['x_0'] = 1 # failed because forbidden clause is violated with self.assertRaisesRegex( ForbiddenValueError, "Given vector violates forbidden clause Forbidden: x3 == 2"): conf['x3'] = 2 self.assertEqual(conf['x3'], 1) # successful operation 1 x0_old = conf['x0'] if x0_old == 1: conf['x0'] = 2 else: conf['x0'] = 1 x0_new = conf['x0'] self.assertNotEqual(x0_old, x0_new) pcs._check_configuration_rigorous(conf) self.assertEqual(conf['x2'], 1) # successful operation 2 x1_old = conf['x1'] if x1_old == 'ab': conf['x1'] = 'cd' else: conf['x1'] = 'ab' x1_new = conf['x1'] self.assertNotEqual(x1_old, x1_new) pcs._check_configuration_rigorous(conf) self.assertRaises(KeyError, conf.__getitem__, 'x2')
def test_add_non_hyperparameter(self): cs = ConfigurationSpace() non_hp = unittest.TestSuite() self.assertRaises(TypeError, cs.add_hyperparameter, non_hp)
def test_add_hyperparameter(self): cs = ConfigurationSpace() hp = UniformIntegerHyperparameter("name", 0, 10) cs.add_hyperparameter(hp)