def get_eips_object_callback( scenario_dict, seed, ta, ta_kwargs, backend, metalearning_configurations, ): scenario_dict['input_psmac_dirs'] = backend.get_smac_output_glob() scenario = Scenario(scenario_dict) types, bounds = get_types(scenario.cs, scenario.feature_array) model_kwargs = dict( target_names=['cost', 'runtime'], types=types, bounds=bounds, instance_features=scenario.feature_array, rf_kwargs={ 'seed': 1, }, ) return SMAC4AC( scenario=scenario, rng=seed, tae_runner=ta, tae_runner_kwargs=ta_kwargs, runhistory2epm=RunHistory2EPM4EIPS, runhistory2epm_kwargs={}, model=UncorrelatedMultiObjectiveRandomForestWithInstances, model_kwargs=model_kwargs, acquisition_function=EIPS, run_id=seed, )
def testRandomImputation(self): rs = numpy.random.RandomState(1) for i in range(0, 150, 15): # First random imputation sanity check num_samples = max(1, i * 10) num_feat = max(1, i) num_censored = int(num_samples * 0.1) X = rs.rand(num_samples, num_feat) y = numpy.sin(X[:, 0:1]) cutoff = max(y) * 0.9 y[y > cutoff] = cutoff # We have some cen data cen_X = X[:num_censored, :] cen_y = y[:num_censored] uncen_X = X[num_censored:, :] uncen_y = y[num_censored:] cen_y /= 2 cs = ConfigurationSpace() for i in range(num_feat): cs.add_hyperparameter( UniformFloatHyperparameter(name="a_%d" % i, lower=0, upper=1, default_value=0.5)) types, bounds = get_types(cs, None) self.model = RandomForestWithInstances( configspace=cs, types=types, bounds=bounds, instance_features=None, seed=1234567980, ) imputor = rfr_imputator.RFRImputator(rng=rs, cutoff=cutoff, threshold=cutoff * 10, change_threshold=0.01, max_iter=5, model=self.model) imp_y = imputor.impute(censored_X=cen_X, censored_y=cen_y, uncensored_X=uncen_X, uncensored_y=uncen_y) if imp_y is None: continue for idx in range(cen_y.shape[0]): self.assertGreater(imp_y[idx], cen_y[idx]) self.assertTrue(numpy.isfinite(imp_y).all())
def get_model(self, config_space): kernel = self.get_kernel(config_space) types, bounds = get_types(config_space, instance_features=None) return GaussianProcess( configspace=config_space, types=types, bounds=bounds, seed=self._get_random_state().randint(MAXINT), kernel=kernel, normalize_y=True, )
def setUp(self): logging.basicConfig(level=logging.DEBUG) self.cs = ConfigurationSpace() self.cs.add_hyperparameter( CategoricalHyperparameter(name="cat_a_b", choices=["a", "b"], default_value="a")) self.cs.add_hyperparameter( UniformFloatHyperparameter(name="float_0_1", lower=0, upper=1, default_value=0.5)) self.cs.add_hyperparameter( UniformIntegerHyperparameter(name='integer_0_100', lower=-10, upper=10, default_value=0)) self.rh = runhistory.RunHistory(aggregate_func=average_cost) rs = numpy.random.RandomState(1) to_count = 0 cn_count = 0 for i in range(500): config, seed, runtime, status, instance_id = \ generate_config(cs=self.cs, rs=rs) if runtime == 40: to_count += 1 if runtime < 40 and status == StatusType.TIMEOUT: cn_count += 1 self.rh.add(config=config, cost=runtime, time=runtime, status=status, instance_id=instance_id, seed=seed, additional_info=None) print("%d TIMEOUTs, %d censored" % (to_count, cn_count)) self.scen = Scen() self.scen.run_obj = "runtime" self.scen.overall_obj = "par10" self.scen.cutoff = 40 types, bounds = get_types(self.cs, None) self.model = RandomForestWithInstances( configspace=self.cs, types=types, bounds=bounds, instance_features=None, seed=1234567980, )
def test_with_ordinal(self): cs = smac.configspace.ConfigurationSpace() _ = cs.add_hyperparameter( CategoricalHyperparameter('a', [0, 1], default_value=0)) _ = cs.add_hyperparameter( OrdinalHyperparameter('b', [0, 1], default_value=1)) _ = cs.add_hyperparameter( UniformFloatHyperparameter('c', lower=0., upper=1., default_value=1)) _ = cs.add_hyperparameter( UniformIntegerHyperparameter('d', lower=0, upper=10, default_value=1)) cs.seed(1) feat_array = np.array([0, 0, 0]).reshape(1, -1) types, bounds = get_types(cs, feat_array) model = RandomForestWithInstances( configspace=cs, types=types, bounds=bounds, instance_features=feat_array, seed=1, ratio_features=1.0, pca_components=9, ) self.assertEqual(bounds[0][0], 2) self.assertTrue(bounds[0][1] is np.nan) self.assertEqual(bounds[1][0], 0) self.assertEqual(bounds[1][1], 1) self.assertEqual(bounds[2][0], 0.) self.assertEqual(bounds[2][1], 1.) self.assertEqual(bounds[3][0], 0.) self.assertEqual(bounds[3][1], 1.) X = np.array( [[0., 0., 0., 0., 0., 0., 0.], [0., 0., 1., 0., 0., 0., 0.], [0., 1., 0., 9., 0., 0., 0.], [0., 1., 1., 4., 0., 0., 0.]], dtype=np.float64) y = np.array([0, 1, 2, 3], dtype=np.float64) X_train = np.vstack((X, X, X, X, X, X, X, X, X, X)) y_train = np.vstack((y, y, y, y, y, y, y, y, y, y)) model.train(X_train, y_train.reshape((-1, 1))) mean, _ = model.predict(X) for idx, m in enumerate(mean): self.assertAlmostEqual(y[idx], m, 0.05)
def get_model(self, cs, instance_features=None): if instance_features: instance_features = numpy.array( [instance_features[key] for key in instance_features]) types, bounds = get_types(cs, instance_features) model = RandomForestWithInstances( configspace=cs, types=types, bounds=bounds, instance_features=instance_features, seed=1234567980, pca_components=7, ) return model
def test_apply_pca(self): cs = self._get_cs(5) instance_features = np.array([np.random.rand(10) for _ in range(5)]) types, bounds = get_types(cs, instance_features) def get_X_y(num_samples, num_instance_features): X = smac.configspace.convert_configurations_to_array( cs.sample_configuration(num_samples)) if num_instance_features: X_inst = np.random.rand(num_samples, num_instance_features) X = np.hstack((X, X_inst)) y = np.random.rand(num_samples) return X, y with unittest.mock.patch.object(AbstractEPM, '_train'): with unittest.mock.patch.object(AbstractEPM, '_predict') as predict_mock: predict_mock.side_effect = lambda x, _: (x, x) epm = AbstractEPM( configspace=cs, types=types, bounds=bounds, seed=1, pca_components=7, instance_features=instance_features, ) X, y = get_X_y(5, 10) epm.train(X, y) self.assertFalse(epm._apply_pca) X_test, _ = get_X_y(5, None) epm.predict_marginalized_over_instances(X_test) # more data points than pca components X, y = get_X_y(8, 10) epm.train(X, y) self.assertTrue(epm._apply_pca) X_test, _ = get_X_y(5, None) epm.predict_marginalized_over_instances(X_test) # and less again - this ensures that the types array inside the epm is reverted # and the pca is disabled again X, y = get_X_y(5, 10) epm.train(X, y) self.assertFalse(epm._apply_pca) X_test, _ = get_X_y(5, None) epm.predict_marginalized_over_instances(X_test)
def test_get_types(self): cs = ConfigurationSpace() cs.add_hyperparameter(CategoricalHyperparameter('a', ['a', 'b'])) cs.add_hyperparameter(UniformFloatHyperparameter('b', 1, 5)) cs.add_hyperparameter(UniformIntegerHyperparameter('c', 3, 7)) cs.add_hyperparameter(Constant('d', -5)) cs.add_hyperparameter(OrdinalHyperparameter('e', ['cold', 'hot'])) cs.add_hyperparameter(CategoricalHyperparameter('f', ['x', 'y'])) types, bounds = get_types(cs, None) np.testing.assert_array_equal(types, [2, 0, 0, 0, 0, 2]) self.assertEqual(bounds[0][0], 2) self.assertFalse(np.isfinite(bounds[0][1])) np.testing.assert_array_equal(bounds[1], [0, 1]) np.testing.assert_array_equal(bounds[2], [0, 1]) self.assertEqual(bounds[3][0], 0) self.assertFalse(np.isfinite(bounds[3][1])) np.testing.assert_array_equal(bounds[4], [0, 1]) self.assertEqual(bounds[5][0], 2) self.assertFalse(np.isfinite(bounds[5][1]))
def setUp(self): unittest.TestCase.setUp(self) self.rh = runhistory.RunHistory() self.cs = get_config_space() self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100}) self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0}) self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100}) self.config4 = Configuration(self.cs, values={'a': 23, 'b': 23}) self.config5 = Configuration(self.cs, values={'a': 5, 'b': 10}) self.scen = Scenario({'run_obj': 'runtime', 'cutoff_time': 20, 'cs': self.cs}) self.types, self.bounds = get_types(self.cs, None) self.scen = Scenario({'run_obj': 'runtime', 'cutoff_time': 20, 'cs': self.cs, 'output_dir': ''})
def test_init_EIPS_as_arguments(self): for objective in ['runtime', 'quality']: self.scenario.run_obj = objective types, bounds = get_types(self.scenario.cs, None) umrfwi = UncorrelatedMultiObjectiveRandomForestWithInstances( ['cost', 'runtime'], self.scenario.cs, types, bounds, seed=1, rf_kwargs={'seed': 1}, ) eips = EIPS(umrfwi) rh2EPM = RunHistory2EPM4EIPS(self.scenario, 2) epils = EPILS(self.scenario, model=umrfwi, acquisition_function=eips, runhistory2epm=rh2EPM).solver self.assertIs(umrfwi, epils.model) self.assertIs(eips, epils.acquisition_func) self.assertIs(rh2EPM, epils.rh2EPM)
def __init__(self, scenario: Scenario, rng: np.random.RandomState = None, method=Method.HYPERBOOST, pca_components=None, **kwargs): # Types and bounds required to initialize EPM types, bounds = get_types(scenario.cs, scenario.feature_array) if method == Method.SCIKIT_OPTIMIZE: # Initialize Scikit-optimize's empirical performance model model = SkoptEPM(types=types, bounds=bounds, instance_features=scenario.feature_array, seed=rng.randint(MAXINT), pca_components=scenario.PCA_DIM) # Pass parameters to SMAC4HPO super().__init__(scenario=scenario, rng=rng, model=model, **kwargs) elif method == Method.HYPERBOOST: # Initialize HyperBoost's empirical performance model model = HyperEPM(types=types, bounds=bounds, instance_features=scenario.feature_array, seed=rng.randint(MAXINT), pca_components_=pca_components, configspace=scenario.cs) # Pass parameters to SMAC4HPO super().__init__(scenario=scenario, rng=rng, model=model, acquisition_function=ScorePlusDistance, **kwargs)
def test_get_types_with_inactive(self): cs = ConfigurationSpace() a = cs.add_hyperparameter(CategoricalHyperparameter('a', ['a', 'b'])) b = cs.add_hyperparameter(UniformFloatHyperparameter('b', 1, 5)) c = cs.add_hyperparameter(UniformIntegerHyperparameter('c', 3, 7)) d = cs.add_hyperparameter(Constant('d', -5)) e = cs.add_hyperparameter(OrdinalHyperparameter('e', ['cold', 'hot'])) f = cs.add_hyperparameter(CategoricalHyperparameter('f', ['x', 'y'])) cs.add_condition(EqualsCondition(b, a, 'a')) cs.add_condition(EqualsCondition(c, a, 'a')) cs.add_condition(EqualsCondition(d, a, 'a')) cs.add_condition(EqualsCondition(e, a, 'a')) cs.add_condition(EqualsCondition(f, a, 'a')) types, bounds = get_types(cs, None) np.testing.assert_array_equal(types, [2, 0, 0, 2, 0, 3]) self.assertEqual(bounds[0][0], 2) self.assertFalse(np.isfinite(bounds[0][1])) np.testing.assert_array_equal(bounds[1], [-1, 1]) np.testing.assert_array_equal(bounds[2], [-1, 1]) self.assertEqual(bounds[3][0], 2) self.assertFalse(np.isfinite(bounds[3][1])) np.testing.assert_array_equal(bounds[4], [0, 2]) self.assertEqual(bounds[5][0], 3) self.assertFalse(np.isfinite(bounds[5][1]))
def run(self): """ Implementation of the forward selection loop. Uses SMACs EPM (RF) wrt the feature space to minimize the OOB error. Returns ------- feature_importance: OrderedDict dict_keys (first key -> most important) -> OOB error """ parameters = [p.name for p in self.scenario.cs.get_hyperparameters()] self.logger.debug("Parameters: %s", parameters) rh2epm = RunHistory2EPM4Cost(scenario=self.scenario, num_params=len(parameters), success_states=[ StatusType.SUCCESS, StatusType.CAPPED, StatusType.CRASHED ], impute_censored_data=False, impute_state=None) X, y = rh2epm.transform(self.rh) # reduce sample size to speedup computation if X.shape[0] > self.MAX_SAMPLES: idx = np.random.choice(X.shape[0], size=self.MAX_SAMPLES, replace=False) X = X[idx, :] y = y[idx] self.logger.debug( "Shape of X: %s, of y: %s, #parameters: %s, #feats: %s", X.shape, y.shape, len(parameters), len(self.scenario.feature_names)) names = copy.deepcopy(self.scenario.feature_names) self.logger.debug("Features: %s", names) used = list(range(0, len(parameters))) feat_ids = {f: i for i, f in enumerate(names, len(used))} ids_feat = {i: f for f, i in feat_ids.items()} self.logger.debug("Used: %s", used) evaluated_feature_importance = OrderedDict() types, bounds = get_types(self.scenario.cs, self.scenario.feature_array) last_error = np.inf for _round in range(self.to_evaluate): # Main Loop errors = [] for f in names: i = feat_ids[f] self.logger.debug('Evaluating %s', f) used.append(i) self.logger.debug( 'Used features: %s', str([ids_feat[j] for j in used[len(parameters):]])) start = time.time() self._refit_model(types[sorted(used)], bounds, X[:, sorted(used)], y) # refit the model every round errors.append(self.model.rf.out_of_bag_error()) used.pop() self.logger.debug('Refitted RF (sec %.2f; error: %.4f)' % (time.time() - start, errors[-1])) else: self.logger.debug('Evaluating None') start = time.time() self._refit_model(types[sorted(used)], bounds, X[:, sorted(used)], y) # refit the model every round errors.append(self.model.rf.out_of_bag_error()) self.logger.debug('Refitted RF (sec %.2f; error: %.4f)' % (time.time() - start, errors[-1])) if _round == 0: evaluated_feature_importance['None'] = errors[-1] best_idx = np.argmin(errors) lowest_error = errors[best_idx] if best_idx == len(errors) - 1: self.logger.info('Best thing to do is add nothing') best_feature = 'None' # evaluated_feature_importance[best_feature] = lowest_error break elif lowest_error >= last_error: break else: last_error = lowest_error best_feature = names.pop(best_idx) used.append(feat_ids[best_feature]) self.logger.debug('%s: %.4f' % (best_feature, lowest_error)) evaluated_feature_importance[best_feature] = lowest_error self.logger.debug(evaluated_feature_importance) self.evaluated_feature_importance = evaluated_feature_importance return evaluated_feature_importance
def __init__(self, api_config, config_space, parallel_setting="LS"): super(SMAC4EPMOpimizer, self).__init__(api_config) self.cs = config_space self.num_hps = len(self.cs.get_hyperparameters()) if parallel_setting not in ["CL_min", "CL_max", "CL_mean", "KB", "LS"]: raise ValueError( "parallel_setting can only be one of the following: " "CL_min, CL_max, CL_mean, KB, LS") self.parallel_setting = parallel_setting rng = np.random.RandomState(seed=0) scenario = Scenario({ "run_obj": "quality", # we optimize quality (alt. to runtime) "runcount-limit": 128, "cs": self.cs, # configuration space "deterministic": True, "limit_resources": False, }) self.stats = Stats(scenario) # traj = TrajLogger(output_dir=None, stats=self.stats) self.runhistory = RunHistory() r2e_def_kwargs = { "scenario": scenario, "num_params": self.num_hps, "success_states": [ StatusType.SUCCESS, ], "impute_censored_data": False, "scale_perc": 5, } self.random_chooser = ChooserProb(rng=rng, prob=0.0) types, bounds = get_types(self.cs, instance_features=None) model_kwargs = { "configspace": self.cs, "types": types, "bounds": bounds, "seed": rng.randint(MAXINT), } models = [] cov_amp = ConstantKernel( 2.0, constant_value_bounds=(np.exp(-10), np.exp(2)), prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng), ) cont_dims = np.array(np.where(np.array(types) == 0)[0], dtype=np.int) cat_dims = np.where(np.array(types) != 0)[0] if len(cont_dims) > 0: exp_kernel = Matern( np.ones([len(cont_dims)]), [(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cont_dims))], nu=2.5, operate_on=cont_dims, ) if len(cat_dims) > 0: ham_kernel = HammingKernel( np.ones([len(cat_dims)]), [(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cat_dims))], operate_on=cat_dims, ) assert len(cont_dims) + len(cat_dims) == len( scenario.cs.get_hyperparameters()) noise_kernel = WhiteKernel( noise_level=1e-8, noise_level_bounds=(np.exp(-25), np.exp(2)), prior=HorseshoePrior(scale=0.1, rng=rng), ) if len(cont_dims) > 0 and len(cat_dims) > 0: # both kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel elif len(cont_dims) > 0 and len(cat_dims) == 0: # only cont kernel = cov_amp * exp_kernel + noise_kernel elif len(cont_dims) == 0 and len(cat_dims) > 0: # only cont kernel = cov_amp * ham_kernel + noise_kernel else: raise ValueError() gp_kwargs = {"kernel": kernel} rf_kwargs = {} rf_kwargs["num_trees"] = model_kwargs.get("num_trees", 10) rf_kwargs["do_bootstrapping"] = model_kwargs.get( "do_bootstrapping", True) rf_kwargs["ratio_features"] = model_kwargs.get("ratio_features", 1.0) rf_kwargs["min_samples_split"] = model_kwargs.get( "min_samples_split", 2) rf_kwargs["min_samples_leaf"] = model_kwargs.get("min_samples_leaf", 1) rf_kwargs["log_y"] = model_kwargs.get("log_y", True) rf_log = RandomForestWithInstances(**model_kwargs, **rf_kwargs) rf_kwargs = copy.deepcopy(rf_kwargs) rf_kwargs["log_y"] = False rf_no_log = RandomForestWithInstances(**model_kwargs, **rf_kwargs) rh2epm_cost = RunHistory2EPM4Cost(**r2e_def_kwargs) rh2epm_log_cost = RunHistory2EPM4LogScaledCost(**r2e_def_kwargs) rh2epm_copula = RunHistory2EPM4GaussianCopulaCorrect(**r2e_def_kwargs) self.combinations = [] # 2 models * 4 acquisition functions acq_funcs = [EI, PI, LogEI, LCB] acq_func_instances = [] # acq_func_maximizer_instances = [] n_sls_iterations = { 1: 10, 2: 10, 3: 10, 4: 10, 5: 10, 6: 10, 7: 8, 8: 6, }.get(len(self.cs.get_hyperparameters()), 5) acq_func_maximizer_kwargs = { "config_space": self.cs, "rng": rng, "max_steps": 5, "n_steps_plateau_walk": 5, "n_sls_iterations": n_sls_iterations, } self.idx_ei = 0 self.num_models = len(models) self.num_acq_funcs = len(acq_funcs) no_transform_gp = GaussianProcess(**copy.deepcopy(model_kwargs), **copy.deepcopy(gp_kwargs)) ei = EI(model=no_transform_gp) acq_func_maximizer_kwargs["acquisition_function"] = ei ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs) self.combinations.append((no_transform_gp, ei, ei_opt, rh2epm_cost)) pi = PI(model=no_transform_gp) acq_func_maximizer_kwargs["acquisition_function"] = pi pi_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs) self.combinations.append((no_transform_gp, pi, pi_opt, rh2epm_cost)) lcb = LCB(model=no_transform_gp) acq_func_maximizer_kwargs["acquisition_function"] = lcb lcb_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs) self.combinations.append((no_transform_gp, lcb, lcb_opt, rh2epm_cost)) gp = GaussianProcess(**copy.deepcopy(model_kwargs), **copy.deepcopy(gp_kwargs)) ei = EI(model=gp) acq_func_maximizer_kwargs["acquisition_function"] = ei ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs) self.combinations.append((gp, ei, ei_opt, rh2epm_copula)) gp = GaussianProcess(**copy.deepcopy(model_kwargs), **copy.deepcopy(gp_kwargs)) ei = LogEI(model=gp) acq_func_maximizer_kwargs["acquisition_function"] = ei ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs) self.combinations.append((gp, ei, ei_opt, rh2epm_log_cost)) ei = EI(model=rf_no_log) acq_func_maximizer_kwargs["acquisition_function"] = ei ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs) self.combinations.append((rf_no_log, ei, ei_opt, rh2epm_cost)) ei = LogEI(model=rf_log) acq_func_maximizer_kwargs["acquisition_function"] = ei ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs) self.combinations.append((rf_log, ei, ei_opt, rh2epm_log_cost)) ei = EI(model=rf_no_log) acq_func_maximizer_kwargs["acquisition_function"] = ei ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs) self.combinations.append((rf_no_log, ei, ei_opt, rh2epm_copula)) self.num_acq_instances = len(acq_func_instances) self.best_observation = np.inf self.next_evaluations = []
def __init__(self, model_type: str = "gp_mcmc", **kwargs: typing.Any): scenario = kwargs["scenario"] if len(scenario.cs.get_hyperparameters()) <= 21201: kwargs["initial_design"] = kwargs.get("initial_design", SobolDesign) else: raise ValueError( 'The default initial design "Sobol sequence" can only handle up to 21201 dimensions. ' 'Please use a different initial design, such as "the Latin Hypercube design".', ) kwargs["runhistory2epm"] = kwargs.get("runhistory2epm", RunHistory2EPM4Cost) init_kwargs = kwargs.get("initial_design_kwargs", dict()) or dict() init_kwargs["n_configs_x_params"] = init_kwargs.get("n_configs_x_params", 8) init_kwargs["max_config_fracs"] = init_kwargs.get("max_config_fracs", 0.25) kwargs["initial_design_kwargs"] = init_kwargs if kwargs.get("model") is None: model_kwargs = kwargs.get("model_kwargs", dict()) or dict() _, rng = get_rng( rng=kwargs.get("rng", None), run_id=kwargs.get("run_id", None), logger=None, ) types, bounds = get_types(kwargs["scenario"].cs, instance_features=None) cov_amp = ConstantKernel( 2.0, constant_value_bounds=(np.exp(-10), np.exp(2)), prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng), ) cont_dims = np.where(np.array(types) == 0)[0] cat_dims = np.where(np.array(types) != 0)[0] if len(cont_dims) > 0: exp_kernel = Matern( np.ones([len(cont_dims)]), [ (np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cont_dims)) ], nu=2.5, operate_on=cont_dims, ) if len(cat_dims) > 0: ham_kernel = HammingKernel( np.ones([len(cat_dims)]), [ (np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cat_dims)) ], operate_on=cat_dims, ) assert (len(cont_dims) + len(cat_dims)) == len( scenario.cs.get_hyperparameters() ) noise_kernel = WhiteKernel( noise_level=1e-8, noise_level_bounds=(np.exp(-25), np.exp(2)), prior=HorseshoePrior(scale=0.1, rng=rng), ) if len(cont_dims) > 0 and len(cat_dims) > 0: # both kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel elif len(cont_dims) > 0 and len(cat_dims) == 0: # only cont kernel = cov_amp * exp_kernel + noise_kernel elif len(cont_dims) == 0 and len(cat_dims) > 0: # only cont kernel = cov_amp * ham_kernel + noise_kernel else: raise ValueError() if model_type == "gp": model_class = GaussianProcess # type: typing.Type[BaseModel] kwargs["model"] = model_class model_kwargs["kernel"] = kernel model_kwargs["normalize_y"] = True model_kwargs["seed"] = rng.randint(0, 2**20) elif model_type == "gp_mcmc": model_class = GaussianProcessMCMC kwargs["model"] = model_class kwargs["integrate_acquisition_function"] = True model_kwargs["kernel"] = kernel n_mcmc_walkers = 3 * len(kernel.theta) if n_mcmc_walkers % 2 == 1: n_mcmc_walkers += 1 model_kwargs["n_mcmc_walkers"] = n_mcmc_walkers model_kwargs["chain_length"] = 250 model_kwargs["burnin_steps"] = 250 model_kwargs["normalize_y"] = True model_kwargs["seed"] = rng.randint(0, 2**20) else: raise ValueError("Unknown model type %s" % model_type) kwargs["model_kwargs"] = model_kwargs if kwargs.get("random_configuration_chooser") is None: random_config_chooser_kwargs = ( kwargs.get( "random_configuration_chooser_kwargs", dict(), ) or dict() ) random_config_chooser_kwargs["prob"] = random_config_chooser_kwargs.get( "prob", 0.08447232371720552 ) kwargs["random_configuration_chooser_kwargs"] = random_config_chooser_kwargs if kwargs.get("acquisition_function_optimizer") is None: acquisition_function_optimizer_kwargs = ( kwargs.get( "acquisition_function_optimizer_kwargs", dict(), ) or dict() ) acquisition_function_optimizer_kwargs["n_sls_iterations"] = 10 kwargs[ "acquisition_function_optimizer_kwargs" ] = acquisition_function_optimizer_kwargs # only 1 configuration per SMBO iteration intensifier_kwargs = kwargs.get("intensifier_kwargs", dict()) or dict() intensifier_kwargs["min_chall"] = 1 kwargs["intensifier_kwargs"] = intensifier_kwargs scenario.intensification_percentage = 1e-10 super().__init__(**kwargs) if self.solver.scenario.n_features > 0: raise NotImplementedError("BOGP cannot handle instances") self.logger.info(self.__class__) self.solver.scenario.acq_opt_challengers = 1000 # type: ignore[attr-defined] # noqa F821 # activate predict incumbent self.solver.epm_chooser.predict_x_best = True
def convert_data_for_epm(scenario: Scenario, runhistory: RunHistory, impute_inactive_parameters=False, rng=None, logger=None): """ converts data from runhistory into EPM format Parameters ---------- scenario: Scenario smac.scenario.scenario.Scenario Object runhistory: RunHistory smac.runhistory.runhistory.RunHistory Object with all necessary data impute_inactive_parameters: bool whether to impute all inactive parameters in all configurations - this is needed for random forests, as they do not accept nan-values Returns ------- X: np.array X matrix with configuartion x features for all observed samples y: np.array y matrix with all observations types: np.array types of X cols -- necessary to train our RF implementation """ if rng is None: rng = np.random.RandomState(42) if impute_inactive_parameters: runhistory = force_finite_runhistory(runhistory) types, bounds = get_types(scenario.cs, scenario.feature_array) if logger is not None: logger.debug("Types: " + str(types) + ", Bounds: " + str(bounds)) model = RandomForestWithInstances(scenario.cs, types, bounds, rng.randint(MAXINT)) params = scenario.cs.get_hyperparameters() num_params = len(params) run_obj = scenario.run_obj if run_obj == "runtime": # if we log the performance data, # the RFRImputator will already get # log transform data from the runhistory cutoff = np.log10(scenario.cutoff) threshold = np.log10(scenario.cutoff * scenario.par_factor) imputor = RFRImputator(rng=rng, cutoff=cutoff, threshold=threshold, model=model, change_threshold=0.01, max_iter=10) # TODO: Adapt runhistory2EPM object based on scenario rh2EPM = RunHistory2EPM4LogCost(scenario=scenario, num_params=num_params, success_states=[ StatusType.SUCCESS, ], impute_censored_data=True, impute_state=[ StatusType.TIMEOUT, ], imputor=imputor) X, Y = rh2EPM.transform(runhistory) else: rh2EPM = RunHistory2EPM4Cost(scenario=scenario, num_params=num_params, success_states=[ StatusType.SUCCESS, ], impute_censored_data=False, impute_state=None) X, Y = rh2EPM.transform(runhistory) return X, Y, types
def validate_epm( self, config_mode: Union[str, typing.List[Configuration]] = 'def', instance_mode: Union[str, typing.List[str]] = 'test', repetitions: int = 1, runhistory: typing.Optional[RunHistory] = None, output_fn: typing.Optional[str] = None, reuse_epm: bool = True, ) -> RunHistory: """ Use EPM to predict costs/runtimes for unknown config/inst-pairs. side effect: if output is specified, saves runhistory to specified output directory. Parameters ---------- output_fn: str path to runhistory to be saved. if the suffix is not '.json', will be interpreted as directory and filename will be 'validated_runhistory_EPM.json' config_mode: str or list<Configuration> string or directly a list of Configuration, string from [def, inc, def+inc, wallclock_time, cpu_time, all]. time evaluates at cpu- or wallclock-timesteps of: [max_time/2^0, max_time/2^1, max_time/2^3, ..., default] with max_time being the highest recorded time instance_mode: str or list<str> what instances to use for validation, either from [train, test, train+test] or directly a list of instances repetitions: int number of repetitions in nondeterministic algorithms runhistory: RunHistory optional, RunHistory-object to reuse runs reuse_epm: bool if true (and if `self.epm`), reuse epm to validate runs Returns ------- runhistory: RunHistory runhistory with predicted runs """ if not isinstance(runhistory, RunHistory) and (self.epm is None or not reuse_epm): raise ValueError( "No runhistory specified for validating with EPM!") elif not reuse_epm or self.epm is None: # Create RandomForest types, bounds = get_types( self.scen.cs, self.scen.feature_array ) # type: ignore[attr-defined] # noqa F821 epm = RandomForestWithInstances( configspace=self.scen. cs, # type: ignore[attr-defined] # noqa F821 types=types, bounds=bounds, instance_features=self.scen.feature_array, seed=self.rng.randint(MAXINT), ratio_features=1.0, ) # Use imputor if objective is runtime imputor = None impute_state = None impute_censored_data = False if self.scen.run_obj == 'runtime': threshold = self.scen.cutoff * self.scen.par_factor # type: ignore[attr-defined] # noqa F821 imputor = RFRImputator( rng=self.rng, cutoff=self.scen. cutoff, # type: ignore[attr-defined] # noqa F821 threshold=threshold, model=epm) impute_censored_data = True impute_state = [StatusType.CAPPED] success_states = [ StatusType.SUCCESS, ] else: success_states = [ StatusType.SUCCESS, StatusType.CRASHED, StatusType.MEMOUT ] # Transform training data (from given rh) rh2epm = RunHistory2EPM4Cost( num_params=len(self.scen.cs.get_hyperparameters() ), # type: ignore[attr-defined] # noqa F821 scenario=self.scen, rng=self.rng, impute_censored_data=impute_censored_data, imputor=imputor, impute_state=impute_state, success_states=success_states) assert runhistory is not None # please mypy X, y = rh2epm.transform(runhistory) self.logger.debug("Training model with data of shape X: %s, y:%s", str(X.shape), str(y.shape)) # Train random forest epm.train(X, y) else: epm = typing.cast(RandomForestWithInstances, self.epm) # Predict desired runs runs, rh_epm = self._get_runs(config_mode, instance_mode, repetitions, runhistory) feature_array_size = len(self.scen.cs.get_hyperparameters() ) # type: ignore[attr-defined] # noqa F821 if self.scen.feature_array is not None: feature_array_size += self.scen.feature_array.shape[1] X_pred = np.empty((len(runs), feature_array_size)) for idx, run in enumerate(runs): if self.scen.feature_array is not None and run.inst is not None: X_pred[idx] = np.hstack([ convert_configurations_to_array([run.config])[0], self.scen.feature_dict[run.inst] ]) else: X_pred[idx] = convert_configurations_to_array([run.config])[0] self.logger.debug("Predicting desired %d runs, data has shape %s", len(runs), str(X_pred.shape)) y_pred = epm.predict(X_pred) self.epm = epm # Add runs to runhistory for run, pred in zip(runs, y_pred[0]): rh_epm.add( config=run.config, cost=float(pred), time=float(pred), status=StatusType.SUCCESS, instance_id=run.inst, seed=-1, additional_info={"additional_info": "ESTIMATED USING EPM!"}) if output_fn: self._save_results(rh_epm, output_fn, backup_fn="validated_runhistory_EPM.json") return rh_epm
def __init__( self, scenario: Scenario, tae_runner: Optional[Union[Type[BaseRunner], Callable]] = None, tae_runner_kwargs: Optional[Dict] = None, runhistory: Optional[Union[Type[RunHistory], RunHistory]] = None, runhistory_kwargs: Optional[Dict] = None, intensifier: Optional[Type[AbstractRacer]] = None, intensifier_kwargs: Optional[Dict] = None, acquisition_function: Optional[ Type[AbstractAcquisitionFunction]] = None, acquisition_function_kwargs: Optional[Dict] = None, integrate_acquisition_function: bool = False, acquisition_function_optimizer: Optional[ Type[AcquisitionFunctionMaximizer]] = None, acquisition_function_optimizer_kwargs: Optional[Dict] = None, model: Optional[Type[AbstractEPM]] = None, model_kwargs: Optional[Dict] = None, runhistory2epm: Optional[Type[AbstractRunHistory2EPM]] = None, runhistory2epm_kwargs: Optional[Dict] = None, initial_design: Optional[Type[InitialDesign]] = None, initial_design_kwargs: Optional[Dict] = None, initial_configurations: Optional[List[Configuration]] = None, stats: Optional[Stats] = None, restore_incumbent: Optional[Configuration] = None, rng: Optional[Union[np.random.RandomState, int]] = None, smbo_class: Optional[Type[SMBO]] = None, run_id: Optional[int] = None, random_configuration_chooser: Optional[ Type[RandomConfigurationChooser]] = None, random_configuration_chooser_kwargs: Optional[Dict] = None, dask_client: Optional[dask.distributed.Client] = None, n_jobs: Optional[int] = 1, ): """ Constructor Parameters ---------- scenario : ~smac.scenario.scenario.Scenario Scenario object tae_runner : ~smac.tae.base.BaseRunner or callable Callable or implementation of :class:`~smac.tae.base.BaseRunner`. In case a callable is passed it will be wrapped by :class:`~smac.tae.execute_func.ExecuteTAFuncDict`. If not set, it will be initialized with the :class:`~smac.tae.execute_ta_run_old.ExecuteTARunOld`. tae_runner_kwargs: Optional[Dict] arguments passed to constructor of '~tae_runner' runhistory : RunHistory runhistory to store all algorithm runs runhistory_kwargs : Optional[Dict] arguments passed to constructor of runhistory. We strongly advise against changing the aggregation function, since it will break some code assumptions intensifier : Intensifier intensification object to issue a racing to decide the current incumbent intensifier_kwargs: Optional[Dict] arguments passed to the constructor of '~intensifier' acquisition_function : ~smac.optimizer.acquisition.AbstractAcquisitionFunction Class or object that implements the :class:`~smac.optimizer.acquisition.AbstractAcquisitionFunction`. Will use :class:`~smac.optimizer.acquisition.EI` or :class:`~smac.optimizer.acquisition.LogEI` if not set. `~acquisition_function_kwargs` is passed to the class constructor. acquisition_function_kwargs : Optional[Dict] dictionary to pass specific arguments to ~acquisition_function integrate_acquisition_function : bool, default=False Whether to integrate the acquisition function. Works only with models which can sample their hyperparameters (i.e. GaussianProcessMCMC). acquisition_function_optimizer : ~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer Object that implements the :class:`~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer`. Will use :class:`smac.optimizer.ei_optimization.LocalAndSortedRandomSearch` if not set. acquisition_function_optimizer_kwargs: Optional[Dict] Arguments passed to constructor of '~acquisition_function_optimizer' model : AbstractEPM Model that implements train() and predict(). Will use a :class:`~smac.epm.rf_with_instances.RandomForestWithInstances` if not set. model_kwargs : Optional[Dict] Arguments passed to constructor of '~model' runhistory2epm : ~smac.runhistory.runhistory2epm.RunHistory2EMP Object that implements the AbstractRunHistory2EPM. If None, will use :class:`~smac.runhistory.runhistory2epm.RunHistory2EPM4Cost` if objective is cost or :class:`~smac.runhistory.runhistory2epm.RunHistory2EPM4LogCost` if objective is runtime. runhistory2epm_kwargs: Optional[Dict] Arguments passed to the constructor of '~runhistory2epm' initial_design : InitialDesign initial sampling design initial_design_kwargs: Optional[Dict] arguments passed to constructor of `~initial_design' initial_configurations : List[Configuration] list of initial configurations for initial design -- cannot be used together with initial_design stats : Stats optional stats object rng : np.random.RandomState Random number generator restore_incumbent : Configuration incumbent used if restoring to previous state smbo_class : ~smac.optimizer.smbo.SMBO Class implementing the SMBO interface which will be used to instantiate the optimizer class. run_id : int (optional) Run ID will be used as subfolder for output_dir. If no ``run_id`` is given, a random ``run_id`` will be chosen. random_configuration_chooser : ~smac.optimizer.random_configuration_chooser.RandomConfigurationChooser How often to choose a random configuration during the intensification procedure. random_configuration_chooser_kwargs : Optional[Dict] arguments of constructor for '~random_configuration_chooser' dask_client : dask.distributed.Client User-created dask client, can be used to start a dask cluster and then attach SMAC to it. n_jobs : int, optional Number of jobs. If > 1 or -1, this creates a dask client if ``dask_client`` is ``None``. Will be ignored if ``dask_client`` is not ``None``. If ``None``, this value will be set to 1, if ``-1``, this will be set to the number of cpu cores. """ self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) self.scenario = scenario self.output_dir = "" if not restore_incumbent: # restore_incumbent is used by the CLI interface which provides a method for restoring a SMAC run given an # output directory. This is the default path. # initial random number generator run_id, rng = get_rng(rng=rng, run_id=run_id, logger=self.logger) self.output_dir = create_output_directory(scenario, run_id) elif scenario.output_dir is not None: # type: ignore[attr-defined] # noqa F821 run_id, rng = get_rng(rng=rng, run_id=run_id, logger=self.logger) # output-directory is created in CLI when restoring from a # folder. calling the function again in the facade results in two # folders being created: run_X and run_X.OLD. if we are # restoring, the output-folder exists already and we omit creating it, # but set the self-output_dir to the dir. # necessary because we want to write traj to new output-dir in CLI. self.output_dir = cast(str, scenario.output_dir_for_this_run ) # type: ignore[attr-defined] # noqa F821 rng = cast(np.random.RandomState, rng) if (scenario.deterministic is True # type: ignore[attr-defined] # noqa F821 and getattr(scenario, 'tuner_timeout', None) is None and scenario.run_obj == 'quality' # type: ignore[attr-defined] # noqa F821 ): self.logger.info( 'Optimizing a deterministic scenario for quality without a tuner timeout - will make ' 'SMAC deterministic and only evaluate one configuration per iteration!' ) scenario.intensification_percentage = 1e-10 # type: ignore[attr-defined] # noqa F821 scenario.min_chall = 1 # type: ignore[attr-defined] # noqa F821 scenario.write() # initialize stats object if stats: self.stats = stats else: self.stats = Stats(scenario) if self.scenario.run_obj == "runtime" and not self.scenario.transform_y == "LOG": # type: ignore[attr-defined] # noqa F821 self.logger.warning( "Runtime as objective automatically activates log(y) transformation" ) self.scenario.transform_y = "LOG" # type: ignore[attr-defined] # noqa F821 # initialize empty runhistory runhistory_def_kwargs = {} if runhistory_kwargs is not None: runhistory_def_kwargs.update(runhistory_kwargs) if runhistory is None: runhistory = RunHistory(**runhistory_def_kwargs) elif inspect.isclass(runhistory): runhistory = runhistory( **runhistory_def_kwargs) # type: ignore[operator] # noqa F821 elif isinstance(runhistory, RunHistory): pass else: raise ValueError( 'runhistory has to be a class or an object of RunHistory') rand_conf_chooser_kwargs = {'rng': rng} if random_configuration_chooser_kwargs is not None: rand_conf_chooser_kwargs.update( random_configuration_chooser_kwargs) if random_configuration_chooser is None: if 'prob' not in rand_conf_chooser_kwargs: rand_conf_chooser_kwargs[ 'prob'] = scenario.rand_prob # type: ignore[attr-defined] # noqa F821 random_configuration_chooser_instance = ( ChooserProb(**rand_conf_chooser_kwargs ) # type: ignore[arg-type] # noqa F821 ) # type: RandomConfigurationChooser elif inspect.isclass(random_configuration_chooser): random_configuration_chooser_instance = random_configuration_chooser( ** rand_conf_chooser_kwargs) # type: ignore[arg-type] # noqa F821 elif not isinstance(random_configuration_chooser, RandomConfigurationChooser): raise ValueError( "random_configuration_chooser has to be" " a class or object of RandomConfigurationChooser") # reset random number generator in config space to draw different # random configurations with each seed given to SMAC scenario.cs.seed( rng.randint(MAXINT)) # type: ignore[attr-defined] # noqa F821 # initial Trajectory Logger traj_logger = TrajLogger(output_dir=self.output_dir, stats=self.stats) # initial EPM types, bounds = get_types( scenario.cs, scenario.feature_array) # type: ignore[attr-defined] # noqa F821 model_def_kwargs = { 'types': types, 'bounds': bounds, 'instance_features': scenario.feature_array, 'seed': rng.randint(MAXINT), 'pca_components': scenario.PCA_DIM, } if model_kwargs is not None: model_def_kwargs.update(model_kwargs) if model is None: for key, value in { 'log_y': scenario.transform_y in ["LOG", "LOGS"], # type: ignore[attr-defined] # noqa F821 'num_trees': scenario. rf_num_trees, # type: ignore[attr-defined] # noqa F821 'do_bootstrapping': scenario. rf_do_bootstrapping, # type: ignore[attr-defined] # noqa F821 'ratio_features': scenario. rf_ratio_features, # type: ignore[attr-defined] # noqa F821 'min_samples_split': scenario. rf_min_samples_split, # type: ignore[attr-defined] # noqa F821 'min_samples_leaf': scenario. rf_min_samples_leaf, # type: ignore[attr-defined] # noqa F821 'max_depth': scenario. rf_max_depth, # type: ignore[attr-defined] # noqa F821 }.items(): if key not in model_def_kwargs: model_def_kwargs[key] = value model_def_kwargs[ 'configspace'] = self.scenario.cs # type: ignore[attr-defined] # noqa F821 model_instance = ( RandomForestWithInstances( **model_def_kwargs) # type: ignore[arg-type] # noqa F821 ) # type: AbstractEPM elif inspect.isclass(model): model_def_kwargs[ 'configspace'] = self.scenario.cs # type: ignore[attr-defined] # noqa F821 model_instance = model( **model_def_kwargs) # type: ignore[arg-type] # noqa F821 else: raise TypeError("Model not recognized: %s" % (type(model))) # initial acquisition function acq_def_kwargs = {'model': model_instance} if acquisition_function_kwargs is not None: acq_def_kwargs.update(acquisition_function_kwargs) if acquisition_function is None: if scenario.transform_y in [ "LOG", "LOGS" ]: # type: ignore[attr-defined] # noqa F821 acquisition_function_instance = ( LogEI(** acq_def_kwargs) # type: ignore[arg-type] # noqa F821 ) # type: AbstractAcquisitionFunction else: acquisition_function_instance = EI( **acq_def_kwargs) # type: ignore[arg-type] # noqa F821 elif inspect.isclass(acquisition_function): acquisition_function_instance = acquisition_function( **acq_def_kwargs) else: raise TypeError( "Argument acquisition_function must be None or an object implementing the " "AbstractAcquisitionFunction, not %s." % type(acquisition_function)) if integrate_acquisition_function: acquisition_function_instance = IntegratedAcquisitionFunction( acquisition_function=acquisition_function_instance, **acq_def_kwargs) # initialize optimizer on acquisition function acq_func_opt_kwargs = { 'acquisition_function': acquisition_function_instance, 'config_space': scenario.cs, # type: ignore[attr-defined] # noqa F821 'rng': rng, } if acquisition_function_optimizer_kwargs is not None: acq_func_opt_kwargs.update(acquisition_function_optimizer_kwargs) if acquisition_function_optimizer is None: for key, value in { 'max_steps': scenario. sls_max_steps, # type: ignore[attr-defined] # noqa F821 'n_steps_plateau_walk': scenario. sls_n_steps_plateau_walk, # type: ignore[attr-defined] # noqa F821 }.items(): if key not in acq_func_opt_kwargs: acq_func_opt_kwargs[key] = value acquisition_function_optimizer_instance = ( LocalAndSortedRandomSearch( ** acq_func_opt_kwargs) # type: ignore[arg-type] # noqa F821 ) # type: AcquisitionFunctionMaximizer elif inspect.isclass(acquisition_function_optimizer): acquisition_function_optimizer_instance = acquisition_function_optimizer( **acq_func_opt_kwargs) # type: ignore[arg-type] # noqa F821 else: raise TypeError( "Argument acquisition_function_optimizer must be None or an object implementing the " "AcquisitionFunctionMaximizer, but is '%s'" % type(acquisition_function_optimizer)) # initialize tae_runner # First case, if tae_runner is None, the target algorithm is a call # string in the scenario file tae_def_kwargs = { 'stats': self.stats, 'run_obj': scenario.run_obj, 'par_factor': scenario.par_factor, # type: ignore[attr-defined] # noqa F821 'cost_for_crash': scenario.cost_for_crash, # type: ignore[attr-defined] # noqa F821 'abort_on_first_run_crash': scenario. abort_on_first_run_crash # type: ignore[attr-defined] # noqa F821 } if tae_runner_kwargs is not None: tae_def_kwargs.update(tae_runner_kwargs) if 'ta' not in tae_def_kwargs: tae_def_kwargs[ 'ta'] = scenario.ta # type: ignore[attr-defined] # noqa F821 if tae_runner is None: tae_def_kwargs[ 'ta'] = scenario.ta # type: ignore[attr-defined] # noqa F821 tae_runner_instance = ( ExecuteTARunOld( **tae_def_kwargs) # type: ignore[arg-type] # noqa F821 ) # type: BaseRunner elif inspect.isclass(tae_runner): tae_runner_instance = cast(BaseRunner, tae_runner( **tae_def_kwargs)) # type: ignore[arg-type] # noqa F821 elif callable(tae_runner): tae_def_kwargs['ta'] = tae_runner tae_def_kwargs[ 'use_pynisher'] = scenario.limit_resources # type: ignore[attr-defined] # noqa F821 tae_runner_instance = ExecuteTAFuncDict( **tae_def_kwargs) # type: ignore[arg-type] # noqa F821 else: raise TypeError( "Argument 'tae_runner' is %s, but must be " "either None, a callable or an object implementing " "BaseRunner. Passing 'None' will result in the " "creation of target algorithm runner based on the " "call string in the scenario file." % type(tae_runner)) # In case of a parallel run, wrap the single worker in a parallel # runner if n_jobs is None or n_jobs == 1: _n_jobs = 1 elif n_jobs == -1: _n_jobs = joblib.cpu_count() elif n_jobs > 0: _n_jobs = n_jobs else: raise ValueError( 'Number of tasks must be positive, None or -1, but is %s' % str(n_jobs)) if _n_jobs > 1 or dask_client is not None: tae_runner_instance = DaskParallelRunner( tae_runner_instance, n_workers=_n_jobs, output_directory=self.output_dir, dask_client=dask_client, ) # Check that overall objective and tae objective are the same # TODO: remove these two ignores once the scenario object knows all its attributes! if tae_runner_instance.run_obj != scenario.run_obj: # type: ignore[union-attr] # noqa F821 raise ValueError( "Objective for the target algorithm runner and " "the scenario must be the same, but are '%s' and " "'%s'" % (tae_runner_instance.run_obj, scenario.run_obj)) # type: ignore[union-attr] # noqa F821 # initialize intensification intensifier_def_kwargs = { 'stats': self.stats, 'traj_logger': traj_logger, 'rng': rng, 'instances': scenario.train_insts, # type: ignore[attr-defined] # noqa F821 'cutoff': scenario.cutoff, # type: ignore[attr-defined] # noqa F821 'deterministic': scenario.deterministic, # type: ignore[attr-defined] # noqa F821 'run_obj_time': scenario.run_obj == "runtime", # type: ignore[attr-defined] # noqa F821 'instance_specifics': scenario. instance_specific, # type: ignore[attr-defined] # noqa F821 'adaptive_capping_slackfactor': scenario. intens_adaptive_capping_slackfactor, # type: ignore[attr-defined] # noqa F821 'min_chall': scenario.intens_min_chall # type: ignore[attr-defined] # noqa F821 } if isinstance(intensifier, Intensifier) \ or (intensifier is not None and inspect.isclass(intensifier) and issubclass(intensifier, Intensifier)): intensifier_def_kwargs[ 'always_race_against'] = scenario.cs.get_default_configuration( ) # type: ignore[attr-defined] # noqa F821 intensifier_def_kwargs[ 'use_ta_time_bound'] = scenario.use_ta_time # type: ignore[attr-defined] # noqa F821 intensifier_def_kwargs[ 'minR'] = scenario.minR # type: ignore[attr-defined] # noqa F821 intensifier_def_kwargs[ 'maxR'] = scenario.maxR # type: ignore[attr-defined] # noqa F821 if intensifier_kwargs is not None: intensifier_def_kwargs.update(intensifier_kwargs) if intensifier is None: intensifier_instance = ( Intensifier(**intensifier_def_kwargs ) # type: ignore[arg-type] # noqa F821 ) # type: AbstractRacer elif inspect.isclass(intensifier): intensifier_instance = intensifier( **intensifier_def_kwargs) # type: ignore[arg-type] # noqa F821 else: raise TypeError( "Argument intensifier must be None or an object implementing the AbstractRacer, but is '%s'" % type(intensifier)) # initial design if initial_design is not None and initial_configurations is not None: raise ValueError( "Either use initial_design or initial_configurations; but not both" ) init_design_def_kwargs = { 'cs': scenario.cs, # type: ignore[attr-defined] # noqa F821 'traj_logger': traj_logger, 'rng': rng, 'ta_run_limit': scenario.ta_run_limit, # type: ignore[attr-defined] # noqa F821 'configs': initial_configurations, 'n_configs_x_params': 0, 'max_config_fracs': 0.0 } if initial_design_kwargs is not None: init_design_def_kwargs.update(initial_design_kwargs) if initial_configurations is not None: initial_design_instance = InitialDesign(**init_design_def_kwargs) elif initial_design is None: if scenario.initial_incumbent == "DEFAULT": # type: ignore[attr-defined] # noqa F821 init_design_def_kwargs['max_config_fracs'] = 0.0 initial_design_instance = DefaultConfiguration( **init_design_def_kwargs) elif scenario.initial_incumbent == "RANDOM": # type: ignore[attr-defined] # noqa F821 init_design_def_kwargs['max_config_fracs'] = 0.0 initial_design_instance = RandomConfigurations( **init_design_def_kwargs) elif scenario.initial_incumbent == "LHD": # type: ignore[attr-defined] # noqa F821 initial_design_instance = LHDesign(**init_design_def_kwargs) elif scenario.initial_incumbent == "FACTORIAL": # type: ignore[attr-defined] # noqa F821 initial_design_instance = FactorialInitialDesign( **init_design_def_kwargs) elif scenario.initial_incumbent == "SOBOL": # type: ignore[attr-defined] # noqa F821 initial_design_instance = SobolDesign(**init_design_def_kwargs) else: raise ValueError("Don't know what kind of initial_incumbent " "'%s' is" % scenario.initial_incumbent ) # type: ignore[attr-defined] # noqa F821 elif inspect.isclass(initial_design): initial_design_instance = initial_design(**init_design_def_kwargs) else: raise TypeError( "Argument initial_design must be None or an object implementing the InitialDesign, but is '%s'" % type(initial_design)) # if we log the performance data, # the RFRImputator will already get # log transform data from the runhistory if scenario.transform_y in [ "LOG", "LOGS" ]: # type: ignore[attr-defined] # noqa F821 cutoff = np.log(np.nanmin([ np.inf, np.float_(scenario.cutoff) ])) # type: ignore[attr-defined] # noqa F821 threshold = cutoff + np.log( scenario.par_factor) # type: ignore[attr-defined] # noqa F821 else: cutoff = np.nanmin([np.inf, np.float_(scenario.cutoff) ]) # type: ignore[attr-defined] # noqa F821 threshold = cutoff * scenario.par_factor # type: ignore[attr-defined] # noqa F821 num_params = len(scenario.cs.get_hyperparameters() ) # type: ignore[attr-defined] # noqa F821 imputor = RFRImputator(rng=rng, cutoff=cutoff, threshold=threshold, model=model_instance, change_threshold=0.01, max_iter=2) r2e_def_kwargs = { 'scenario': scenario, 'num_params': num_params, 'success_states': [ StatusType.SUCCESS, ], 'impute_censored_data': True, 'impute_state': [ StatusType.CAPPED, ], 'imputor': imputor, 'scale_perc': 5 } if scenario.run_obj == 'quality': r2e_def_kwargs.update({ 'success_states': [StatusType.SUCCESS, StatusType.CRASHED, StatusType.MEMOUT], 'impute_censored_data': False, 'impute_state': None, }) if isinstance( intensifier_instance, (SuccessiveHalving, Hyperband)) and scenario.run_obj == "quality": r2e_def_kwargs.update({ 'success_states': [ StatusType.SUCCESS, StatusType.CRASHED, StatusType.MEMOUT, StatusType.DONOTADVANCE, ], 'consider_for_higher_budgets_state': [ StatusType.DONOTADVANCE, StatusType.TIMEOUT, StatusType.CRASHED, StatusType.MEMOUT, ], }) if runhistory2epm_kwargs is not None: r2e_def_kwargs.update(runhistory2epm_kwargs) if runhistory2epm is None: if scenario.run_obj == 'runtime': rh2epm = ( RunHistory2EPM4LogCost( **r2e_def_kwargs) # type: ignore[arg-type] # noqa F821 ) # type: AbstractRunHistory2EPM elif scenario.run_obj == 'quality': if scenario.transform_y == "NONE": # type: ignore[attr-defined] # noqa F821 rh2epm = RunHistory2EPM4Cost( **r2e_def_kwargs) # type: ignore[arg-type] # noqa F821 elif scenario.transform_y == "LOG": # type: ignore[attr-defined] # noqa F821 rh2epm = RunHistory2EPM4LogCost( **r2e_def_kwargs) # type: ignore[arg-type] # noqa F821 elif scenario.transform_y == "LOGS": # type: ignore[attr-defined] # noqa F821 rh2epm = RunHistory2EPM4LogScaledCost( **r2e_def_kwargs) # type: ignore[arg-type] # noqa F821 elif scenario.transform_y == "INVS": # type: ignore[attr-defined] # noqa F821 rh2epm = RunHistory2EPM4InvScaledCost( **r2e_def_kwargs) # type: ignore[arg-type] # noqa F821 else: raise ValueError('Unknown run objective: %s. Should be either ' 'quality or runtime.' % self.scenario.run_obj) elif inspect.isclass(runhistory2epm): rh2epm = runhistory2epm( **r2e_def_kwargs) # type: ignore[arg-type] # noqa F821 else: raise TypeError( "Argument runhistory2epm must be None or an object implementing the RunHistory2EPM, but is '%s'" % type(runhistory2epm)) smbo_args = { 'scenario': scenario, 'stats': self.stats, 'initial_design': initial_design_instance, 'runhistory': runhistory, 'runhistory2epm': rh2epm, 'intensifier': intensifier_instance, 'num_run': run_id, 'model': model_instance, 'acq_optimizer': acquisition_function_optimizer_instance, 'acquisition_func': acquisition_function_instance, 'rng': rng, 'restore_incumbent': restore_incumbent, 'random_configuration_chooser': random_configuration_chooser_instance, 'tae_runner': tae_runner_instance, } # type: Dict[str, Any] if smbo_class is None: self.solver = SMBO(** smbo_args) # type: ignore[arg-type] # noqa F821 else: self.solver = smbo_class( **smbo_args) # type: ignore[arg-type] # noqa F821
def get_kernel(self, config_space): """ Code is very similar to those from :class:`smac.SMAC4BO`. Length scale were chosen from SMAC where they were obtained by hyperparameter optimization made in https://arxiv.org/abs/1908.06674. :params config_space: SMAC configuration space with parameters. :type config_space: :class:`smac.configspace.ConfigurationSpace` """ # First of all get type of kernel kernel_cls, nu = self._get_kernel_class_and_nu() # get types and bounds for config_space types, bounds = get_types(config_space, instance_features=None) # get random state for priors rng = self._get_random_state() # create kernel to hold variance cov_amp = smac.epm.gp_kernels.ConstantKernel( 2.0, constant_value_bounds=(np.exp(-10), np.exp(2)), prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng), ) # Understand information about parameters cont_dims = np.where(np.array(types) == 0)[0] cat_dims = np.where(np.array(types) != 0)[0] # bounds for length scale from https://arxiv.org/abs/1908.06674 lslims = (np.exp(-6.754111155189306), np.exp(0.0858637988771976)) # create kernel for continuous parameters if len(cont_dims) > 0: exp_kwargs = { "length_scale": np.ones([len(cont_dims)]), "length_scale_bounds": [lslims for _ in range(len(cont_dims))], "operate_on": cont_dims, "prior": None, "has_conditions": False, } if nu is not None: exp_kwargs["nu"] = nu exp_kernel = kernel_cls(**exp_kwargs) # kernel for categorical parameters if len(cat_dims) > 0: ham_kernel = smac.epm.gp_kernels.HammingKernel( length_scale=np.ones([len(cat_dims)]), length_scale_bounds=[lslims for _ in range(len(cat_dims))], operate_on=cat_dims, ) # create noise kernel noise_kernel = smac.epm.gp_kernels.WhiteKernel( noise_level=1e-8, noise_level_bounds=(np.exp(-25), np.exp(2)), prior=HorseshoePrior(scale=0.1, rng=rng), ) # create final kernel as combination if len(cont_dims) > 0 and len(cat_dims) > 0: # both kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel elif len(cont_dims) > 0 and len(cat_dims) == 0: # only continuous parameters kernel = cov_amp * exp_kernel + noise_kernel elif len(cont_dims) == 0 and len(cat_dims) > 0: # only categorical parameters kernel = cov_amp * ham_kernel + noise_kernel return kernel
def _get_types(self, scenario, features): types, bounds = get_types(scenario, features) types = np.array(types, dtype='uint') bounds = np.array(bounds, dtype='object') return types, bounds
def __init__(self, scenario: Scenario, tae_runner: Optional[Union[Type[ExecuteTARun], Callable]] = None, tae_runner_kwargs: Optional[dict] = None, runhistory: Optional[Union[Type[RunHistory], RunHistory]] = None, runhistory_kwargs: Optional[dict] = None, intensifier: Optional[Type[Intensifier]] = None, intensifier_kwargs: Optional[dict] = None, acquisition_function: Optional[Type[AbstractAcquisitionFunction]] = None, acquisition_function_kwargs: Optional[dict] = None, integrate_acquisition_function: bool = False, acquisition_function_optimizer: Optional[Type[AcquisitionFunctionMaximizer]] = None, acquisition_function_optimizer_kwargs: Optional[dict] = None, model: Optional[Type[AbstractEPM]] = None, model_kwargs: Optional[dict] = None, runhistory2epm: Optional[Type[AbstractRunHistory2EPM]] = None, runhistory2epm_kwargs: Optional[dict] = None, initial_design: Optional[Type[InitialDesign]] = None, initial_design_kwargs: Optional[dict] = None, initial_configurations: Optional[List[Configuration]] = None, stats: Optional[Stats] = None, restore_incumbent: Optional[Configuration] = None, rng: Optional[Union[np.random.RandomState, int]] = None, smbo_class: Optional[SMBO] = None, run_id: Optional[int] = None, random_configuration_chooser: Optional[Type[RandomConfigurationChooser]] = None, random_configuration_chooser_kwargs: Optional[dict] = None ): """ Constructor Parameters ---------- scenario : ~smac.scenario.scenario.Scenario Scenario object tae_runner : ~smac.tae.execute_ta_run.ExecuteTARun or callable Callable or implementation of :class:`~smac.tae.execute_ta_run.ExecuteTARun`. In case a callable is passed it will be wrapped by :class:`~smac.tae.execute_func.ExecuteTAFuncDict`. If not set, it will be initialized with the :class:`~smac.tae.execute_ta_run_old.ExecuteTARunOld`. tae_runner_kwargs: Optional[dict] arguments passed to constructor of '~tae_runner' runhistory : RunHistory runhistory to store all algorithm runs runhistory_kwargs : Optional[dict] arguments passed to constructor of runhistory. We strongly advise against changing the aggregation function, since it will break some code assumptions intensifier : Intensifier intensification object to issue a racing to decide the current incumbent intensifier_kwargs: Optional[dict] arguments passed to the constructor of '~intensifier' acquisition_function : ~smac.optimizer.acquisition.AbstractAcquisitionFunction Class or object that implements the :class:`~smac.optimizer.acquisition.AbstractAcquisitionFunction`. Will use :class:`~smac.optimizer.acquisition.EI` or :class:`~smac.optimizer.acquisition.LogEI` if not set. `~acquisition_function_kwargs` is passed to the class constructor. acquisition_function_kwargs : Optional[dict] dictionary to pass specific arguments to ~acquisition_function integrate_acquisition_function : bool, default=False Whether to integrate the acquisition function. Works only with models which can sample their hyperparameters (i.e. GaussianProcessMCMC). acquisition_function_optimizer : ~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer Object that implements the :class:`~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer`. Will use :class:`smac.optimizer.ei_optimization.InterleavedLocalAndRandomSearch` if not set. acquisition_function_optimizer_kwargs: Optional[dict] Arguments passed to constructor of '~acquisition_function_optimizer' model : AbstractEPM Model that implements train() and predict(). Will use a :class:`~smac.epm.rf_with_instances.RandomForestWithInstances` if not set. model_kwargs : Optional[dict] Arguments passed to constructor of '~model' runhistory2epm : ~smac.runhistory.runhistory2epm.RunHistory2EMP Object that implements the AbstractRunHistory2EPM. If None, will use :class:`~smac.runhistory.runhistory2epm.RunHistory2EPM4Cost` if objective is cost or :class:`~smac.runhistory.runhistory2epm.RunHistory2EPM4LogCost` if objective is runtime. runhistory2epm_kwargs: Optional[dict] Arguments passed to the constructor of '~runhistory2epm' initial_design : InitialDesign initial sampling design initial_design_kwargs: Optional[dict] arguments passed to constructor of `~initial_design' initial_configurations : List[Configuration] list of initial configurations for initial design -- cannot be used together with initial_design stats : Stats optional stats object rng : np.random.RandomState Random number generator restore_incumbent : Configuration incumbent used if restoring to previous state smbo_class : ~smac.optimizer.smbo.SMBO Class implementing the SMBO interface which will be used to instantiate the optimizer class. run_id : int (optional) Run ID will be used as subfolder for output_dir. If no ``run_id`` is given, a random ``run_id`` will be chosen. random_configuration_chooser : ~smac.optimizer.random_configuration_chooser.RandomConfigurationChooser How often to choose a random configuration during the intensification procedure. random_configuration_chooser_kwargs : Optional[dict] arguments of constructor for '~random_configuration_chooser' """ self.logger = logging.getLogger( self.__module__ + "." + self.__class__.__name__) aggregate_func = average_cost self.scenario = scenario self.output_dir = "" if not restore_incumbent: # restore_incumbent is used by the CLI interface which provides a method for restoring a SMAC run given an # output directory. This is the default path. # initial random number generator # run_id, rng = get_rng(rng=rng, run_id=run_id, logger=self.logger) # run_id=datetime.now().strftime("%Y%m%d%H%M%S%f") run_id=uuid1() self.output_dir = create_output_directory(scenario, run_id) # fixme run_id elif scenario.output_dir is not None: run_id, rng = get_rng(rng=rng, run_id=run_id, logger=self.logger) # output-directory is created in CLI when restoring from a # folder. calling the function again in the facade results in two # folders being created: run_X and run_X.OLD. if we are # restoring, the output-folder exists already and we omit creating it, # but set the self-output_dir to the dir. # necessary because we want to write traj to new output-dir in CLI. self.output_dir = scenario.output_dir_for_this_run if ( scenario.deterministic is True and getattr(scenario, 'tuner_timeout', None) is None and scenario.run_obj == 'quality' ): self.logger.info('Optimizing a deterministic scenario for quality without a tuner timeout - will make ' 'SMAC deterministic and only evaluate one configuration per iteration!') scenario.intensification_percentage = 1e-10 scenario.min_chall = 1 scenario.write() # initialize stats object if stats: self.stats = stats else: self.stats = Stats(scenario,file_system=scenario.file_system) if self.scenario.run_obj == "runtime" and not self.scenario.transform_y == "LOG": self.logger.warning("Runtime as objective automatically activates log(y) transformation") self.scenario.transform_y = "LOG" # initialize empty runhistory runhistory_def_kwargs = {'aggregate_func': aggregate_func} if runhistory_kwargs is not None: runhistory_def_kwargs.update(runhistory_kwargs) if runhistory is None: runhistory = RunHistory(**runhistory_def_kwargs,file_system=scenario.file_system) elif inspect.isclass(runhistory): runhistory = runhistory(**runhistory_def_kwargs) else: if runhistory.aggregate_func is None: runhistory.aggregate_func = aggregate_func rand_conf_chooser_kwargs = { 'rng': rng } if random_configuration_chooser_kwargs is not None: rand_conf_chooser_kwargs.update(random_configuration_chooser_kwargs) if random_configuration_chooser is None: if 'prob' not in rand_conf_chooser_kwargs: rand_conf_chooser_kwargs['prob'] = scenario.rand_prob random_configuration_chooser = ChooserProb(**rand_conf_chooser_kwargs) elif inspect.isclass(random_configuration_chooser): random_configuration_chooser = random_configuration_chooser(**rand_conf_chooser_kwargs) elif not isinstance(random_configuration_chooser, RandomConfigurationChooser): raise ValueError("random_configuration_chooser has to be" " a class or object of RandomConfigurationChooser") # reset random number generator in config space to draw different # random configurations with each seed given to SMAC scenario.cs.seed(rng.randint(MAXINT)) # initial Trajectory Logger traj_logger = TrajLogger(output_dir=self.output_dir, stats=self.stats,file_system=scenario.file_system) # initial EPM types, bounds = get_types(scenario.cs, scenario.feature_array) model_def_kwargs = { 'types': types, 'bounds': bounds, 'instance_features': scenario.feature_array, 'seed': rng.randint(MAXINT), 'pca_components': scenario.PCA_DIM, } if model_kwargs is not None: model_def_kwargs.update(model_kwargs) if model is None: for key, value in { 'log_y': scenario.transform_y in ["LOG", "LOGS"], 'num_trees': scenario.rf_num_trees, 'do_bootstrapping': scenario.rf_do_bootstrapping, 'ratio_features': scenario.rf_ratio_features, 'min_samples_split': scenario.rf_min_samples_split, 'min_samples_leaf': scenario.rf_min_samples_leaf, 'max_depth': scenario.rf_max_depth, }.items(): if key not in model_def_kwargs: model_def_kwargs[key] = value model_def_kwargs['configspace'] = self.scenario.cs model = RandomForestWithInstances(**model_def_kwargs) elif inspect.isclass(model): model_def_kwargs['configspace'] = self.scenario.cs model = model(**model_def_kwargs) else: raise TypeError( "Model not recognized: %s" %(type(model))) # initial acquisition function acq_def_kwargs = {'model': model} if acquisition_function_kwargs is not None: acq_def_kwargs.update(acquisition_function_kwargs) if acquisition_function is None: if scenario.transform_y in ["LOG", "LOGS"]: acquisition_function = LogEI(**acq_def_kwargs) else: acquisition_function = EI(**acq_def_kwargs) elif inspect.isclass(acquisition_function): acquisition_function = acquisition_function(**acq_def_kwargs) else: raise TypeError( "Argument acquisition_function must be None or an object implementing the " "AbstractAcquisitionFunction, not %s." % type(acquisition_function) ) if integrate_acquisition_function: acquisition_function = IntegratedAcquisitionFunction( acquisition_function=acquisition_function, **acq_def_kwargs ) # initialize optimizer on acquisition function acq_func_opt_kwargs = { 'acquisition_function': acquisition_function, 'config_space': scenario.cs, 'rng': rng, } if acquisition_function_optimizer_kwargs is not None: acq_func_opt_kwargs.update(acquisition_function_optimizer_kwargs) if acquisition_function_optimizer is None: for key, value in { 'max_steps': scenario.sls_max_steps, 'n_steps_plateau_walk': scenario.sls_n_steps_plateau_walk, }.items(): if key not in acq_func_opt_kwargs: acq_func_opt_kwargs[key] = value acquisition_function_optimizer = InterleavedLocalAndRandomSearch(**acq_func_opt_kwargs) elif inspect.isclass(acquisition_function_optimizer): acquisition_function_optimizer = acquisition_function_optimizer(**acq_func_opt_kwargs) else: raise TypeError( "Argument acquisition_function_optimizer must be None or an object implementing the " "AcquisitionFunctionMaximizer, but is '%s'" % type(acquisition_function_optimizer) ) # initialize tae_runner # First case, if tae_runner is None, the target algorithm is a call # string in the scenario file tae_def_kwargs = { 'stats': self.stats, 'run_obj': scenario.run_obj, 'runhistory': runhistory, 'par_factor': scenario.par_factor, 'cost_for_crash': scenario.cost_for_crash, 'abort_on_first_run_crash': scenario.abort_on_first_run_crash } if tae_runner_kwargs is not None: tae_def_kwargs.update(tae_runner_kwargs) if 'ta' not in tae_def_kwargs: tae_def_kwargs['ta'] = scenario.ta if tae_runner is None: tae_def_kwargs['ta'] = scenario.ta tae_runner = ExecuteTARunOld(**tae_def_kwargs) elif inspect.isclass(tae_runner): tae_runner = tae_runner(**tae_def_kwargs) elif callable(tae_runner): tae_def_kwargs['ta'] = tae_runner tae_runner = ExecuteTAFuncDict(**tae_def_kwargs) else: raise TypeError("Argument 'tae_runner' is %s, but must be " "either None, a callable or an object implementing " "ExecuteTaRun. Passing 'None' will result in the " "creation of target algorithm runner based on the " "call string in the scenario file." % type(tae_runner)) # Check that overall objective and tae objective are the same if tae_runner.run_obj != scenario.run_obj: raise ValueError("Objective for the target algorithm runner and " "the scenario must be the same, but are '%s' and " "'%s'" % (tae_runner.run_obj, scenario.run_obj)) # initialize intensification intensifier_def_kwargs = { 'tae_runner': tae_runner, 'stats': self.stats, 'traj_logger': traj_logger, 'rng': rng, 'instances': scenario.train_insts, 'cutoff': scenario.cutoff, 'deterministic': scenario.deterministic, 'run_obj_time': scenario.run_obj == "runtime", 'always_race_against': scenario.cs.get_default_configuration() if scenario.always_race_default else None, 'use_ta_time_bound': scenario.use_ta_time, 'instance_specifics': scenario.instance_specific, 'minR': scenario.minR, 'maxR': scenario.maxR, 'adaptive_capping_slackfactor': scenario.intens_adaptive_capping_slackfactor, 'min_chall': scenario.intens_min_chall } if intensifier_kwargs is not None: intensifier_def_kwargs.update(intensifier_kwargs) if intensifier is None: intensifier = Intensifier(**intensifier_def_kwargs) elif inspect.isclass(intensifier): intensifier = intensifier(**intensifier_def_kwargs) else: raise TypeError( "Argument intensifier must be None or an object implementing the Intensifier, but is '%s'" % type(intensifier) ) # initial design if initial_design is not None and initial_configurations is not None: raise ValueError( "Either use initial_design or initial_configurations; but not both") init_design_def_kwargs = { 'tae_runner': tae_runner, 'scenario': scenario, 'stats': self.stats, 'traj_logger': traj_logger, 'runhistory': runhistory, 'rng': rng, 'configs': initial_configurations, 'intensifier': intensifier, 'aggregate_func': aggregate_func, 'n_configs_x_params': 0, 'max_config_fracs': 0.0 } if initial_design_kwargs is not None: init_design_def_kwargs.update(initial_design_kwargs) if initial_configurations is not None: initial_design = InitialDesign(**init_design_def_kwargs) elif initial_design is None: if scenario.initial_incumbent == "DEFAULT": init_design_def_kwargs['max_config_fracs'] = 0.0 initial_design = DefaultConfiguration(**init_design_def_kwargs) elif scenario.initial_incumbent == "RANDOM": init_design_def_kwargs['max_config_fracs'] = 0.0 initial_design = RandomConfigurations(**init_design_def_kwargs) elif scenario.initial_incumbent == "LHD": initial_design = LHDesign(**init_design_def_kwargs) elif scenario.initial_incumbent == "FACTORIAL": initial_design = FactorialInitialDesign(**init_design_def_kwargs) elif scenario.initial_incumbent == "SOBOL": initial_design = SobolDesign(**init_design_def_kwargs) else: raise ValueError("Don't know what kind of initial_incumbent " "'%s' is" % scenario.initial_incumbent) elif inspect.isclass(initial_design): initial_design = initial_design(**init_design_def_kwargs) else: raise TypeError( "Argument initial_design must be None or an object implementing the InitialDesign, but is '%s'" % type(initial_design) ) # if we log the performance data, # the RFRImputator will already get # log transform data from the runhistory if scenario.transform_y in ["LOG", "LOGS"]: cutoff = np.log(np.nanmin([np.inf, np.float_(scenario.cutoff)])) threshold = cutoff + np.log(scenario.par_factor) else: cutoff = np.nanmin([np.inf, np.float_(scenario.cutoff)]) threshold = cutoff * scenario.par_factor num_params = len(scenario.cs.get_hyperparameters()) imputor = RFRImputator(rng=rng, cutoff=cutoff, threshold=threshold, model=model, change_threshold=0.01, max_iter=2) r2e_def_kwargs = { 'scenario': scenario, 'num_params': num_params, 'success_states': [StatusType.SUCCESS, ], 'impute_censored_data': True, 'impute_state': [StatusType.CAPPED, ], 'imputor': imputor, 'scale_perc': 5 } if scenario.run_obj == 'quality': r2e_def_kwargs.update({ 'success_states': [StatusType.SUCCESS, StatusType.CRASHED], 'impute_censored_data': False, 'impute_state': None, }) if runhistory2epm_kwargs is not None: r2e_def_kwargs.update(runhistory2epm_kwargs) if runhistory2epm is None: if scenario.run_obj == 'runtime': runhistory2epm = RunHistory2EPM4LogCost(**r2e_def_kwargs) elif scenario.run_obj == 'quality': if scenario.transform_y == "NONE": runhistory2epm = RunHistory2EPM4Cost(**r2e_def_kwargs) elif scenario.transform_y == "LOG": runhistory2epm = RunHistory2EPM4LogCost(**r2e_def_kwargs) elif scenario.transform_y == "LOGS": runhistory2epm = RunHistory2EPM4LogScaledCost(**r2e_def_kwargs) elif scenario.transform_y == "INVS": runhistory2epm = RunHistory2EPM4InvScaledCost(**r2e_def_kwargs) else: raise ValueError('Unknown run objective: %s. Should be either ' 'quality or runtime.' % self.scenario.run_obj) elif inspect.isclass(runhistory2epm): runhistory2epm = runhistory2epm(**r2e_def_kwargs) else: raise TypeError( "Argument runhistory2epm must be None or an object implementing the RunHistory2EPM, but is '%s'" % type(runhistory2epm) ) smbo_args = { 'scenario': scenario, 'stats': self.stats, 'initial_design': initial_design, 'runhistory': runhistory, 'runhistory2epm': runhistory2epm, 'intensifier': intensifier, 'aggregate_func': aggregate_func, 'num_run': run_id, 'model': model, 'acq_optimizer': acquisition_function_optimizer, 'acquisition_func': acquisition_function, 'rng': rng, 'restore_incumbent': restore_incumbent, 'random_configuration_chooser': random_configuration_chooser } if smbo_class is None: self.solver = SMBO(**smbo_args) else: self.solver = smbo_class(**smbo_args)
def __init__(self, model_type='gp_mcmc', **kwargs): """ Constructor see ~smac.facade.smac_facade for documentation """ scenario = kwargs['scenario'] kwargs['initial_design'] = kwargs.get('initial_design', SobolDesign) kwargs['runhistory2epm'] = kwargs.get('runhistory2epm', RunHistory2EPM4Cost) init_kwargs = kwargs.get('initial_design_kwargs', dict()) init_kwargs['n_configs_x_params'] = init_kwargs.get( 'n_configs_x_params', 8) init_kwargs['max_config_fracs'] = init_kwargs.get( 'max_config_fracs', 0.25) kwargs['initial_design_kwargs'] = init_kwargs if kwargs.get('model') is None: from smac.epm.gp_kernels import ConstantKernel, Matern, WhiteKernel, HammingKernel model_kwargs = kwargs.get('model_kwargs', dict()) _, rng = get_rng(rng=kwargs.get("rng", None), run_id=kwargs.get("run_id", None), logger=None) types, bounds = get_types(kwargs['scenario'].cs, instance_features=None) cov_amp = ConstantKernel( 2.0, constant_value_bounds=(np.exp(-10), np.exp(2)), prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng), ) cont_dims = np.nonzero(types == 0)[0] cat_dims = np.nonzero(types != 0)[0] if len(cont_dims) > 0: exp_kernel = Matern( np.ones([len(cont_dims)]), [(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cont_dims))], nu=2.5, operate_on=cont_dims, ) if len(cat_dims) > 0: ham_kernel = HammingKernel( np.ones([len(cat_dims)]), [(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cat_dims))], operate_on=cat_dims, ) noise_kernel = WhiteKernel( noise_level=1e-8, noise_level_bounds=(np.exp(-25), np.exp(2)), prior=HorseshoePrior(scale=0.1, rng=rng), ) if len(cont_dims) > 0 and len(cat_dims) > 0: # both kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel elif len(cont_dims) > 0 and len(cat_dims) == 0: # only cont kernel = cov_amp * exp_kernel + noise_kernel elif len(cont_dims) == 0 and len(cat_dims) > 0: # only cont kernel = cov_amp * ham_kernel + noise_kernel else: raise ValueError() if model_type == "gp": model_class = GaussianProcess kwargs['model'] = model_class model_kwargs['kernel'] = kernel model_kwargs['normalize_y'] = True model_kwargs['seed'] = rng.randint(0, 2**20) elif model_type == "gp_mcmc": model_class = GaussianProcessMCMC kwargs['model'] = model_class kwargs['integrate_acquisition_function'] = True model_kwargs['kernel'] = kernel n_mcmc_walkers = 3 * len(kernel.theta) if n_mcmc_walkers % 2 == 1: n_mcmc_walkers += 1 model_kwargs['n_mcmc_walkers'] = n_mcmc_walkers model_kwargs['chain_length'] = 250 model_kwargs['burnin_steps'] = 250 model_kwargs['normalize_y'] = True model_kwargs['seed'] = rng.randint(0, 2**20) else: raise ValueError('Unknown model type %s' % model_type) kwargs['model_kwargs'] = model_kwargs if kwargs.get('random_configuration_chooser') is None: random_config_chooser_kwargs = kwargs.get( 'random_configuration_chooser_kwargs', dict()) random_config_chooser_kwargs[ 'prob'] = random_config_chooser_kwargs.get( 'prob', 0.08447232371720552) kwargs[ 'random_configuration_chooser_kwargs'] = random_config_chooser_kwargs if kwargs.get('acquisition_function_optimizer') is None: acquisition_function_optimizer_kwargs = kwargs.get( 'acquisition_function_optimizer_kwargs', dict()) acquisition_function_optimizer_kwargs['n_sls_iterations'] = 10 kwargs[ 'acquisition_function_optimizer_kwargs'] = acquisition_function_optimizer_kwargs # only 1 configuration per SMBO iteration intensifier_kwargs = kwargs.get('intensifier_kwargs', dict()) intensifier_kwargs['min_chall'] = 1 kwargs['intensifier_kwargs'] = intensifier_kwargs scenario.intensification_percentage = 1e-10 super().__init__(**kwargs) if self.solver.scenario.n_features > 0: raise NotImplementedError("BOGP cannot handle instances") self.logger.info(self.__class__) self.solver.scenario.acq_opt_challengers = 1000 # activate predict incumbent self.solver.predict_incumbent = True
def _get_mean_var_time(self, validator, traj, use_epm, rh): """ Parameters ---------- validator: Validator validator (smac-based) traj: List[Configuraton] trajectory to set in validator use_epm: bool validated or not (no need to use epm if validated) rh: RunHistory ?? Returns ------- mean, var times: List[float] times to plot (x-values) configs """ # TODO kinda important: docstrings, what is this function doing? if validator: validator.traj = traj # set trajectory time, configs = [], [] if use_epm and not self.block_epm: for entry in traj: time.append(entry["wallclock_time"]) configs.append(entry["incumbent"]) # self.logger.debug('Time: %d Runs: %d', time[-1], len(rh.get_runs_for_config(configs[-1]))) self.logger.debug( "Using %d samples (%d distinct) from trajectory.", len(time), len(set(configs))) # Initialize EPM if validator.epm: # not log as validator epm is trained on cost, not log cost epm = validator.epm else: self.logger.debug( "No EPM passed! Training new one from runhistory.") # Train random forest and transform training data (from given rh) # Not using validator because we want to plot uncertainties rh2epm = RunHistory2EPM4Cost(num_params=len( self.scenario.cs.get_hyperparameters()), scenario=self.scenario) X, y = rh2epm.transform(rh) self.logger.debug( "Training model with data of shape X: %s, y: %s", str(X.shape), str(y.shape)) types, bounds = get_types(self.scenario.cs, self.scenario.feature_array) epm = RandomForestWithInstances( self.scenario.cs, types=types, bounds=bounds, seed=self.rng.randint(MAXINT), instance_features=self.scenario.feature_array, ratio_features=1.0) epm.train(X, y) config_array = convert_configurations_to_array(configs) mean, var = epm.predict_marginalized_over_instances(config_array) var = np.zeros(mean.shape) # We don't want to show the uncertainty of the model but uncertainty over multiple optimizer runs # This variance is computed in an outer loop. else: mean, var = [], [] for entry in traj: #self.logger.debug(entry) time.append(entry["wallclock_time"]) configs.append(entry["incumbent"]) costs = _cost(configs[-1], rh, rh.get_runs_for_config(configs[-1])) # self.logger.debug(len(costs), time[-1] if not costs: time.pop() else: mean.append(np.mean(costs)) var.append(0) # No variance over instances mean, var = np.array(mean).reshape(-1, 1), np.array(var).reshape( -1, 1) return mean, var, time, configs
def __init__( self, scenario: Scenario, # TODO: once we drop python3.4 add type hint # typing.Union[ExecuteTARun, callable] tae_runner=None, runhistory: RunHistory = None, intensifier: Intensifier = None, acquisition_function: AbstractAcquisitionFunction = None, model: AbstractEPM = None, runhistory2epm: AbstractRunHistory2EPM = None, initial_design: InitialDesign = None, initial_configurations: typing.List[Configuration] = None, stats: Stats = None, rng: np.random.RandomState = None, run_id: int = 1): """Constructor""" self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) aggregate_func = average_cost self.runhistory = None self.trajectory = None # initialize stats object if stats: self.stats = stats else: self.stats = Stats(scenario, file_system=scenario.file_system) self.output_dir = create_output_directory(scenario, run_id) scenario.write() # initialize empty runhistory if runhistory is None: runhistory = RunHistory(aggregate_func=aggregate_func, file_system=scenario.file_system) # inject aggr_func if necessary if runhistory.aggregate_func is None: runhistory.aggregate_func = aggregate_func # initial random number generator num_run, rng = self._get_rng(rng=rng) # reset random number generator in config space to draw different # random configurations with each seed given to SMAC scenario.cs.seed(rng.randint(MAXINT)) # initial Trajectory Logger traj_logger = TrajLogger(output_dir=self.output_dir, stats=self.stats, file_system=scenario.file_system) # initial EPM types, bounds = get_types(scenario.cs, scenario.feature_array) if model is None: model = RandomForestWithInstances( configspace=scenario.cs, types=types, bounds=bounds, instance_features=scenario.feature_array, seed=rng.randint(MAXINT), pca_components=scenario.PCA_DIM, num_trees=scenario.rf_num_trees, do_bootstrapping=scenario.rf_do_bootstrapping, ratio_features=scenario.rf_ratio_features, min_samples_split=scenario.rf_min_samples_split, min_samples_leaf=scenario.rf_min_samples_leaf, max_depth=scenario.rf_max_depth, ) # initial acquisition function if acquisition_function is None: if scenario.run_obj == "runtime": acquisition_function = LogEI(model=model) else: acquisition_function = EI(model=model) # inject model if necessary if acquisition_function.model is None: acquisition_function.model = model # initialize optimizer on acquisition function local_search = LocalSearch( acquisition_function, scenario.cs, max_steps=scenario.sls_max_steps, n_steps_plateau_walk=scenario.sls_n_steps_plateau_walk) # initialize tae_runner # First case, if tae_runner is None, the target algorithm is a call # string in the scenario file if tae_runner is None: tae_runner = ExecuteTARunOld( ta=scenario.ta, stats=self.stats, run_obj=scenario.run_obj, runhistory=runhistory, par_factor=scenario.par_factor, cost_for_crash=scenario.cost_for_crash) # Second case, the tae_runner is a function to be optimized elif callable(tae_runner): tae_runner = ExecuteTAFuncDict( ta=tae_runner, stats=self.stats, run_obj=scenario.run_obj, memory_limit=scenario.memory_limit, runhistory=runhistory, par_factor=scenario.par_factor, cost_for_crash=scenario.cost_for_crash) # Third case, if it is an ExecuteTaRun we can simply use the # instance. Otherwise, the next check raises an exception elif not isinstance(tae_runner, ExecuteTARun): raise TypeError("Argument 'tae_runner' is %s, but must be " "either a callable or an instance of " "ExecuteTaRun. Passing 'None' will result in the " "creation of target algorithm runner based on the " "call string in the scenario file." % type(tae_runner)) # Check that overall objective and tae objective are the same if tae_runner.run_obj != scenario.run_obj: raise ValueError("Objective for the target algorithm runner and " "the scenario must be the same, but are '%s' and " "'%s'" % (tae_runner.run_obj, scenario.run_obj)) # inject stats if necessary if tae_runner.stats is None: tae_runner.stats = self.stats # inject runhistory if necessary if tae_runner.runhistory is None: tae_runner.runhistory = runhistory # inject cost_for_crash if tae_runner.crash_cost != scenario.cost_for_crash: tae_runner.crash_cost = scenario.cost_for_crash # initialize intensification if intensifier is None: intensifier = Intensifier( tae_runner=tae_runner, stats=self.stats, traj_logger=traj_logger, rng=rng, instances=scenario.train_insts, cutoff=scenario.cutoff, deterministic=scenario.deterministic, run_obj_time=scenario.run_obj == "runtime", always_race_against=scenario.cs.get_default_configuration() if scenario.always_race_default else None, instance_specifics=scenario.instance_specific, minR=scenario.minR, maxR=scenario.maxR, adaptive_capping_slackfactor=scenario. intens_adaptive_capping_slackfactor, min_chall=scenario.intens_min_chall) # inject deps if necessary if intensifier.tae_runner is None: intensifier.tae_runner = tae_runner if intensifier.stats is None: intensifier.stats = self.stats if intensifier.traj_logger is None: intensifier.traj_logger = traj_logger # initial design if initial_design is not None and initial_configurations is not None: raise ValueError( "Either use initial_design or initial_configurations; but not both" ) if initial_configurations is not None: initial_design = InitialDesign(tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, runhistory=runhistory, rng=rng, configs=initial_configurations, intensifier=intensifier, aggregate_func=aggregate_func) elif initial_design is None: if scenario.initial_incumbent == "DEFAULT": initial_design = DefaultConfiguration( tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, runhistory=runhistory, rng=rng, intensifier=intensifier, aggregate_func=aggregate_func, max_config_fracs=0.0) elif scenario.initial_incumbent == "RANDOM": initial_design = RandomConfigurations( tae_runner=tae_runner, scenario=scenario, stats=self.stats, traj_logger=traj_logger, runhistory=runhistory, rng=rng, intensifier=intensifier, aggregate_func=aggregate_func, max_config_fracs=0.0) else: raise ValueError("Don't know what kind of initial_incumbent " "'%s' is" % scenario.initial_incumbent) # inject deps if necessary if initial_design.tae_runner is None: initial_design.tae_runner = tae_runner if initial_design.scenario is None: initial_design.scenario = scenario if initial_design.stats is None: initial_design.stats = self.stats if initial_design.traj_logger is None: initial_design.traj_logger = traj_logger # initial conversion of runhistory into EPM data if runhistory2epm is None: num_params = len(scenario.cs.get_hyperparameters()) if scenario.run_obj == "runtime": # if we log the performance data, # the RFRImputator will already get # log transform data from the runhistory cutoff = np.log(scenario.cutoff) threshold = np.log(scenario.cutoff * scenario.par_factor) imputor = RFRImputator(rng=rng, cutoff=cutoff, threshold=threshold, model=model, change_threshold=0.01, max_iter=2) runhistory2epm = RunHistory2EPM4LogCost( scenario=scenario, num_params=num_params, success_states=[ StatusType.SUCCESS, ], impute_censored_data=True, impute_state=[ StatusType.CAPPED, ], imputor=imputor) elif scenario.run_obj == 'quality': runhistory2epm = RunHistory2EPM4Cost( scenario=scenario, num_params=num_params, success_states=[ StatusType.SUCCESS, ], impute_censored_data=False, impute_state=None) else: raise ValueError('Unknown run objective: %s. Should be either ' 'quality or runtime.' % self.scenario.run_obj) # inject scenario if necessary: if runhistory2epm.scenario is None: runhistory2epm.scenario = scenario self.solver = EPILS_Solver(scenario=scenario, stats=self.stats, initial_design=initial_design, runhistory=runhistory, runhistory2epm=runhistory2epm, intensifier=intensifier, aggregate_func=aggregate_func, num_run=num_run, model=model, acq_optimizer=local_search, acquisition_func=acquisition_function, rng=rng)
def __init__( self, scenario: Scenario, tae_runner: Optional[Union[Type[BaseRunner], Callable]] = None, tae_runner_kwargs: Optional[Dict] = None, runhistory: Optional[Union[Type[RunHistory], RunHistory]] = None, runhistory_kwargs: Optional[Dict] = None, intensifier: Optional[Type[AbstractRacer]] = None, intensifier_kwargs: Optional[Dict] = None, acquisition_function: Optional[ Type[AbstractAcquisitionFunction]] = None, acquisition_function_kwargs: Optional[Dict] = None, integrate_acquisition_function: bool = False, acquisition_function_optimizer: Optional[ Type[AcquisitionFunctionMaximizer]] = None, acquisition_function_optimizer_kwargs: Optional[Dict] = None, model: Optional[Type[AbstractEPM]] = None, model_kwargs: Optional[Dict] = None, runhistory2epm: Optional[Type[AbstractRunHistory2EPM]] = None, runhistory2epm_kwargs: Optional[Dict] = None, multi_objective_algorithm: Optional[ Type[AbstractMultiObjectiveAlgorithm]] = None, multi_objective_kwargs: Optional[Dict] = None, initial_design: Optional[Type[InitialDesign]] = None, initial_design_kwargs: Optional[Dict] = None, initial_configurations: Optional[List[Configuration]] = None, stats: Optional[Stats] = None, restore_incumbent: Optional[Configuration] = None, rng: Optional[Union[np.random.RandomState, int]] = None, smbo_class: Optional[Type[SMBO]] = None, run_id: Optional[int] = None, random_configuration_chooser: Optional[ Type[RandomConfigurationChooser]] = None, random_configuration_chooser_kwargs: Optional[Dict] = None, dask_client: Optional[dask.distributed.Client] = None, n_jobs: Optional[int] = 1, ): self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) self.scenario = scenario self.output_dir = "" if not restore_incumbent: # restore_incumbent is used by the CLI interface which provides a method for restoring a SMAC run given an # output directory. This is the default path. # initial random number generator run_id, rng = get_rng(rng=rng, run_id=run_id, logger=self.logger) self.output_dir = create_output_directory(scenario, run_id) elif scenario.output_dir is not None: # type: ignore[attr-defined] # noqa F821 run_id, rng = get_rng(rng=rng, run_id=run_id, logger=self.logger) # output-directory is created in CLI when restoring from a # folder. calling the function again in the facade results in two # folders being created: run_X and run_X.OLD. if we are # restoring, the output-folder exists already and we omit creating it, # but set the self-output_dir to the dir. # necessary because we want to write traj to new output-dir in CLI. self.output_dir = cast(str, scenario.output_dir_for_this_run ) # type: ignore[attr-defined] # noqa F821 rng = cast(np.random.RandomState, rng) if (scenario.deterministic is True # type: ignore[attr-defined] # noqa F821 and getattr(scenario, "tuner_timeout", None) is None and scenario.run_obj == "quality" # type: ignore[attr-defined] # noqa F821 ): self.logger.info( "Optimizing a deterministic scenario for quality without a tuner timeout - will make " "SMAC deterministic and only evaluate one configuration per iteration!" ) scenario.intensification_percentage = 1e-10 # type: ignore[attr-defined] # noqa F821 scenario.min_chall = 1 # type: ignore[attr-defined] # noqa F821 scenario.write() # initialize stats object if stats: self.stats = stats else: self.stats = Stats(scenario) if self.scenario.run_obj == "runtime" and not self.scenario.transform_y == "LOG": # type: ignore[attr-defined] # noqa F821 self.logger.warning( "Runtime as objective automatically activates log(y) transformation" ) self.scenario.transform_y = "LOG" # type: ignore[attr-defined] # noqa F821 # initialize empty runhistory num_obj = len(scenario.multi_objectives ) # type: ignore[attr-defined] # noqa F821 runhistory_def_kwargs = {} if runhistory_kwargs is not None: runhistory_def_kwargs.update(runhistory_kwargs) if runhistory is None: runhistory = RunHistory(**runhistory_def_kwargs) elif inspect.isclass(runhistory): runhistory = runhistory( **runhistory_def_kwargs) # type: ignore[operator] # noqa F821 elif isinstance(runhistory, RunHistory): pass else: raise ValueError( "runhistory has to be a class or an object of RunHistory") rand_conf_chooser_kwargs = {"rng": rng} if random_configuration_chooser_kwargs is not None: rand_conf_chooser_kwargs.update( random_configuration_chooser_kwargs) if random_configuration_chooser is None: if "prob" not in rand_conf_chooser_kwargs: rand_conf_chooser_kwargs[ "prob"] = scenario.rand_prob # type: ignore[attr-defined] # noqa F821 random_configuration_chooser_instance = ChooserProb( ** rand_conf_chooser_kwargs # type: ignore[arg-type] # noqa F821 # type: RandomConfigurationChooser ) elif inspect.isclass(random_configuration_chooser): random_configuration_chooser_instance = random_configuration_chooser( # type: ignore # noqa F821 ** rand_conf_chooser_kwargs # type: ignore[arg-type] # noqa F821 ) elif not isinstance(random_configuration_chooser, RandomConfigurationChooser): raise ValueError( "random_configuration_chooser has to be" " a class or object of RandomConfigurationChooser") # reset random number generator in config space to draw different # random configurations with each seed given to SMAC scenario.cs.seed( rng.randint(MAXINT)) # type: ignore[attr-defined] # noqa F821 # initial Trajectory Logger traj_logger = TrajLogger(output_dir=self.output_dir, stats=self.stats) # initial EPM types, bounds = get_types( scenario.cs, scenario.feature_array) # type: ignore[attr-defined] # noqa F821 model_def_kwargs = { "types": types, "bounds": bounds, "instance_features": scenario.feature_array, "seed": rng.randint(MAXINT), "pca_components": scenario.PCA_DIM, } if model_kwargs is not None: model_def_kwargs.update(model_kwargs) if model is None: for key, value in { "log_y": scenario.transform_y in ["LOG", "LOGS"], # type: ignore[attr-defined] # noqa F821 "num_trees": scenario. rf_num_trees, # type: ignore[attr-defined] # noqa F821 "do_bootstrapping": scenario. rf_do_bootstrapping, # type: ignore[attr-defined] # noqa F821 "ratio_features": scenario. rf_ratio_features, # type: ignore[attr-defined] # noqa F821 "min_samples_split": scenario. rf_min_samples_split, # type: ignore[attr-defined] # noqa F821 "min_samples_leaf": scenario. rf_min_samples_leaf, # type: ignore[attr-defined] # noqa F821 "max_depth": scenario. rf_max_depth, # type: ignore[attr-defined] # noqa F821 }.items(): if key not in model_def_kwargs: model_def_kwargs[key] = value model_def_kwargs[ "configspace"] = self.scenario.cs # type: ignore[attr-defined] # noqa F821 model_instance = RandomForestWithInstances( ** model_def_kwargs # type: ignore[arg-type] # noqa F821 # type: AbstractEPM ) elif inspect.isclass(model): model_def_kwargs[ "configspace"] = self.scenario.cs # type: ignore[attr-defined] # noqa F821 model_instance = model( **model_def_kwargs) # type: ignore # noqa F821 else: raise TypeError("Model not recognized: %s" % (type(model))) # initial acquisition function acq_def_kwargs = {"model": model_instance} if acquisition_function_kwargs is not None: acq_def_kwargs.update(acquisition_function_kwargs) acquisition_function_instance = ( None) # type: Optional[AbstractAcquisitionFunction] if acquisition_function is None: if scenario.transform_y in [ "LOG", "LOGS" ]: # type: ignore[attr-defined] # noqa F821 acquisition_function_instance = LogEI( **acq_def_kwargs # type: ignore[arg-type] # noqa F821 ) else: acquisition_function_instance = EI( **acq_def_kwargs # type: ignore[arg-type] # noqa F821 ) elif inspect.isclass(acquisition_function): acquisition_function_instance = acquisition_function( **acq_def_kwargs) else: raise TypeError( "Argument acquisition_function must be None or an object implementing the " "AbstractAcquisitionFunction, not %s." % type(acquisition_function)) if integrate_acquisition_function: acquisition_function_instance = IntegratedAcquisitionFunction( acquisition_function= acquisition_function_instance, # type: ignore **acq_def_kwargs, ) # initialize optimizer on acquisition function acq_func_opt_kwargs = { "acquisition_function": acquisition_function_instance, "config_space": scenario.cs, # type: ignore[attr-defined] # noqa F821 "rng": rng, } if acquisition_function_optimizer_kwargs is not None: acq_func_opt_kwargs.update(acquisition_function_optimizer_kwargs) if acquisition_function_optimizer is None: for key, value in { "max_steps": scenario. sls_max_steps, # type: ignore[attr-defined] # noqa F821 "n_steps_plateau_walk": scenario. sls_n_steps_plateau_walk, # type: ignore[attr-defined] # noqa F821 }.items(): if key not in acq_func_opt_kwargs: acq_func_opt_kwargs[key] = value acquisition_function_optimizer_instance = LocalAndSortedRandomSearch( **acq_func_opt_kwargs # type: ignore ) elif inspect.isclass(acquisition_function_optimizer): acquisition_function_optimizer_instance = acquisition_function_optimizer( # type: ignore # noqa F821 **acq_func_opt_kwargs) # type: ignore # noqa F821 else: raise TypeError( "Argument acquisition_function_optimizer must be None or an object implementing the " "AcquisitionFunctionMaximizer, but is '%s'" % type(acquisition_function_optimizer)) # initialize tae_runner # First case, if tae_runner is None, the target algorithm is a call # string in the scenario file tae_def_kwargs = { "stats": self.stats, "run_obj": scenario.run_obj, "par_factor": scenario.par_factor, # type: ignore[attr-defined] # noqa F821 "cost_for_crash": scenario.cost_for_crash, # type: ignore[attr-defined] # noqa F821 "abort_on_first_run_crash": scenario. abort_on_first_run_crash, # type: ignore[attr-defined] # noqa F821 "multi_objectives": scenario. multi_objectives, # type: ignore[attr-defined] # noqa F821 } if tae_runner_kwargs is not None: tae_def_kwargs.update(tae_runner_kwargs) if "ta" not in tae_def_kwargs: tae_def_kwargs[ "ta"] = scenario.ta # type: ignore[attr-defined] # noqa F821 if tae_runner is None: tae_def_kwargs[ "ta"] = scenario.ta # type: ignore[attr-defined] # noqa F821 tae_runner_instance = ExecuteTARunOld( **tae_def_kwargs ) # type: ignore[arg-type] # noqa F821 # type: BaseRunner elif inspect.isclass(tae_runner): tae_runner_instance = cast( BaseRunner, tae_runner(**tae_def_kwargs)) # type: ignore elif callable(tae_runner): tae_def_kwargs["ta"] = tae_runner tae_def_kwargs[ "use_pynisher"] = scenario.limit_resources # type: ignore[attr-defined] # noqa F821 tae_def_kwargs[ "memory_limit"] = scenario.memory_limit # type: ignore[attr-defined] # noqa F821 tae_runner_instance = ExecuteTAFuncDict( **tae_def_kwargs) # type: ignore else: raise TypeError( "Argument 'tae_runner' is %s, but must be " "either None, a callable or an object implementing " "BaseRunner. Passing 'None' will result in the " "creation of target algorithm runner based on the " "call string in the scenario file." % type(tae_runner)) # In case of a parallel run, wrap the single worker in a parallel # runner if n_jobs is None or n_jobs == 1: _n_jobs = 1 elif n_jobs == -1: _n_jobs = joblib.cpu_count() elif n_jobs > 0: _n_jobs = n_jobs else: raise ValueError( "Number of tasks must be positive, None or -1, but is %s" % str(n_jobs)) if _n_jobs > 1 or dask_client is not None: tae_runner_instance = DaskParallelRunner( # type: ignore tae_runner_instance, n_workers=_n_jobs, output_directory=self.output_dir, dask_client=dask_client, ) # Check that overall objective and tae objective are the same # TODO: remove these two ignores once the scenario object knows all its attributes! if tae_runner_instance.run_obj != scenario.run_obj: # type: ignore[union-attr] # noqa F821 raise ValueError( "Objective for the target algorithm runner and " "the scenario must be the same, but are '%s' and " "'%s'" % (tae_runner_instance.run_obj, scenario.run_obj)) # type: ignore[union-attr] # noqa F821 if intensifier is None: intensifier = Intensifier if isinstance(intensifier, AbstractRacer): intensifier_instance = intensifier elif inspect.isclass(intensifier): # initialize intensification intensifier_def_kwargs = { "stats": self.stats, "traj_logger": traj_logger, "rng": rng, "instances": scenario.train_insts, # type: ignore[attr-defined] # noqa F821 "cutoff": scenario.cutoff, # type: ignore[attr-defined] # noqa F821 "deterministic": scenario. deterministic, # type: ignore[attr-defined] # noqa F821 "run_obj_time": scenario.run_obj == "runtime", # type: ignore[attr-defined] # noqa F821 "instance_specifics": scenario. instance_specific, # type: ignore[attr-defined] # noqa F821 "adaptive_capping_slackfactor": scenario. intens_adaptive_capping_slackfactor, # type: ignore[attr-defined] # noqa F821 "min_chall": scenario. intens_min_chall, # type: ignore[attr-defined] # noqa F821 } if issubclass(intensifier, Intensifier): intensifier_def_kwargs[ "always_race_against"] = scenario.cs.get_default_configuration( ) # type: ignore[attr-defined] # noqa F821 intensifier_def_kwargs[ "use_ta_time_bound"] = scenario.use_ta_time # type: ignore[attr-defined] # noqa F821 intensifier_def_kwargs[ "minR"] = scenario.minR # type: ignore[attr-defined] # noqa F821 intensifier_def_kwargs[ "maxR"] = scenario.maxR # type: ignore[attr-defined] # noqa F821 if intensifier_kwargs is not None: intensifier_def_kwargs.update(intensifier_kwargs) intensifier_instance = intensifier( **intensifier_def_kwargs) # type: ignore[arg-type] # noqa F821 else: raise TypeError( "Argument intensifier must be None or an object implementing the AbstractRacer, but is '%s'" % type(intensifier)) # initialize multi objective # the multi_objective_algorithm_instance will be passed to the runhistory2epm object multi_objective_algorithm_instance = ( None) # type: Optional[AbstractMultiObjectiveAlgorithm] if scenario.multi_objectives is not None and num_obj > 1: # type: ignore[attr-defined] # noqa F821 # define any defaults here _multi_objective_kwargs = {"rng": rng, "num_obj": num_obj} if multi_objective_kwargs is not None: _multi_objective_kwargs.update(multi_objective_kwargs) if multi_objective_algorithm is None: multi_objective_algorithm_instance = MeanAggregationStrategy( **_multi_objective_kwargs ) # type: ignore[arg-type] # noqa F821 elif inspect.isclass(multi_objective_algorithm): multi_objective_algorithm_instance = multi_objective_algorithm( **_multi_objective_kwargs) else: raise TypeError( "Multi-objective algorithm not recognized: %s" % (type(multi_objective_algorithm))) # initial design if initial_design is not None and initial_configurations is not None: raise ValueError( "Either use initial_design or initial_configurations; but not both" ) init_design_def_kwargs = { "cs": scenario.cs, # type: ignore[attr-defined] # noqa F821 "traj_logger": traj_logger, "rng": rng, "ta_run_limit": scenario.ta_run_limit, # type: ignore[attr-defined] # noqa F821 "configs": initial_configurations, "n_configs_x_params": 0, "max_config_fracs": 0.0, } if initial_design_kwargs is not None: init_design_def_kwargs.update(initial_design_kwargs) if initial_configurations is not None: initial_design_instance = InitialDesign(**init_design_def_kwargs) elif initial_design is None: if scenario.initial_incumbent == "DEFAULT": # type: ignore[attr-defined] # noqa F821 init_design_def_kwargs["max_config_fracs"] = 0.0 initial_design_instance = DefaultConfiguration( **init_design_def_kwargs) elif scenario.initial_incumbent == "RANDOM": # type: ignore[attr-defined] # noqa F821 init_design_def_kwargs["max_config_fracs"] = 0.0 initial_design_instance = RandomConfigurations( **init_design_def_kwargs) elif scenario.initial_incumbent == "LHD": # type: ignore[attr-defined] # noqa F821 initial_design_instance = LHDesign(**init_design_def_kwargs) elif scenario.initial_incumbent == "FACTORIAL": # type: ignore[attr-defined] # noqa F821 initial_design_instance = FactorialInitialDesign( **init_design_def_kwargs) elif scenario.initial_incumbent == "SOBOL": # type: ignore[attr-defined] # noqa F821 initial_design_instance = SobolDesign(**init_design_def_kwargs) else: raise ValueError("Don't know what kind of initial_incumbent " "'%s' is" % scenario.initial_incumbent # type: ignore ) # type: ignore[attr-defined] # noqa F821 elif inspect.isclass(initial_design): initial_design_instance = initial_design(**init_design_def_kwargs) else: raise TypeError( "Argument initial_design must be None or an object implementing the InitialDesign, but is '%s'" % type(initial_design)) # if we log the performance data, # the RFRImputator will already get # log transform data from the runhistory if scenario.transform_y in [ "LOG", "LOGS" ]: # type: ignore[attr-defined] # noqa F821 cutoff = np.log(np.nanmin([ np.inf, np.float_(scenario.cutoff) ])) # type: ignore[attr-defined] # noqa F821 threshold = cutoff + np.log( scenario.par_factor) # type: ignore[attr-defined] # noqa F821 else: cutoff = np.nanmin([np.inf, np.float_(scenario.cutoff) ]) # type: ignore[attr-defined] # noqa F821 threshold = cutoff * scenario.par_factor # type: ignore[attr-defined] # noqa F821 num_params = len(scenario.cs.get_hyperparameters() ) # type: ignore[attr-defined] # noqa F821 imputor = RFRImputator( rng=rng, cutoff=cutoff, threshold=threshold, model=model_instance, change_threshold=0.01, max_iter=2, ) r2e_def_kwargs = { "scenario": scenario, "num_params": num_params, "success_states": [ StatusType.SUCCESS, ], "impute_censored_data": True, "impute_state": [ StatusType.CAPPED, ], "imputor": imputor, "scale_perc": 5, } # TODO: consider other sorts of multi-objective algorithms if isinstance(multi_objective_algorithm_instance, AggregationStrategy): r2e_def_kwargs.update({ "multi_objective_algorithm": multi_objective_algorithm_instance }) if scenario.run_obj == "quality": r2e_def_kwargs.update({ "success_states": [ StatusType.SUCCESS, StatusType.CRASHED, StatusType.MEMOUT, ], "impute_censored_data": False, "impute_state": None, }) if (isinstance(intensifier_instance, (SuccessiveHalving, Hyperband)) and scenario.run_obj == "quality"): r2e_def_kwargs.update({ "success_states": [ StatusType.SUCCESS, StatusType.CRASHED, StatusType.MEMOUT, StatusType.DONOTADVANCE, ], "consider_for_higher_budgets_state": [ StatusType.DONOTADVANCE, StatusType.TIMEOUT, StatusType.CRASHED, StatusType.MEMOUT, ], }) if runhistory2epm_kwargs is not None: r2e_def_kwargs.update(runhistory2epm_kwargs) if runhistory2epm is None: if scenario.run_obj == "runtime": rh2epm = RunHistory2EPM4LogCost( **r2e_def_kwargs # type: ignore ) # type: ignore[arg-type] # noqa F821 # type: AbstractRunHistory2EPM elif scenario.run_obj == "quality": if scenario.transform_y == "NONE": # type: ignore[attr-defined] # noqa F821 rh2epm = RunHistory2EPM4Cost( **r2e_def_kwargs) # type: ignore # noqa F821 elif scenario.transform_y == "LOG": # type: ignore[attr-defined] # noqa F821 rh2epm = RunHistory2EPM4LogCost( **r2e_def_kwargs) # type: ignore # noqa F821 elif scenario.transform_y == "LOGS": # type: ignore[attr-defined] # noqa F821 rh2epm = RunHistory2EPM4LogScaledCost( **r2e_def_kwargs) # type: ignore # noqa F821 elif scenario.transform_y == "INVS": # type: ignore[attr-defined] # noqa F821 rh2epm = RunHistory2EPM4InvScaledCost( **r2e_def_kwargs) # type: ignore # noqa F821 else: raise ValueError( "Unknown run objective: %s. Should be either " "quality or runtime." % self.scenario.run_obj # type: ignore # noqa F821 ) elif inspect.isclass(runhistory2epm): rh2epm = runhistory2epm(** r2e_def_kwargs) # type: ignore # noqa F821 else: raise TypeError( "Argument runhistory2epm must be None or an object implementing the RunHistory2EPM, but is '%s'" % type(runhistory2epm)) smbo_args = { "scenario": scenario, "stats": self.stats, "initial_design": initial_design_instance, "runhistory": runhistory, "runhistory2epm": rh2epm, "intensifier": intensifier_instance, "num_run": run_id, "model": model_instance, "acq_optimizer": acquisition_function_optimizer_instance, "acquisition_func": acquisition_function_instance, "rng": rng, "restore_incumbent": restore_incumbent, "random_configuration_chooser": random_configuration_chooser_instance, "tae_runner": tae_runner_instance, } # type: Dict[str, Any] if smbo_class is None: self.solver = SMBO(** smbo_args) # type: ignore[arg-type] # noqa F821 else: self.solver = smbo_class( **smbo_args) # type: ignore[arg-type] # noqa F821
def _component_builder(self, conf:typing.Union[Configuration, dict]) \ -> typing.Tuple[AbstractAcquisitionFunction, AbstractEPM]: """ builds new Acquisition function object and EPM object and returns these Parameters ---------- conf: typing.Union[Configuration, dict] configuration specificing "model" and "acq_func" Returns ------- typing.Tuple[AbstractAcquisitionFunction, AbstractEPM] """ types, bounds = get_types( self.config_space, instance_features=self.scenario.feature_array) if conf["model"] == "RF": model = RandomForestWithInstances( configspace=self.config_space, types=types, bounds=bounds, instance_features=self.scenario.feature_array, seed=self.rng.randint(MAXINT), pca_components=conf.get("pca_dim", self.scenario.PCA_DIM), log_y=conf.get("log_y", self.scenario.transform_y in ["LOG", "LOGS"]), num_trees=conf.get("num_trees", self.scenario.rf_num_trees), do_bootstrapping=conf.get("do_bootstrapping", self.scenario.rf_do_bootstrapping), ratio_features=conf.get("ratio_features", self.scenario.rf_ratio_features), min_samples_split=conf.get("min_samples_split", self.scenario.rf_min_samples_split), min_samples_leaf=conf.get("min_samples_leaf", self.scenario.rf_min_samples_leaf), max_depth=conf.get("max_depth", self.scenario.rf_max_depth), ) elif conf["model"] == "GP": from smac.epm.gp_kernels import ConstantKernel, HammingKernel, WhiteKernel, Matern cov_amp = ConstantKernel( 2.0, constant_value_bounds=(np.exp(-10), np.exp(2)), prior=LognormalPrior(mean=0.0, sigma=1.0, rng=self.rng), ) cont_dims = np.nonzero(types == 0)[0] cat_dims = np.nonzero(types != 0)[0] if len(cont_dims) > 0: exp_kernel = Matern( np.ones([len(cont_dims)]), [(np.exp(-10), np.exp(2)) for _ in range(len(cont_dims))], nu=2.5, operate_on=cont_dims, ) if len(cat_dims) > 0: ham_kernel = HammingKernel( np.ones([len(cat_dims)]), [(np.exp(-10), np.exp(2)) for _ in range(len(cat_dims))], operate_on=cat_dims, ) noise_kernel = WhiteKernel( noise_level=1e-8, noise_level_bounds=(np.exp(-25), np.exp(2)), prior=HorseshoePrior(scale=0.1, rng=self.rng), ) if len(cont_dims) > 0 and len(cat_dims) > 0: # both kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel elif len(cont_dims) > 0 and len(cat_dims) == 0: # only cont kernel = cov_amp * exp_kernel + noise_kernel elif len(cont_dims) == 0 and len(cat_dims) > 0: # only cont kernel = cov_amp * ham_kernel + noise_kernel else: raise ValueError() n_mcmc_walkers = 3 * len(kernel.theta) if n_mcmc_walkers % 2 == 1: n_mcmc_walkers += 1 model = GaussianProcessMCMC( self.config_space, types=types, bounds=bounds, kernel=kernel, n_mcmc_walkers=n_mcmc_walkers, chain_length=250, burnin_steps=250, normalize_y=True, seed=self.rng.randint(low=0, high=10000), ) if conf["acq_func"] == "EI": acq = EI(model=model, par=conf.get("par_ei", 0)) elif conf["acq_func"] == "LCB": acq = LCB(model=model, par=conf.get("par_lcb", 0)) elif conf["acq_func"] == "PI": acq = PI(model=model, par=conf.get("par_pi", 0)) elif conf["acq_func"] == "LogEI": # par value should be in log-space acq = LogEI(model=model, par=conf.get("par_logei", 0)) return acq, model