def test_multiple_budgets(self): rh = RunHistory() cs = get_config_space() config1 = Configuration(cs, values={"a": 1, "b": 2}) rh.add( config=config1, cost=[10, 50], time=20, status=StatusType.SUCCESS, instance_id=1, seed=1, budget=1, ) self.assertEqual(rh.get_cost(config1), 1.0) # Only the higher budget gets included in the config cost # However, we expect that the bounds are changed rh.add( config=config1, cost=[20, 25], time=25, status=StatusType.SUCCESS, instance_id=1, seed=1, budget=5, ) self.assertEqual(rh.get_cost(config1), 0.5)
def test_full_update(self): rh = RunHistory() cs = get_config_space() config1 = Configuration(cs, values={'a': 1, 'b': 2}) config2 = Configuration(cs, values={'a': 1, 'b': 3}) rh.add(config=config1, cost=10, time=20, status=StatusType.SUCCESS, instance_id=1, seed=1) rh.add(config=config2, cost=10, time=20, status=StatusType.SUCCESS, instance_id=1, seed=1) rh.add(config=config2, cost=20, time=20, status=StatusType.SUCCESS, instance_id=2, seed=2) cost_config2 = rh.get_cost(config2) rh.compute_all_costs() updated_cost_config2 = rh.get_cost(config2) self.assertTrue(cost_config2 == updated_cost_config2) rh.compute_all_costs(instances=[2]) updated_cost_config2 = rh.get_cost(config2) self.assertTrue(cost_config2 != updated_cost_config2) self.assertTrue(updated_cost_config2 == 20)
def test_incremental_update(self): rh = RunHistory() cs = get_config_space() config1 = Configuration(cs, values={"a": 1, "b": 2}) rh.add( config=config1, cost=10, time=20, status=StatusType.SUCCESS, instance_id=1, seed=1, ) self.assertEqual(rh.get_cost(config1), 10) rh.add( config=config1, cost=20, time=20, status=StatusType.SUCCESS, instance_id=2, seed=1, ) self.assertEqual(rh.get_cost(config1), 15)
def test_multiple_budgets(self): rh = RunHistory() cs = get_config_space() config1 = Configuration(cs, values={'a': 1, 'b': 2}) rh.add(config=config1, cost=10, time=20, status=StatusType.SUCCESS, instance_id=1, seed=1, budget=1) self.assertEqual(rh.get_cost(config1), 10) # only the higher budget gets included in the config cost rh.add(config=config1, cost=20, time=20, status=StatusType.SUCCESS, instance_id=1, seed=1, budget=2) self.assertEqual(rh.get_cost(config1), 20) self.assertEqual(rh.get_min_cost(config1), 10)
def test_full_update(self): rh = RunHistory(overwrite_existing_runs=True) cs = get_config_space() config1 = Configuration(cs, values={"a": 1, "b": 2}) config2 = Configuration(cs, values={"a": 1, "b": 3}) rh.add( config=config1, cost=[10, 40], time=20, status=StatusType.SUCCESS, instance_id=1, seed=1, ) rh.add( config=config1, cost=[0, 100], time=20, status=StatusType.SUCCESS, instance_id=2, seed=2, ) rh.add( config=config2, cost=[10, 40], time=20, status=StatusType.SUCCESS, instance_id=1, seed=1, ) rh.add( config=config2, cost=[20, 80], time=20, status=StatusType.SUCCESS, instance_id=2, seed=2, ) cost_config2 = rh.get_cost(config2) rh.compute_all_costs() updated_cost_config2 = rh.get_cost(config2) self.assertEqual(cost_config2, updated_cost_config2) rh.compute_all_costs(instances=[2]) updated_cost_config2 = rh.get_cost(config2) self.assertAlmostEqual(updated_cost_config2, 0.833, places=3)
def test_multi_config_design(self): stats = Stats(scenario=self.scenario) stats.start_timing() self.ta.stats = stats tj = TrajLogger(output_dir=None, stats=stats) rh = RunHistory(aggregate_func=average_cost) self.ta.runhistory = rh rng = np.random.RandomState(seed=12345) intensifier = Intensifier(tae_runner=self.ta, stats=stats, traj_logger=tj, rng=rng, instances=[None], run_obj_time=False) configs = [ Configuration(configuration_space=self.cs, values={"x1": 4}), Configuration(configuration_space=self.cs, values={"x1": 2}) ] dc = MultiConfigInitialDesign(tae_runner=self.ta, scenario=self.scenario, stats=stats, traj_logger=tj, runhistory=rh, rng=rng, configs=configs, intensifier=intensifier, aggregate_func=average_cost) inc = dc.run() self.assertTrue(stats.ta_runs == 2) self.assertTrue(len(rh.data) == 2) self.assertTrue(rh.get_cost(inc) == 4)
def process_results(self, run_info: RunInfo, incumbent: typing.Optional[Configuration], run_history: RunHistory, time_bound: float, result: RunValue, log_traj: bool = True, ) -> \ typing.Tuple[Configuration, float]: """ The intensifier stage will be updated based on the results/status of a configuration execution. Also, a incumbent will be determined. Parameters ---------- run_info : RunInfo A RunInfo containing the configuration that was evaluated incumbent : typing.Optional[Configuration] Best configuration seen so far run_history : RunHistory stores all runs we ran so far if False, an evaluated configuration will not be generated again time_bound : float time in [sec] available to perform intensify result: RunValue Contain the result (status and other methadata) of exercising a challenger/incumbent. log_traj: bool Whether to log changes of incumbents in trajectory Returns ------- incumbent: Configuration() current (maybe new) incumbent configuration inc_perf: float empirical performance of incumbent configuration """ # Mark the fact that we processed this configuration self.run_tracker[(run_info.config, run_info.instance, run_info.seed, run_info.budget)] = True # If The incumbent is None we use the challenger if not incumbent: self.logger.info( "First run, no incumbent provided; challenger is assumed to be the incumbent" ) incumbent = run_info.config self.num_run += 1 incumbent = self._compare_configs(challenger=run_info.config, incumbent=incumbent, run_history=run_history, log_traj=log_traj) # get incumbent cost inc_perf = run_history.get_cost(incumbent) return incumbent, inc_perf
def test_incremental_update(self): rh = RunHistory(aggregate_func=average_cost) cs = get_config_space() config1 = Configuration(cs, values={'a': 1, 'b': 2}) rh.add(config=config1, cost=10, time=20, status=StatusType.SUCCESS, instance_id=1, seed=1) self.assertTrue(rh.get_cost(config1) == 10) rh.add(config=config1, cost=20, time=20, status=StatusType.SUCCESS, instance_id=2, seed=1) self.assertTrue(rh.get_cost(config1) == 15)
def test_merge_foreign_data(self): ''' test smac.utils.merge_foreign_data ''' scenario = Scenario(self.test_scenario_dict) scenario_2 = Scenario(self.test_scenario_dict) scenario_2.feature_dict = {"inst_new": [4]} # init cs cs = ConfigurationSpace() cs.add_hyperparameter(UniformIntegerHyperparameter(name='a', lower=0, upper=100)) cs.add_hyperparameter(UniformIntegerHyperparameter(name='b', lower=0, upper=100)) # build runhistory rh_merge = RunHistory() config = Configuration(cs, values={'a': 1, 'b': 2}) rh_merge.add(config=config, instance_id="inst_new", cost=10, time=20, status=StatusType.SUCCESS, seed=None, additional_info=None) # "d" is an instance in <scenario> rh_merge.add(config=config, instance_id="d", cost=5, time=20, status=StatusType.SUCCESS, seed=None, additional_info=None) # build empty rh rh_base = RunHistory() merge_foreign_data(scenario=scenario, runhistory=rh_base, in_scenario_list=[scenario_2], in_runhistory_list=[rh_merge]) # both runs should be in the runhistory # but we should not use the data to update the cost of config self.assertTrue(len(rh_base.data) == 2) self.assertTrue(np.isnan(rh_base.get_cost(config))) # we should not get direct access to external run data runs = rh_base.get_runs_for_config(config, only_max_observed_budget=True) self.assertTrue(len(runs) == 0) rh_merge.add(config=config, instance_id="inst_new_2", cost=10, time=20, status=StatusType.SUCCESS, seed=None, additional_info=None) self.assertRaises(ValueError, merge_foreign_data, **{ "scenario": scenario, "runhistory": rh_base, "in_scenario_list": [scenario_2], "in_runhistory_list": [rh_merge]})
def test_incremental_update(self): rh = RunHistory() cs = get_config_space() config1 = Configuration(cs, values={"a": 1, "b": 2}) rh.add( config=config1, cost=[10, 100], time=20, status=StatusType.SUCCESS, instance_id=1, seed=1, ) self.assertEqual(rh.get_cost(config1), 1.0) rh.add( config=config1, cost=[20, 50], time=20, status=StatusType.SUCCESS, instance_id=2, seed=1, ) # We except 0.75 because of moving average # First we have 1 and then 0.5, the moving average is then 0.75 self.assertEqual(rh.get_cost(config1), 0.75) rh.add( config=config1, cost=[0, 100], time=20, status=StatusType.SUCCESS, instance_id=3, seed=1, ) self.assertAlmostEqual(rh.get_cost(config1), 0.694, places=3)
def _process_inc_run( self, incumbent: Configuration, run_history: RunHistory, log_traj: bool = True, ) -> None: """Method to process the results of a challenger that races an incumbent Parameters ---------- incumbent: Configuration Either challenger or incumbent run_history : RunHistory stores all runs we ran so far log_traj: bool Whether to log changes of incumbents in trajectory """ # output estimated performance of incumbent inc_runs = run_history.get_runs_for_config( incumbent, only_max_observed_budget=True) inc_perf = run_history.get_cost(incumbent) format_value = format_array(inc_perf) self.logger.info( f"Updated estimated cost of incumbent on {len(inc_runs)} runs: {format_value}" ) # if running first configuration, go to next stage after 1st run if self.stage in [ IntensifierStage.RUN_FIRST_CONFIG, IntensifierStage.PROCESS_FIRST_CONFIG_RUN ]: self.stage = IntensifierStage.RUN_INCUMBENT self._next_iteration() else: # Termination condition; after each run, this checks # whether further runs are necessary due to minR if len(inc_runs) >= self.minR or len(inc_runs) >= self.maxR: self.stage = IntensifierStage.RUN_CHALLENGER else: self.stage = IntensifierStage.RUN_INCUMBENT self._compare_configs(incumbent=incumbent, challenger=incumbent, run_history=run_history, log_traj=log_traj)
def _top_k(self, configs: typing.List[Configuration], run_history: RunHistory, k: int) -> typing.List[Configuration]: """ Selects the top 'k' configurations from the given list based on their performance. This retrieves the performance for each configuration from the runhistory and checks that the highest budget they've been evaluated on is the same for each of the configurations. Parameters ---------- configs: typing.List[Configuration] list of configurations to filter from run_history: smac.runhistory.runhistory.RunHistory stores all runs we ran so far k: int number of configurations to select Returns ------- typing.List[Configuration] top challenger configurations, sorted in increasing costs """ # extracting costs for each given configuration config_costs = {} # sample list instance-seed-budget key to act as base run_key = run_history.get_runs_for_config( configs[0], only_max_observed_budget=True) for c in configs: # ensuring that all configurations being compared are run on the same set of instance, seed & budget cur_run_key = run_history.get_runs_for_config( c, only_max_observed_budget=True) if cur_run_key != run_key: raise ValueError( 'Cannot compare configs that were run on different instances-seeds-budgets: %s vs %s' % (run_key, cur_run_key)) config_costs[c] = run_history.get_cost(c) configs_sorted = sorted(config_costs, key=config_costs.get) # select top configurations only top_configs = configs_sorted[:k] return top_configs
def run_smbo(self, max_iters=1000): global evaluator # == first things first: load the datamanager self.reset_data_manager() # == Initialize SMBO stuff # first create a scenario seed = self.seed # TODO num_params = len(self.config_space.get_hyperparameters()) # allocate a run history run_history = RunHistory() meta_runhistory = RunHistory() meta_runs_dataset_indices = {} num_run = self.start_num_run instance_id = self.dataset_name + SENTINEL # == Train on subset # before doing anything, let us run the default_cfg # on a subset of the available data to ensure that # we at least have some models # we will try three different ratios of decreasing magnitude # in the hope that at least on the last one we will be able # to get a model n_data = self.datamanager.data['X_train'].shape[0] subset_ratio = 10000. / n_data if subset_ratio >= 0.5: subset_ratio = 0.33 subset_ratios = [subset_ratio, subset_ratio * 0.10] else: subset_ratios = [subset_ratio, 500. / n_data] self.logger.info("Training default configurations on a subset of " "%d/%d data points." % (int(n_data * subset_ratio), n_data)) # the time limit for these function evaluations is rigorously # set to only 1/2 of a full function evaluation subset_time_limit = max(5, int(self.func_eval_time_limit / 2)) # the configs we want to run on the data subset are: # 1) the default configs # 2) a set of configs we selected for training on a subset subset_configs = [self.config_space.get_default_configuration()] \ + self.collect_additional_subset_defaults() subset_config_succesful = [False] * len(subset_configs) for subset_config_id, next_config in enumerate(subset_configs): for i, ratio in enumerate(subset_ratios): self.reset_data_manager() n_data_subsample = int(n_data * ratio) # run the config, but throw away the result afterwards # since this cfg was evaluated only on a subset # and we don't want to confuse SMAC self.logger.info( "Starting to evaluate %d on SUBSET " "with size %d and time limit %ds.", num_run, n_data_subsample, subset_time_limit) self.logger.info(next_config) _info = eval_with_limits(self.datamanager, self.tmp_dir, next_config, seed, num_run, self.resampling_strategy, self.resampling_strategy_args, self.memory_limit, subset_time_limit, n_data_subsample) (duration, result, _, additional_run_info, status) = _info self.logger.info( "Finished evaluating %d. configuration on SUBSET. " "Duration %f; loss %f; status %s; additional run " "info: %s ", num_run, duration, result, str(status), additional_run_info) num_run += 1 if i < len(subset_ratios) - 1: if status != StatusType.SUCCESS: # Do not increase num_run here, because we will try # the same configuration with less data self.logger.info( "A CONFIG did not finish " " for subset ratio %f -> going smaller", ratio) continue else: self.logger.info( "Finished SUBSET training sucessfully " "with ratio %f", ratio) subset_config_succesful[subset_config_id] = True break else: if status != StatusType.SUCCESS: self.logger.info( "A CONFIG did not finish " " for subset ratio %f.", ratio) continue else: self.logger.info( "Finished SUBSET training sucessfully " "with ratio %f", ratio) subset_config_succesful[subset_config_id] = True break # Use the first non-failing configuration from the subsets as the new # default configuration -> this guards us against the random forest # failing on large, sparse datasets default_cfg = None for subset_config_id, next_config in enumerate(subset_configs): if subset_config_succesful[subset_config_id]: default_cfg = next_config break if default_cfg is None: default_cfg = self.config_space.get_default_configuration() # == METALEARNING suggestions # we start by evaluating the defaults on the full dataset again # and add the suggestions from metalearning behind it if self.metadata_directory is None: metalearning_directory = os.path.dirname( autosklearn.metalearning.__file__) # There is no multilabel data in OpenML if self.task == MULTILABEL_CLASSIFICATION: meta_task = BINARY_CLASSIFICATION else: meta_task = self.task metadata_directory = os.path.join( metalearning_directory, 'files', '%s_%s_%s' % (METRIC_TO_STRING[self.metric], TASK_TYPES_TO_STRING[meta_task], 'sparse' if self.datamanager.info['is_sparse'] else 'dense')) self.metadata_directory = metadata_directory self.logger.info('Metadata directory: %s', self.metadata_directory) meta_base = MetaBase(self.config_space, self.metadata_directory) metafeature_calculation_time_limit = int(self.total_walltime_limit / 4) metafeature_calculation_start_time = time.time() meta_features = self._calculate_metafeatures_with_limits( metafeature_calculation_time_limit) metafeature_calculation_end_time = time.time() metafeature_calculation_time_limit = \ metafeature_calculation_time_limit - ( metafeature_calculation_end_time - metafeature_calculation_start_time) if metafeature_calculation_time_limit < 1: self.logger.warning( 'Time limit for metafeature calculation less ' 'than 1 seconds (%f). Skipping calculation ' 'of metafeatures for encoded dataset.', metafeature_calculation_time_limit) meta_features_encoded = None else: self.datamanager.perform1HotEncoding() meta_features_encoded = \ self._calculate_metafeatures_encoded_with_limits( metafeature_calculation_time_limit) # In case there is a problem calculating the encoded meta-features if meta_features is None: if meta_features_encoded is not None: meta_features = meta_features_encoded else: if meta_features_encoded is not None: meta_features.metafeature_values.update( meta_features_encoded.metafeature_values) if meta_features is not None: meta_base.add_dataset(instance_id, meta_features) # Do mean imputation of the meta-features - should be done specific # for each prediction model! all_metafeatures = meta_base.get_metafeatures( features=list(meta_features.keys())) all_metafeatures.fillna(all_metafeatures.mean(), inplace=True) metalearning_configurations = self.collect_metalearning_suggestions( meta_base) if metalearning_configurations is None: metalearning_configurations = [] self.reset_data_manager() self.logger.info('%s', meta_features) # Convert meta-features into a dictionary because the scenario # expects a dictionary meta_features_dict = {} for dataset, series in all_metafeatures.iterrows(): meta_features_dict[dataset] = series.values meta_features_list = [] for meta_feature_name in all_metafeatures.columns: meta_features_list.append( meta_features[meta_feature_name].value) meta_features_list = np.array(meta_features_list).reshape((1, -1)) self.logger.info(list(meta_features_dict.keys())) meta_runs = meta_base.get_all_runs(METRIC_TO_STRING[self.metric]) meta_runs_index = 0 try: meta_durations = meta_base.get_all_runs('runtime') read_runtime_data = True except KeyError: read_runtime_data = False self.logger.critical('Cannot read runtime data.') if self.acquisition_function == 'EIPS': self.logger.critical( 'Reverting to acquisition function EI!') self.acquisition_function = 'EI' for meta_dataset in meta_runs.index: meta_dataset_start_index = meta_runs_index for meta_configuration in meta_runs.columns: if np.isfinite(meta_runs.loc[meta_dataset, meta_configuration]): try: config = meta_base.get_configuration_from_algorithm_index( meta_configuration) cost = meta_runs.loc[meta_dataset, meta_configuration] if read_runtime_data: runtime = meta_durations.loc[ meta_dataset, meta_configuration] else: runtime = 1 # TODO read out other status types! meta_runhistory.add(config, cost, runtime, StatusType.SUCCESS, instance_id=meta_dataset) meta_runs_index += 1 except: # TODO maybe add warning pass meta_runs_dataset_indices[meta_dataset] = ( meta_dataset_start_index, meta_runs_index) else: if self.acquisition_function == 'EIPS': self.logger.critical('Reverting to acquisition function EI!') self.acquisition_function = 'EI' meta_features_list = [] meta_features_dict = {} metalearning_configurations = [] self.scenario = AutoMLScenario(self.config_space, self.total_walltime_limit, self.func_eval_time_limit, meta_features_dict, self.tmp_dir, self.shared_mode) types = get_types(self.config_space, self.scenario.feature_array) if self.acquisition_function == 'EI': rh2EPM = RunHistory2EPM4Cost(num_params=num_params, scenario=self.scenario, success_states=None, impute_censored_data=False, impute_state=None) model = RandomForestWithInstances( types, instance_features=meta_features_list, seed=1, num_trees=10) smac = SMBO(self.scenario, model=model, rng=seed) elif self.acquisition_function == 'EIPS': rh2EPM = RunHistory2EPM4EIPS(num_params=num_params, scenario=self.scenario, success_states=None, impute_censored_data=False, impute_state=None) model = UncorrelatedMultiObjectiveRandomForestWithInstances( ['cost', 'runtime'], types, num_trees=10, instance_features=meta_features_list, seed=1) acquisition_function = EIPS(model) smac = SMBO(self.scenario, acquisition_function=acquisition_function, model=model, runhistory2epm=rh2EPM, rng=seed) else: raise ValueError('Unknown acquisition function value %s!' % self.acquisition_function) # Build a runtime model # runtime_rf = RandomForestWithInstances(types, # instance_features=meta_features_list, # seed=1, num_trees=10) # runtime_rh2EPM = RunHistory2EPM4EIPS(num_params=num_params, # scenario=self.scenario, # success_states=None, # impute_censored_data=False, # impute_state=None) # X_runtime, y_runtime = runtime_rh2EPM.transform(meta_runhistory) # runtime_rf.train(X_runtime, y_runtime[:, 1].flatten()) X_meta, Y_meta = rh2EPM.transform(meta_runhistory) # Transform Y_meta on a per-dataset base for meta_dataset in meta_runs_dataset_indices: start_index, end_index = meta_runs_dataset_indices[meta_dataset] end_index += 1 # Python indexing Y_meta[start_index:end_index, 0]\ [Y_meta[start_index:end_index, 0] >2.0] = 2.0 dataset_minimum = np.min(Y_meta[start_index:end_index, 0]) Y_meta[start_index:end_index, 0] = 1 - ((1. - Y_meta[start_index:end_index, 0]) / (1. - dataset_minimum)) Y_meta[start_index:end_index, 0]\ [Y_meta[start_index:end_index, 0] > 2] = 2 # == first, evaluate all metelearning and default configurations for i, next_config in enumerate( ([default_cfg] + metalearning_configurations)): # Do not evaluate default configurations more than once if i >= len([default_cfg]) and next_config in [default_cfg]: continue config_name = 'meta-learning' if i >= len([default_cfg]) \ else 'default' self.logger.info( "Starting to evaluate %d. configuration " "(%s configuration) with time limit %ds.", num_run, config_name, self.func_eval_time_limit) self.logger.info(next_config) self.reset_data_manager() info = eval_with_limits(self.datamanager, self.tmp_dir, next_config, seed, num_run, self.resampling_strategy, self.resampling_strategy_args, self.memory_limit, self.func_eval_time_limit) (duration, result, _, additional_run_info, status) = info run_history.add(config=next_config, cost=result, time=duration, status=status, instance_id=instance_id, seed=seed) run_history.update_cost(next_config, result) self.logger.info( "Finished evaluating %d. configuration. " "Duration %f; loss %f; status %s; additional run " "info: %s ", num_run, duration, result, str(status), additional_run_info) num_run += 1 if smac.incumbent is None: smac.incumbent = next_config elif result < run_history.get_cost(smac.incumbent): smac.incumbent = next_config if self.scenario.shared_model: pSMAC.write(run_history=run_history, output_directory=self.scenario.output_dir, num_run=self.seed) # == after metalearning run SMAC loop smac.runhistory = run_history smac_iter = 0 finished = False while not finished: if self.scenario.shared_model: pSMAC.read(run_history=run_history, output_directory=self.scenario.output_dir, configuration_space=self.config_space, logger=self.logger) next_configs = [] time_for_choose_next = -1 try: X_cfg, Y_cfg = rh2EPM.transform(run_history) if not run_history.empty(): # Update costs by normalization dataset_minimum = np.min(Y_cfg[:, 0]) Y_cfg[:, 0] = 1 - ((1. - Y_cfg[:, 0]) / (1. - dataset_minimum)) Y_cfg[:, 0][Y_cfg[:, 0] > 2] = 2 if len(X_meta) > 0 and len(X_cfg) > 0: pass #X_cfg = np.concatenate((X_meta, X_cfg)) #Y_cfg = np.concatenate((Y_meta, Y_cfg)) elif len(X_meta) > 0: X_cfg = X_meta.copy() Y_cfg = Y_meta.copy() elif len(X_cfg) > 0: X_cfg = X_cfg.copy() Y_cfg = Y_cfg.copy() else: raise ValueError( 'No training data for SMAC random forest!') self.logger.info('Using %d training points for SMAC.' % X_cfg.shape[0]) choose_next_start_time = time.time() next_configs_tmp = smac.choose_next( X_cfg, Y_cfg, num_interleaved_random=110, num_configurations_by_local_search=10, num_configurations_by_random_search_sorted=100) time_for_choose_next = time.time() - choose_next_start_time self.logger.info('Used %g seconds to find next ' 'configurations' % (time_for_choose_next)) next_configs.extend(next_configs_tmp) # TODO put Exception here! except Exception as e: self.logger.error(e) self.logger.error("Error in getting next configurations " "with SMAC. Using random configuration!") next_config = self.config_space.sample_configuration() next_configs.append(next_config) models_fitted_this_iteration = 0 start_time_this_iteration = time.time() for next_config in next_configs: x_runtime = impute_inactive_values(next_config) x_runtime = impute_inactive_values(x_runtime).get_array() # predicted_runtime = runtime_rf.predict_marginalized_over_instances( # x_runtime.reshape((1, -1))) # predicted_runtime = np.exp(predicted_runtime[0][0][0]) - 1 self.logger.info( "Starting to evaluate %d. configuration (from " "SMAC) with time limit %ds.", num_run, self.func_eval_time_limit) self.logger.info(next_config) self.reset_data_manager() info = eval_with_limits(self.datamanager, self.tmp_dir, next_config, seed, num_run, self.resampling_strategy, self.resampling_strategy_args, self.memory_limit, self.func_eval_time_limit) (duration, result, _, additional_run_info, status) = info run_history.add(config=next_config, cost=result, time=duration, status=status, instance_id=instance_id, seed=seed) run_history.update_cost(next_config, result) #self.logger.info('Predicted runtime %g, true runtime %g', # predicted_runtime, duration) # TODO add unittest to make sure everything works fine and # this does not get outdated! if smac.incumbent is None: smac.incumbent = next_config elif result < run_history.get_cost(smac.incumbent): smac.incumbent = next_config self.logger.info( "Finished evaluating %d. configuration. " "Duration: %f; loss: %f; status %s; additional " "run info: %s ", num_run, duration, result, str(status), additional_run_info) smac_iter += 1 num_run += 1 models_fitted_this_iteration += 1 time_used_this_iteration = time.time( ) - start_time_this_iteration if models_fitted_this_iteration >= 2 and \ time_for_choose_next > 0 and \ time_used_this_iteration > time_for_choose_next: break elif time_for_choose_next <= 0 and \ models_fitted_this_iteration >= 1: break elif models_fitted_this_iteration >= 50: break if max_iters is not None: finished = (smac_iter < max_iters) if self.scenario.shared_model: pSMAC.write(run_history=run_history, output_directory=self.scenario.output_dir, num_run=self.seed)
def run_smbo(self, max_iters=1000): global evaluator # == first things first: load the datamanager self.reset_data_manager() # == Initialize SMBO stuff # first create a scenario seed = self.seed # TODO num_params = len(self.config_space.get_hyperparameters()) # allocate a run history run_history = RunHistory() meta_runhistory = RunHistory() meta_runs_dataset_indices = {} num_run = self.start_num_run instance_id = self.dataset_name + SENTINEL # == Train on subset # before doing anything, let us run the default_cfg # on a subset of the available data to ensure that # we at least have some models # we will try three different ratios of decreasing magnitude # in the hope that at least on the last one we will be able # to get a model n_data = self.datamanager.data['X_train'].shape[0] subset_ratio = 10000. / n_data if subset_ratio >= 0.5: subset_ratio = 0.33 subset_ratios = [subset_ratio, subset_ratio * 0.10] else: subset_ratios = [subset_ratio, 500. / n_data] self.logger.info("Training default configurations on a subset of " "%d/%d data points." % (int(n_data * subset_ratio), n_data)) # the time limit for these function evaluations is rigorously # set to only 1/2 of a full function evaluation subset_time_limit = max(5, int(self.func_eval_time_limit / 2)) # the configs we want to run on the data subset are: # 1) the default configs # 2) a set of configs we selected for training on a subset subset_configs = [self.config_space.get_default_configuration()] \ + self.collect_additional_subset_defaults() subset_config_succesful = [False] * len(subset_configs) for subset_config_id, next_config in enumerate(subset_configs): for i, ratio in enumerate(subset_ratios): self.reset_data_manager() n_data_subsample = int(n_data * ratio) # run the config, but throw away the result afterwards # since this cfg was evaluated only on a subset # and we don't want to confuse SMAC self.logger.info("Starting to evaluate %d on SUBSET " "with size %d and time limit %ds.", num_run, n_data_subsample, subset_time_limit) self.logger.info(next_config) _info = eval_with_limits( self.datamanager, self.tmp_dir, next_config, seed, num_run, self.resampling_strategy, self.resampling_strategy_args, self.memory_limit, subset_time_limit, n_data_subsample) (duration, result, _, additional_run_info, status) = _info self.logger.info("Finished evaluating %d. configuration on SUBSET. " "Duration %f; loss %f; status %s; additional run " "info: %s ", num_run, duration, result, str(status), additional_run_info) num_run += 1 if i < len(subset_ratios) - 1: if status != StatusType.SUCCESS: # Do not increase num_run here, because we will try # the same configuration with less data self.logger.info("A CONFIG did not finish " " for subset ratio %f -> going smaller", ratio) continue else: self.logger.info("Finished SUBSET training sucessfully " "with ratio %f", ratio) subset_config_succesful[subset_config_id] = True break else: if status != StatusType.SUCCESS: self.logger.info("A CONFIG did not finish " " for subset ratio %f.", ratio) continue else: self.logger.info("Finished SUBSET training sucessfully " "with ratio %f", ratio) subset_config_succesful[subset_config_id] = True break # Use the first non-failing configuration from the subsets as the new # default configuration -> this guards us against the random forest # failing on large, sparse datasets default_cfg = None for subset_config_id, next_config in enumerate(subset_configs): if subset_config_succesful[subset_config_id]: default_cfg = next_config break if default_cfg is None: default_cfg = self.config_space.get_default_configuration() # == METALEARNING suggestions # we start by evaluating the defaults on the full dataset again # and add the suggestions from metalearning behind it if self.metadata_directory is None: metalearning_directory = os.path.dirname( autosklearn.metalearning.__file__) # There is no multilabel data in OpenML if self.task == MULTILABEL_CLASSIFICATION: meta_task = BINARY_CLASSIFICATION else: meta_task = self.task metadata_directory = os.path.join( metalearning_directory, 'files', '%s_%s_%s' % (METRIC_TO_STRING[self.metric], TASK_TYPES_TO_STRING[meta_task], 'sparse' if self.datamanager.info['is_sparse'] else 'dense')) self.metadata_directory = metadata_directory self.logger.info('Metadata directory: %s', self.metadata_directory) meta_base = MetaBase(self.config_space, self.metadata_directory) metafeature_calculation_time_limit = int( self.total_walltime_limit / 4) metafeature_calculation_start_time = time.time() meta_features = self._calculate_metafeatures_with_limits( metafeature_calculation_time_limit) metafeature_calculation_end_time = time.time() metafeature_calculation_time_limit = \ metafeature_calculation_time_limit - ( metafeature_calculation_end_time - metafeature_calculation_start_time) if metafeature_calculation_time_limit < 1: self.logger.warning('Time limit for metafeature calculation less ' 'than 1 seconds (%f). Skipping calculation ' 'of metafeatures for encoded dataset.', metafeature_calculation_time_limit) meta_features_encoded = None else: self.datamanager.perform1HotEncoding() meta_features_encoded = \ self._calculate_metafeatures_encoded_with_limits( metafeature_calculation_time_limit) # In case there is a problem calculating the encoded meta-features if meta_features is None: if meta_features_encoded is not None: meta_features = meta_features_encoded else: if meta_features_encoded is not None: meta_features.metafeature_values.update( meta_features_encoded.metafeature_values) if meta_features is not None: meta_base.add_dataset(instance_id, meta_features) # Do mean imputation of the meta-features - should be done specific # for each prediction model! all_metafeatures = meta_base.get_metafeatures( features=list(meta_features.keys())) all_metafeatures.fillna(all_metafeatures.mean(), inplace=True) metalearning_configurations = self.collect_metalearning_suggestions( meta_base) if metalearning_configurations is None: metalearning_configurations = [] self.reset_data_manager() self.logger.info('%s', meta_features) # Convert meta-features into a dictionary because the scenario # expects a dictionary meta_features_dict = {} for dataset, series in all_metafeatures.iterrows(): meta_features_dict[dataset] = series.values meta_features_list = [] for meta_feature_name in all_metafeatures.columns: meta_features_list.append(meta_features[meta_feature_name].value) meta_features_list = np.array(meta_features_list).reshape((1, -1)) self.logger.info(list(meta_features_dict.keys())) meta_runs = meta_base.get_all_runs(METRIC_TO_STRING[self.metric]) meta_runs_index = 0 try: meta_durations = meta_base.get_all_runs('runtime') read_runtime_data = True except KeyError: read_runtime_data = False self.logger.critical('Cannot read runtime data.') if self.acquisition_function == 'EIPS': self.logger.critical('Reverting to acquisition function EI!') self.acquisition_function = 'EI' for meta_dataset in meta_runs.index: meta_dataset_start_index = meta_runs_index for meta_configuration in meta_runs.columns: if np.isfinite(meta_runs.loc[meta_dataset, meta_configuration]): try: config = meta_base.get_configuration_from_algorithm_index( meta_configuration) cost = meta_runs.loc[meta_dataset, meta_configuration] if read_runtime_data: runtime = meta_durations.loc[meta_dataset, meta_configuration] else: runtime = 1 # TODO read out other status types! meta_runhistory.add(config, cost, runtime, StatusType.SUCCESS, instance_id=meta_dataset) meta_runs_index += 1 except: # TODO maybe add warning pass meta_runs_dataset_indices[meta_dataset] = ( meta_dataset_start_index, meta_runs_index) else: if self.acquisition_function == 'EIPS': self.logger.critical('Reverting to acquisition function EI!') self.acquisition_function = 'EI' meta_features_list = [] meta_features_dict = {} metalearning_configurations = [] self.scenario = AutoMLScenario(self.config_space, self.total_walltime_limit, self.func_eval_time_limit, meta_features_dict, self.tmp_dir, self.shared_mode) types = get_types(self.config_space, self.scenario.feature_array) if self.acquisition_function == 'EI': rh2EPM = RunHistory2EPM4Cost(num_params=num_params, scenario=self.scenario, success_states=None, impute_censored_data=False, impute_state=None) model = RandomForestWithInstances(types, instance_features=meta_features_list, seed=1, num_trees=10) smac = SMBO(self.scenario, model=model, rng=seed) elif self.acquisition_function == 'EIPS': rh2EPM = RunHistory2EPM4EIPS(num_params=num_params, scenario=self.scenario, success_states=None, impute_censored_data=False, impute_state=None) model = UncorrelatedMultiObjectiveRandomForestWithInstances( ['cost', 'runtime'], types, num_trees = 10, instance_features=meta_features_list, seed=1) acquisition_function = EIPS(model) smac = SMBO(self.scenario, acquisition_function=acquisition_function, model=model, runhistory2epm=rh2EPM, rng=seed) else: raise ValueError('Unknown acquisition function value %s!' % self.acquisition_function) # Build a runtime model # runtime_rf = RandomForestWithInstances(types, # instance_features=meta_features_list, # seed=1, num_trees=10) # runtime_rh2EPM = RunHistory2EPM4EIPS(num_params=num_params, # scenario=self.scenario, # success_states=None, # impute_censored_data=False, # impute_state=None) # X_runtime, y_runtime = runtime_rh2EPM.transform(meta_runhistory) # runtime_rf.train(X_runtime, y_runtime[:, 1].flatten()) X_meta, Y_meta = rh2EPM.transform(meta_runhistory) # Transform Y_meta on a per-dataset base for meta_dataset in meta_runs_dataset_indices: start_index, end_index = meta_runs_dataset_indices[meta_dataset] end_index += 1 # Python indexing Y_meta[start_index:end_index, 0]\ [Y_meta[start_index:end_index, 0] >2.0] = 2.0 dataset_minimum = np.min(Y_meta[start_index:end_index, 0]) Y_meta[start_index:end_index, 0] = 1 - ( (1. - Y_meta[start_index:end_index, 0]) / (1. - dataset_minimum)) Y_meta[start_index:end_index, 0]\ [Y_meta[start_index:end_index, 0] > 2] = 2 # == first, evaluate all metelearning and default configurations for i, next_config in enumerate(([default_cfg] + metalearning_configurations)): # Do not evaluate default configurations more than once if i >= len([default_cfg]) and next_config in [default_cfg]: continue config_name = 'meta-learning' if i >= len([default_cfg]) \ else 'default' self.logger.info("Starting to evaluate %d. configuration " "(%s configuration) with time limit %ds.", num_run, config_name, self.func_eval_time_limit) self.logger.info(next_config) self.reset_data_manager() info = eval_with_limits(self.datamanager, self.tmp_dir, next_config, seed, num_run, self.resampling_strategy, self.resampling_strategy_args, self.memory_limit, self.func_eval_time_limit) (duration, result, _, additional_run_info, status) = info run_history.add(config=next_config, cost=result, time=duration , status=status, instance_id=instance_id, seed=seed) run_history.update_cost(next_config, result) self.logger.info("Finished evaluating %d. configuration. " "Duration %f; loss %f; status %s; additional run " "info: %s ", num_run, duration, result, str(status), additional_run_info) num_run += 1 if smac.incumbent is None: smac.incumbent = next_config elif result < run_history.get_cost(smac.incumbent): smac.incumbent = next_config if self.scenario.shared_model: pSMAC.write(run_history=run_history, output_directory=self.scenario.output_dir, num_run=self.seed) # == after metalearning run SMAC loop smac.runhistory = run_history smac_iter = 0 finished = False while not finished: if self.scenario.shared_model: pSMAC.read(run_history=run_history, output_directory=self.scenario.output_dir, configuration_space=self.config_space, logger=self.logger) next_configs = [] time_for_choose_next = -1 try: X_cfg, Y_cfg = rh2EPM.transform(run_history) if not run_history.empty(): # Update costs by normalization dataset_minimum = np.min(Y_cfg[:, 0]) Y_cfg[:, 0] = 1 - ((1. - Y_cfg[:, 0]) / (1. - dataset_minimum)) Y_cfg[:, 0][Y_cfg[:, 0] > 2] = 2 if len(X_meta) > 0 and len(X_cfg) > 0: pass #X_cfg = np.concatenate((X_meta, X_cfg)) #Y_cfg = np.concatenate((Y_meta, Y_cfg)) elif len(X_meta) > 0: X_cfg = X_meta.copy() Y_cfg = Y_meta.copy() elif len(X_cfg) > 0: X_cfg = X_cfg.copy() Y_cfg = Y_cfg.copy() else: raise ValueError('No training data for SMAC random forest!') self.logger.info('Using %d training points for SMAC.' % X_cfg.shape[0]) choose_next_start_time = time.time() next_configs_tmp = smac.choose_next(X_cfg, Y_cfg, num_interleaved_random=110, num_configurations_by_local_search=10, num_configurations_by_random_search_sorted=100) time_for_choose_next = time.time() - choose_next_start_time self.logger.info('Used %g seconds to find next ' 'configurations' % (time_for_choose_next)) next_configs.extend(next_configs_tmp) # TODO put Exception here! except Exception as e: self.logger.error(e) self.logger.error("Error in getting next configurations " "with SMAC. Using random configuration!") next_config = self.config_space.sample_configuration() next_configs.append(next_config) models_fitted_this_iteration = 0 start_time_this_iteration = time.time() for next_config in next_configs: x_runtime = impute_inactive_values(next_config) x_runtime = impute_inactive_values(x_runtime).get_array() # predicted_runtime = runtime_rf.predict_marginalized_over_instances( # x_runtime.reshape((1, -1))) # predicted_runtime = np.exp(predicted_runtime[0][0][0]) - 1 self.logger.info("Starting to evaluate %d. configuration (from " "SMAC) with time limit %ds.", num_run, self.func_eval_time_limit) self.logger.info(next_config) self.reset_data_manager() info = eval_with_limits(self.datamanager, self.tmp_dir, next_config, seed, num_run, self.resampling_strategy, self.resampling_strategy_args, self.memory_limit, self.func_eval_time_limit) (duration, result, _, additional_run_info, status) = info run_history.add(config=next_config, cost=result, time=duration , status=status, instance_id=instance_id, seed=seed) run_history.update_cost(next_config, result) #self.logger.info('Predicted runtime %g, true runtime %g', # predicted_runtime, duration) # TODO add unittest to make sure everything works fine and # this does not get outdated! if smac.incumbent is None: smac.incumbent = next_config elif result < run_history.get_cost(smac.incumbent): smac.incumbent = next_config self.logger.info("Finished evaluating %d. configuration. " "Duration: %f; loss: %f; status %s; additional " "run info: %s ", num_run, duration, result, str(status), additional_run_info) smac_iter += 1 num_run += 1 models_fitted_this_iteration += 1 time_used_this_iteration = time.time() - start_time_this_iteration if models_fitted_this_iteration >= 2 and \ time_for_choose_next > 0 and \ time_used_this_iteration > time_for_choose_next: break elif time_for_choose_next <= 0 and \ models_fitted_this_iteration >= 1: break elif models_fitted_this_iteration >= 50: break if max_iters is not None: finished = (smac_iter < max_iters) if self.scenario.shared_model: pSMAC.write(run_history=run_history, output_directory=self.scenario.output_dir, num_run=self.seed)
global_original_rh = RunHistory(average_cost) global_validated_rh = RunHistory(average_cost) global_epm_rh = RunHistory(average_cost) for run in runs: global_original_rh.update(run.original_runhistory, origin=DataOrigin.INTERNAL) global_validated_rh.update(run.original_runhistory, origin=DataOrigin.INTERNAL) if run.validated_runhistory: global_validated_rh.update(run.validated_runhistory, origin=DataOrigin.EXTERNAL_SAME_INSTANCES) global_epm_rh.update(global_validated_rh) runs = sorted(runs, key=lambda run: global_epm_rh.get_cost(run.solver.incumbent)) class TestRunhistory(unittest.TestCase): def test_classification(self): """Function to test, if random and local runhistory created correctly""" # combined = help.combine_runhistories(runs) random, local = help.create_random_runhistories(global_original_rh) random = str(random.data) result = "OrderedDict([(RunKey(config_id=1, instance_id=None, seed=0), RunValue(cost=15.336866, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=2, instance_id=None, seed=0), RunValue(cost=6.042356, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=3, instance_id=None, seed=0), RunValue(cost=76.191931, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=4, instance_id=None, seed=0), RunValue(cost=2.102303, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=5, instance_id=None, seed=0), RunValue(cost=11.997325, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=6, instance_id=None, seed=0), RunValue(cost=27.718052, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=7, instance_id=None, seed=0), RunValue(cost=123.166119, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=8, instance_id=None, seed=0), RunValue(cost=13.556812, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=9, instance_id=None, seed=0), RunValue(cost=3.265477, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=10, instance_id=None, seed=0), RunValue(cost=20.514643, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=11, instance_id=None, seed=0), RunValue(cost=2.522322, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=12, instance_id=None, seed=0), RunValue(cost=20.305772, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=13, instance_id=None, seed=0), RunValue(cost=11.241301, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=14, instance_id=None, seed=0), RunValue(cost=91.710919, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=15, instance_id=None, seed=0), RunValue(cost=20.195214, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=16, instance_id=None, seed=0), RunValue(cost=188.443026, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=17, instance_id=None, seed=0), RunValue(cost=82.295484, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=18, instance_id=None, seed=0), RunValue(cost=31.528635, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=19, instance_id=None, seed=0), RunValue(cost=77.229825, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=20, instance_id=None, seed=0), RunValue(cost=97.872458, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=21, instance_id=None, seed=0), RunValue(cost=140.605172, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=22, instance_id=None, seed=0), RunValue(cost=18.810533, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=23, instance_id=None, seed=0), RunValue(cost=0.933445, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=24, instance_id=None, seed=0), RunValue(cost=14.273374, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={})), (RunKey(config_id=25, instance_id=None, seed=0), RunValue(cost=37.695976, time=-1.0, status=<StatusType.SUCCESS: 1>, additional_info={}))])" # self.assertEqual(random, result) def test_combination(self): """Test if the right combined runhistory is created using random and local runhistory""" # combine_random_local pass
def _plot_parallel_coordinates( self, original_rh: RunHistory, validated_rh: RunHistory, validator: Validator, scenario: Scenario, default: Configuration, incumbent: Configuration, param_imp: Union[None, Dict[str, float]], output_dir: str, cs: ConfigurationSpace, runtime: bool = False, ): """ Parameters: ----------- original_rh: RunHistory runhistory that should contain only runs that were executed during search validated_rh: RunHistory runhistory that may contain as many runs as possible, also external runs. this runhistory will be used to build the EPM validator: Validator validator to be used to estimate costs for configurations scenario: Scenario scenario object to take instances from default, incumbent: Configuration default and incumbent, they will surely be displayed param_imp: Union[None, Dict[str->float] if given, maps parameter-names to importance output_dir: str output directory for plots cs: ConfigurationSpace parameter configuration space to be visualized runtime: boolean runtime will be on logscale """ # Sorting parameters by importance, if possible (choose first executed parameter-importance) method, importance = "", {} if self.pc_sort_by == 'all': self.logger.debug("Sorting by average importance") method = 'average' for m, i in param_imp.items(): if i: for p, imp in i.items(): if p in importance: importance[p].append(imp) else: importance[p] = [imp] importance = {k: sum(v) / len(v) for k, v in importance.items()} elif self.pc_sort_by in param_imp: method, importance = self.pc_sort_by, param_imp[self.pc_sort_by] else: self.logger.debug("%s not evaluated.. choosing at random from: %s", self.pc_sort_by, str(list(param_imp.keys()))) for m, i in param_imp.items(): if i: method, importance = m, i self.logger.debug("Chose %s", method) break hp_names = sorted([hp for hp in cs.get_hyperparameter_names()], key=lambda x: importance.get(x, 0), reverse=True) self.logger.debug("Sorted hp's by method \'%s\': %s", method, str(hp_names)) # To be set self.plots = [] # Define set of configurations (limiting to max and choosing most interesting ones) all_configs = original_rh.get_all_configs() max_runs_epm = self.max_runs_epm # Maximum total number of runs considered for epm to limit maximum possible number configs max_configs = int( max_runs_epm / (len(scenario.train_insts) + len(scenario.test_insts))) if len(all_configs) > max_configs: self.logger.debug( "Limiting number of configs to train epm from %d to %d (based on max runs %d) and choosing " "the ones with the most runs (for parallel coordinates)", len(all_configs), max_configs, max_runs_epm) all_configs = sorted( all_configs, key=lambda c: len(original_rh.get_runs_for_config(c) ))[:max_configs] if not default in all_configs: all_configs = [default] + all_configs if not incumbent in all_configs: all_configs.append(incumbent) # Get costs for those configurations epm_rh = RunHistory(average_cost) epm_rh.update(validated_rh) if scenario.feature_dict: # if instances are available epm_rh.update( timing(validator.validate_epm)(all_configs, 'train+test', 1, runhistory=validated_rh)) config_to_cost = {c: epm_rh.get_cost(c) for c in all_configs} pcp = ParallelCoordinatesPlotter(config_to_cost, output_dir, cs, runtime) try: plots = [ pcp.plot_n_configs( self.n_configs, self.get_params(self.params, importance, hp_names)) ] self.logger.debug("Paths to plot(s): %s", str(plots)) return {'figure': plots} except ValueError as err: self.logger.debug("Error: %s", str(err)) return {'else': str(err)}
class TestIntensify(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.rh = RunHistory() self.cs = get_config_space() self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100}) self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0}) self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100}) self.scen = Scenario({ "cutoff_time": 2, 'cs': self.cs, "run_obj": 'runtime', "output_dir": '' }) self.stats = Stats(scenario=self.scen) self.stats.start_timing() self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) def test_race_challenger(self): """ test _race_challenger without adaptive capping """ def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) intensifier.N = 1 inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.num_run, 1) self.assertEqual(intensifier.num_chall_run, 1) def test_race_challenger_2(self): """ test _race_challenger with adaptive capping """ def target(x): time.sleep(1.5) return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1]) self.rh.add(config=self.config1, cost=.001, time=0.001, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None) intensifier.N = 1 # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc = intensifier._race_challenger( challenger=self.config2, incumbent=self.config1, run_history=self.rh, ) self.assertEqual(inc, self.config1) self.assertEqual(intensifier.num_run, 1) self.assertEqual(intensifier.num_chall_run, 1) def test_race_challenger_3(self): """ test _race_challenger with adaptive capping on a previously capped configuration """ def target(config: Configuration, seed: int, instance: str): if instance == 1: time.sleep(2.1) else: time.sleep(0.6) return (config['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime", par_factor=1) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), cutoff=2, instances=[1]) self.rh.add(config=self.config1, cost=0.5, time=.5, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None) # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) self.assertEqual(inc, self.config1) # further run for incumbent self.rh.add(config=self.config1, cost=2, time=2, status=StatusType.TIMEOUT, instance_id=2, seed=12345, additional_info=None) # give config2 a second chance - now it should run on both instances # run on instance 1 config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) # run on instance 2 config, _ = intensifier.get_next_challenger(challengers=[self.config3], chooser=None) self.assertEqual(config, self.config2) self.assertTrue(intensifier.continue_challenger) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) # the incumbent should still be config1 because # config2 should get on inst 1 a full timeout # such that c(config1) = 1.25 and c(config2) close to 1.3 self.assertEqual(inc, self.config1) # the capped run should not be counted in runs_perf_config self.assertAlmostEqual(self.rh.num_runs_per_config[2], 2) self.assertFalse(intensifier.continue_challenger) self.assertEqual(intensifier.num_run, 3) self.assertEqual(intensifier.num_chall_run, 3) def test_race_challenger_large(self): """ test _race_challenger using solution_quality """ def target(x): return 1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), run_obj_time=False, deterministic=True) for i in range(10): self.rh.add(config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=12345, additional_info=None) intensifier.stage = IntensifierStage.RUN_CHALLENGER # tie on first instances and then challenger should always win # and be returned as inc while True: config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) # stop when challenger evaluation is over if not intensifier.stage == IntensifierStage.RUN_CHALLENGER: break self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True) self.assertEqual(len(runs), 10) self.assertEqual(intensifier.num_run, 10) self.assertEqual(intensifier.num_chall_run, 10) def test_race_challenger_large_blocked_seed(self): """ test _race_challenger whether seeds are blocked for challenger runs """ def target(x): return 1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), run_obj_time=False, deterministic=False) for i in range(10): self.rh.add(config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) intensifier.stage = IntensifierStage.RUN_CHALLENGER # tie on first instances and then challenger should always win # and be returned as inc while True: config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) # stop when challenger evaluation is over if not intensifier.stage == IntensifierStage.RUN_CHALLENGER: break self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True) self.assertEqual(len(runs), 10) seeds = sorted([r.seed for r in runs]) self.assertEqual(seeds, list(range(10)), seeds) self.assertEqual(intensifier.num_run, 10) self.assertEqual(intensifier.num_chall_run, 10) def test_add_inc_run_det(self): """ test _add_inc_run() """ def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) # since we assume deterministic=1, # the second call should not add any more runs # given only one instance intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) # The following two tests evaluate to zero because _next_iteration is triggered by _add_inc_run # as it is the first evaluation of this intensifier self.assertEqual(intensifier.num_run, 0) self.assertEqual(intensifier.num_chall_run, 0) def test_add_inc_run_nondet(self): """ test _add_inc_run() """ def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, runhistory=self.rh, run_obj="solution_quality") intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], deterministic=False) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 2, self.rh.data) runs = self.rh.get_runs_for_config(config=self.config1, only_max_observed_budget=True) # exactly one run on each instance self.assertIn(1, [runs[0].instance, runs[1].instance]) self.assertIn(2, [runs[0].instance, runs[1].instance]) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 3, self.rh.data) self.assertEqual(intensifier.num_run, 2) self.assertEqual(intensifier.num_chall_run, 0) def test_get_next_challenger(self): """ test get_next_challenger() """ intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True) intensifier.stage = IntensifierStage.RUN_CHALLENGER # get a new challenger to evaluate config, new = intensifier.get_next_challenger( challengers=[self.config1, self.config2], chooser=None) self.assertEqual(config, self.config1, intensifier.current_challenger) self.assertEqual(intensifier._chall_indx, 1) self.assertEqual(intensifier.N, 1) self.assertTrue(new) # when already evaluating a challenger, return the same challenger intensifier.to_run = [(1, 1, 0)] config, new = intensifier.get_next_challenger( challengers=[self.config2], chooser=None) self.assertEqual(config, self.config1, intensifier.current_challenger) self.assertEqual(intensifier._chall_indx, 1) self.assertFalse(new) def test_generate_challenger(self): """ test generate_challenger() """ # test get generator from a list of challengers intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=None, rng=np.random.RandomState(12345), instances=[1], deterministic=True) gen = intensifier._generate_challengers( challengers=[self.config1, self.config2], chooser=None) self.assertEqual(next(gen), self.config1) self.assertEqual(next(gen), self.config2) self.assertRaises(StopIteration, next, gen) # test get generator from a chooser - would return only 1 configuration intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=None, rng=np.random.RandomState(12345), instances=[1], deterministic=True) chooser = SMAC4AC(self.scen, rng=1).solver.epm_chooser gen = intensifier._generate_challengers(challengers=None, chooser=chooser) self.assertEqual(next(gen).get_dictionary(), {'a': 24, 'b': 68}) self.assertRaises(StopIteration, next, gen) # when both are none, raise error with self.assertRaisesRegex(ValueError, "No configurations/chooser provided"): intensifier._generate_challengers(challengers=None, chooser=None) def test_eval_challenger_1(self): """ test eval_challenger() - a complete intensification run with a `always_race_against` configuration """ def target(x): if x['a'] == 100: time.sleep(1) return x['a'] taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], run_obj_time=True, cutoff=2, deterministic=False, always_race_against=self.config3, run_limit=1) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # intensification iteration #1 # run first config as incumbent if incumbent is None config, _ = intensifier.get_next_challenger(challengers=[self.config2], chooser=None) self.assertEqual(config, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # eval config 2 (=first run) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=None, run_history=self.rh, ) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual(intensifier.n_iters, 1) # 1 intensification run complete! # intensification iteration #2 # regular intensification begins - run incumbent first config, _ = intensifier.get_next_challenger( challengers=None, # don't need a new list here as old one is cont'd chooser=None) self.assertEqual(config, inc) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(self.stats.ta_runs, 2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(self.stats.inc_changed, 1) # run challenger now that the incumbent has been executed config, _ = intensifier.get_next_challenger(challengers=[self.config1], chooser=None) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(config, self.config1) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # challenger has a better performance, but not run on all instances yet. so incumbent stays the same self.assertEqual(inc, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertTrue(intensifier.continue_challenger) # run challenger again on the other instance config, _ = intensifier.get_next_challenger( challengers=None, # don't need a new list here as old one is cont'd chooser=None) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(config, self.config1) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # challenger better than incumbent in both instances. so incumbent changed self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_BASIS) self.assertFalse(intensifier.continue_challenger) # run basis configuration (`always_race_against`) config, _ = intensifier.get_next_challenger( challengers=None, # don't need a new list here as old one is cont'd chooser=None) self.assertEqual(config, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_BASIS) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # the basis configuration (config3) not better than incumbent, so can move on self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 2) self.assertEqual(self.stats.ta_runs, 5) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(list(self.rh.data.values())[4][2], StatusType.CAPPED) self.assertEqual( intensifier.n_iters, 1) # iteration continues as `min_chall` condition is not met self.assertIsInstance(intensifier.configs_to_run, collections.abc.Iterator) # no configs should be left at the end with self.assertRaises(StopIteration): next(intensifier.configs_to_run) # intensification continues running incumbent again in same iteration... config, _ = intensifier.get_next_challenger( challengers=None, # don't need a new list here as old one is cont'd chooser=None) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(inc, self.config1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual( len( self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 3) self.assertEqual( len( self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True)), 2) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 0) # capped def test_eval_challenger_2(self): """ test eval_challenger() - a complete intensification run without a `always_race_against` configuration """ def target(x): return 2 * x['a'] + x['b'] taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=True, always_race_against=None, run_limit=1) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # intensification iteration #1 # run first config as incumbent if incumbent is None config, _ = intensifier.get_next_challenger(challengers=[self.config3], chooser=None) self.assertEqual(config, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # eval config 2 (=first run) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=None, run_history=self.rh, ) self.assertEqual(inc, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual(intensifier.n_iters, 1) # 1 intensification run complete! # regular intensification begins - run incumbent config, _ = intensifier.get_next_challenger( challengers=None, # since incumbent is run, no configs required chooser=None) self.assertEqual(config, inc) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # no new TA runs as there are no more instances to run self.assertEqual(inc, self.config3) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual(self.stats.ta_runs, 1) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) # run challenger now that the incumbent has been executed config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config1], chooser=None) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(config, self.config2) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # challenger has a better performance, so incumbent has changed self.assertEqual(inc, self.config2) self.assertEqual(self.stats.inc_changed, 2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT ) # since there is no `always_race_against` self.assertFalse(intensifier.continue_challenger) self.assertEqual( intensifier.n_iters, 1) # iteration continues as `min_chall` condition is not met # intensification continues running incumbent again in same iteration... # run incumbent config, _ = intensifier.get_next_challenger( challengers=None, # don't need a new list here as old one is cont'd chooser=None) self.assertEqual(config, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # run challenger config, _ = intensifier.get_next_challenger( challengers=None, # don't need a new list here as old one is cont'd chooser=None) self.assertEqual(config, self.config1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(intensifier.n_iters, 2) # 2 intensification run complete! # no configs should be left at the end with self.assertRaises(StopIteration): next(intensifier.configs_to_run) self.assertEqual( len( self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 1) self.assertEqual( len( self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True)), 1) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 1) def test_eval_challenger_3(self): """ test eval_challenger for a resumed SMAC run (first run with incumbent) """ def target(x): return x['a'] taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=False, always_race_against=None, run_limit=1) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # adding run for incumbent configuration self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) # intensification - incumbent will be run, but not as RUN_FIRST_CONFIG stage config, _ = intensifier.get_next_challenger(challengers=[self.config2], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual( len( self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 2) def test_no_new_intensification_wo_challenger_run(self): """ This test ensures that no new iteration is started if no challenger run was conducted """ def target(x): return 2 * x['a'] + x['b'] taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="quality") taf.runhistory = self.rh intensifier = Intensifier( tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=True, always_race_against=None, run_limit=1, min_chall=1, ) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) config, _ = intensifier.get_next_challenger(challengers=[self.config3], chooser=None) self.assertEqual(config, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=None, run_history=self.rh, ) self.assertEqual(inc, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(intensifier.n_iters, 1) # 1 intensification run complete! # regular intensification begins - run incumbent config, _ = intensifier.get_next_challenger( challengers=None, # since incumbent is run, no configs required chooser=None) self.assertEqual(config, inc) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(intensifier.n_iters, 1) # Check that we don't walk into the next iteration if the challenger is passed again config, _ = intensifier.get_next_challenger(challengers=[self.config3], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(intensifier.n_iters, 1) intensifier._next_iteration() # Add a configuration, then try to execute it afterwards self.assertEqual(intensifier.n_iters, 2) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=0, additional_info=None) intensifier.stage = IntensifierStage.RUN_CHALLENGER config, _ = intensifier.get_next_challenger(challengers=[self.config1], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(intensifier.n_iters, 2) self.assertEqual(intensifier.num_chall_run, 0) # This returns the config evaluating the incumbent again config, _ = intensifier.get_next_challenger(challengers=None, chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # This doesn't return a config because the array of configs is exhausted config, _ = intensifier.get_next_challenger(challengers=None, chooser=None) self.assertIsNone(config) # This finally gives a runable configuration config, _ = intensifier.get_next_challenger(challengers=[self.config2], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(intensifier.n_iters, 3) self.assertEqual(intensifier.num_chall_run, 1)
class TestIntensify(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.rh = RunHistory() self.cs = get_config_space() self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100}) self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0}) self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100}) self.scen = Scenario({ "cutoff_time": 2, 'cs': self.cs, "run_obj": 'runtime', "output_dir": '' }) self.stats = Stats(scenario=self.scen) self.stats.start_timing() self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) def test_race_challenger(self): """ test _race_challenger without adaptive capping """ def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) intensifier.N = 1 inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh) self.assertEqual(inc, self.config2) def test_race_challenger_2(self): """ test _race_challenger with adaptive capping """ def target(x): time.sleep(1.5) return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1]) self.rh.add(config=self.config1, cost=.001, time=0.001, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None) intensifier.N = 1 # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc = intensifier._race_challenger( challenger=self.config2, incumbent=self.config1, run_history=self.rh, ) # self.assertTrue(False) self.assertEqual(inc, self.config1) def test_race_challenger_3(self): """ test _race_challenger with adaptive capping on a previously capped configuration """ def target(config: Configuration, seed: int, instance: str): if instance == 1: time.sleep(2.1) else: time.sleep(0.6) return (config['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime", par_factor=1) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), cutoff=2, instances=[1]) self.rh.add(config=self.config1, cost=0.5, time=.5, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None) # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) self.assertEqual(inc, self.config1) # further run for incumbent self.rh.add(config=self.config1, cost=2, time=2, status=StatusType.TIMEOUT, instance_id=2, seed=12345, additional_info=None) # give config2 a second chance - now it should run on both instances # run on instance 1 config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) # run on instance 2 config, _ = intensifier.get_next_challenger(challengers=[self.config3], chooser=None) self.assertEqual(config, self.config2) self.assertTrue(intensifier.continue_challenger) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) # the incumbent should still be config1 because # config2 should get on inst 1 a full timeout # such that c(config1) = 1.25 and c(config2) close to 1.3 self.assertEqual(inc, self.config1) # the capped run should not be counted in runs_perf_config self.assertAlmostEqual(self.rh.num_runs_per_config[2], 2) self.assertFalse(intensifier.continue_challenger) def test_race_challenger_large(self): """ test _race_challenger using solution_quality """ def target(x): return 1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), run_obj_time=False, deterministic=True) for i in range(10): self.rh.add(config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=12345, additional_info=None) intensifier.stage = IntensifierStage.RUN_CHALLENGER # tie on first instances and then challenger should always win # and be returned as inc while True: config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) # stop when challenger evaluation is over if not intensifier.stage == IntensifierStage.RUN_CHALLENGER: break self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True) self.assertEqual(len(runs), 10) def test_race_challenger_large_blocked_seed(self): """ test _race_challenger whether seeds are blocked for challenger runs """ def target(x): return 1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), run_obj_time=False, deterministic=False) for i in range(10): self.rh.add(config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) intensifier.stage = IntensifierStage.RUN_CHALLENGER # tie on first instances and then challenger should always win # and be returned as inc while True: config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) # stop when challenger evaluation is over if not intensifier.stage == IntensifierStage.RUN_CHALLENGER: break self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True) self.assertEqual(len(runs), 10) seeds = sorted([r.seed for r in runs]) self.assertEqual(seeds, list(range(10)), seeds) def test_add_inc_run_det(self): """ test _add_inc_run() """ def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) # since we assume deterministic=1, # the second call should not add any more runs # given only one instance intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) def test_add_inc_run_nondet(self): """ test _add_inc_run() """ def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, runhistory=self.rh, run_obj="solution_quality") intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], deterministic=False) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 2, self.rh.data) runs = self.rh.get_runs_for_config(config=self.config1, only_max_observed_budget=True) # exactly one run on each instance self.assertIn(1, [runs[0].instance, runs[1].instance]) self.assertIn(2, [runs[0].instance, runs[1].instance]) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 3, self.rh.data) def test_get_next_challenger(self): """ test get_next_challenger() """ intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True) intensifier.stage = IntensifierStage.RUN_CHALLENGER # get a new challenger to evaluate config, new = intensifier.get_next_challenger( challengers=[self.config1, self.config2], chooser=None) self.assertEqual(config, self.config1, intensifier.current_challenger) self.assertEqual(intensifier._chall_indx, 1) self.assertEqual(intensifier.N, 1) self.assertTrue(new) # when already evaluating a challenger, return the same challenger intensifier.to_run = [(1, 1, 0)] config, new = intensifier.get_next_challenger( challengers=[self.config2], chooser=None) self.assertEqual(config, self.config1, intensifier.current_challenger) self.assertEqual(intensifier._chall_indx, 1) self.assertFalse(new) def test_generate_challenger(self): """ test generate_challenger() """ # test get generator from a list of challengers intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=None, rng=np.random.RandomState(12345), instances=[1], deterministic=True) gen = intensifier._generate_challengers( challengers=[self.config1, self.config2], chooser=None) self.assertEqual(next(gen), self.config1) self.assertEqual(next(gen), self.config2) self.assertRaises(StopIteration, next, gen) # test get generator from a chooser - would return only 1 configuration intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=None, rng=np.random.RandomState(12345), instances=[1], deterministic=True) chooser = SMAC4AC(self.scen, rng=1).solver.epm_chooser gen = intensifier._generate_challengers(challengers=None, chooser=chooser) self.assertEqual(next(gen).get_dictionary(), {'a': 24, 'b': 68}) self.assertRaises(StopIteration, next, gen) # when both are none, raise error with self.assertRaisesRegex(ValueError, "No configurations/chooser provided"): intensifier._generate_challengers(challengers=None, chooser=None) def test_eval_challenger(self): """ test eval_challenger() - a complete intensification run """ def target(x): return x['a'] taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=False, always_race_against=self.config3, run_limit=1) # run incumbent first if it was not run before config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config1, self.config3], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=None, run_history=self.rh, ) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) # run challenger now that the incumbent has been executed config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config1, self.config3], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # challenger should have a better performance, so incumbent should have changed self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_DEFAULT) self.assertFalse(intensifier.continue_challenger) # run `always_race_against` now since the incumbent has changed config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config1, self.config3], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(inc, self.config1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 1) self.assertEqual(intensifier.n_iters, 1) self.assertIsInstance(intensifier.configs_to_run, collections.Iterator) with self.assertRaises(StopIteration): next(intensifier.configs_to_run)
def _get_incumbent(self, challenger: Configuration, incumbent: typing.Optional[Configuration], run_history: RunHistory, log_traj: bool = True) -> Configuration: """ Compares the challenger with current incumbent and returns the best configuration Parameters ---------- challenger : Configuration promising configuration incumbent : Configuration best configuration so far run_history : smac.runhistory.runhistory.RunHistory stores all runs we ran so far log_traj : bool whether to log changes of incumbents in trajectory Returns ------- typing.Tuple[Configuration, float] incumbent and incumbent cost """ # compare challenger with current incumbent if incumbent is None: # first intensify run from initial design new_incumbent = challenger inc_perf = run_history.get_cost(new_incumbent) self.logger.info("First Incumbent found! Cost of incumbent is (%.4f)" % inc_perf) self.logger.info("incumbent configuration: %s" % str(challenger)) if log_traj: # adding incumbent entry self.stats.inc_changed += 1 # first incumbent self.traj_logger.add_entry(train_perf=inc_perf, incumbent_id=self.stats.inc_changed, incumbent=new_incumbent) elif self.instance_as_budget: new_incumbent = self._compare_configs(incumbent, challenger, run_history, log_traj) # if compare config returned none, then it is undecided. So return old incumbent new_incumbent = incumbent if new_incumbent is None else new_incumbent else: inc_runs = run_history.get_runs_for_config(incumbent, only_max_observed_budget=True) chall_runs = run_history.get_runs_for_config(challenger, only_max_observed_budget=True) if len(inc_runs) > 1: raise ValueError( 'Number of incumbent runs on budget %f must not exceed 1, but is %d', inc_runs[0].budget, len(inc_runs), ) if len(chall_runs) > 1: raise ValueError( 'Number of challenger runs on budget %f must not exceed 1, but is %d', chall_runs[0].budget, len(chall_runs), ) inc_run = inc_runs[0] chall_run = chall_runs[0] if inc_run.budget > chall_run.budget: self.logger.debug('Incumbent evaluated on higher budget than challenger (%.4f > %.4f), ' 'not changing the incumbent', inc_run.budget, chall_run.budget) new_incumbent = incumbent elif inc_run.budget < chall_run.budget: self.logger.debug('Challenger evaluated on higher budget than incumbent (%.4f > %.4f), ' 'changing the incumbent', chall_run.budget, inc_run.budget) new_incumbent = challenger if log_traj: # adding incumbent entry self.stats.inc_changed += 1 new_inc_cost = run_history.get_cost(new_incumbent) self.traj_logger.add_entry(train_perf=new_inc_cost, incumbent_id=self.stats.inc_changed, incumbent=new_incumbent) else: chall_cost = run_history.get_cost(challenger) inc_cost = run_history.get_cost(incumbent) if chall_cost < inc_cost: self.logger.info("Challenger (%.4f) is better than incumbent (%.4f) on budget %.4f.", chall_cost, inc_cost, chall_run.budget) self._log_incumbent_changes(incumbent, challenger) new_incumbent = challenger if log_traj: # adding incumbent entry self.stats.inc_changed += 1 # first incumbent self.traj_logger.add_entry(train_perf=chall_cost, incumbent_id=self.stats.inc_changed, incumbent=new_incumbent) else: self.logger.debug("Incumbent (%.4f) is at least as good as the challenger (%.4f) on budget %.4f.", inc_cost, chall_cost, inc_run.budget) new_incumbent = incumbent return new_incumbent
def eval_challenger(self, challenger: Configuration, incumbent: typing.Optional[Configuration], run_history: RunHistory, time_bound: float = float(MAXINT), log_traj: bool = True) -> typing.Tuple[Configuration, float]: """ Running intensification via successive halving to determine the incumbent configuration. *Side effect:* adds runs to run_history Parameters ---------- challenger : Configuration promising configuration incumbent : typing.Optional[Configuration] best configuration so far, None in 1st run run_history : smac.runhistory.runhistory.RunHistory stores all runs we ran so far time_bound : float, optional (default=2 ** 31 - 1) time in [sec] available to perform intensify log_traj : bool whether to log changes of incumbents in trajectory Returns ------- typing.Tuple[Configuration, float] incumbent and incumbent cost """ # calculating the incumbent's performance for adaptive capping # this check is required because: # - there is no incumbent performance for the first ever 'intensify' run (from initial design) # - during the 1st intensify run, the incumbent shouldn't be capped after being compared against itself if incumbent and incumbent != challenger: inc_runs = run_history.get_runs_for_config(incumbent, only_max_observed_budget=True) inc_sum_cost = run_history.sum_cost(config=incumbent, instance_seed_budget_keys=inc_runs) else: inc_sum_cost = np.inf if self.first_run: self.logger.info("First run, no incumbent provided; challenger is assumed to be the incumbent") incumbent = challenger self.first_run = False # select which instance to run current config on curr_budget = self.all_budgets[self.stage] # selecting instance-seed subset for this budget, depending on the kind of budget if self.instance_as_budget: prev_budget = int(self.all_budgets[self.stage - 1]) if self.stage > 0 else 0 curr_insts = self.inst_seed_pairs[int(prev_budget):int(curr_budget)] else: curr_insts = self.inst_seed_pairs n_insts_remaining = len(curr_insts) - self.curr_inst_idx - 1 self.logger.debug(" Running challenger - %s" % str(challenger)) # run the next instance-seed pair for the given configuration instance, seed = curr_insts[self.curr_inst_idx] # selecting cutoff if running adaptive capping cutoff = self.cutoff if self.run_obj_time: cutoff = self._adapt_cutoff(challenger=challenger, run_history=run_history, inc_sum_cost=inc_sum_cost) if cutoff is not None and cutoff <= 0: # ran out of time to validate challenger self.logger.debug("Stop challenger intensification due to adaptive capping.") self.curr_inst_idx = np.inf self.logger.debug('Cutoff for challenger: %s' % str(cutoff)) try: # run target algorithm for each instance-seed pair self.logger.debug("Execute target algorithm") try: status, cost, dur, res = self.tae_runner.start( config=challenger, instance=instance, seed=seed, cutoff=cutoff, budget=0.0 if self.instance_as_budget else curr_budget, instance_specific=self.instance_specifics.get(instance, "0"), # Cutoff might be None if self.cutoff is None, but then the first if statement prevents # evaluation of the second if statement capped=(self.cutoff is not None) and (cutoff < self.cutoff) # type: ignore[operator] # noqa F821 ) self._ta_time += dur self.num_run += 1 self.curr_inst_idx += 1 except CappedRunException: # We move on to the next configuration if a configuration is capped self.logger.debug("Budget exhausted by adaptive capping; " "Interrupting current challenger and moving on to the next one") # ignore all pending instances self.curr_inst_idx = np.inf n_insts_remaining = 0 status = StatusType.CAPPED # adding challengers to the list of evaluated challengers # - Stop: CAPPED/CRASHED/TIMEOUT/MEMOUT (!= SUCCESS & DONOTADVANCE) # - Advance to next stage: SUCCESS # curr_challengers is a set, so "at least 1" success can be counted by set addition (no duplicates) # If a configuration is successful, it is added to curr_challengers. # if it fails it is added to fail_challengers. if np.isfinite(self.curr_inst_idx) and status in [StatusType.SUCCESS, StatusType.DONOTADVANCE]: self.success_challengers.add(challenger) # successful configs else: self.fail_challengers.add(challenger) # capped/crashed/do not advance configs # get incumbent if all instances have been evaluated if n_insts_remaining <= 0: incumbent = self._compare_configs(challenger=challenger, incumbent=incumbent, run_history=run_history, log_traj=log_traj) except BudgetExhaustedException: # Returning the final incumbent selected so far because we ran out of optimization budget self.logger.debug("Budget exhausted; " "Interrupting optimization run and returning current incumbent") # if all configurations for the current stage have been evaluated, reset stage num_chal_evaluated = len(self.success_challengers.union(self.fail_challengers)) + self.fail_chal_offset if num_chal_evaluated == self.n_configs_in_stage[self.stage] and n_insts_remaining <= 0: self.logger.info('Successive Halving iteration-step: %d-%d with ' 'budget [%.2f / %d] - evaluated %d challenger(s)' % (self.sh_iters + 1, self.stage + 1, self.all_budgets[self.stage], self.max_budget, self.n_configs_in_stage[self.stage])) self._update_stage(run_history=run_history) # get incumbent cost inc_perf = run_history.get_cost(incumbent) return incumbent, inc_perf
class TestIntensify(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.rh = RunHistory() self.cs = get_config_space() self.config1 = Configuration(self.cs, values={"a": 0, "b": 100}) self.config2 = Configuration(self.cs, values={"a": 100, "b": 0}) self.config3 = Configuration(self.cs, values={"a": 100, "b": 100}) self.scen = Scenario({ "cutoff_time": 2, "cs": self.cs, "run_obj": "runtime", "output_dir": "", "deterministic": False, "limit_resources": True, }) self.stats = Stats(scenario=self.scen) self.stats.start_timing() self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) def test_race_challenger_1(self): """ Makes sure that a racing configuration with better performance, is selected as incumbent No adaptive capping """ def target(x): return (x["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, ) self.rh.add( config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None, ) intensifier.N = 1 inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=self.config2, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=self.config2, instance=instance, instance_specific="0", cutoff=cutoff, seed=seed, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.num_run, 1) self.assertEqual(intensifier.num_chall_run, 1) def test_race_challenger_2(self): """ Makes sure that a racing configuration with better performance, that is capped, doesn't substitute the incumbent. """ def target(x): time.sleep(1.5) return (x["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], ) self.rh.add( config=self.config1, cost=0.001, time=0.001, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None, ) intensifier.N = 1 # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=self.config2, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=self.config2, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=True, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config1) self.assertEqual(intensifier.num_run, 1) self.assertEqual(intensifier.num_chall_run, 1) def test_race_challenger_3(self): """ test _race_challenger with adaptive capping on a previously capped configuration """ def target(config: Configuration, seed: int, instance: str): if instance == 1: time.sleep(2.1) else: time.sleep(0.6) return (config["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict( use_pynisher=False, ta=target, stats=self.stats, run_obj="runtime", par_factor=1, ) taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), cutoff=2, instances=[1], ) self.rh.add( config=self.config1, cost=0.5, time=0.5, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None, ) # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=config, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=config, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=True, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config1) # further run for incumbent self.rh.add( config=self.config1, cost=2, time=2, status=StatusType.TIMEOUT, instance_id=2, seed=12345, additional_info=None, ) # give config2 a second chance - now it should run on both instances # run on instance 1 config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=config, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=config, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) # run on instance 2 config, _ = intensifier.get_next_challenger(challengers=[self.config3], chooser=None) self.assertEqual(config, self.config2) self.assertTrue(intensifier.continue_challenger) inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=config, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=config, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) # the incumbent should still be config1 because # config2 should get on inst 1 a full timeout # such that c(config1) = 1.25 and c(config2) close to 1.3 self.assertEqual(inc, self.config1) # the capped run should not be counted in runs_perf_config self.assertAlmostEqual(self.rh.num_runs_per_config[2], 2) self.assertFalse(intensifier.continue_challenger) self.assertEqual(intensifier.num_run, 3) self.assertEqual(intensifier.num_chall_run, 3) def test_race_challenger_large(self): """ test _race_challenger using solution_quality """ def target(x): return 1 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), run_obj_time=False, deterministic=True, ) for i in range(10): self.rh.add( config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=12345, additional_info=None, ) intensifier.stage = IntensifierStage.RUN_CHALLENGER # tie on first instances and then challenger should always win # and be returned as inc while True: if intensifier.continue_challenger: config = intensifier.current_challenger else: config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=config, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=config, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) # stop when challenger evaluation is over if not intensifier.stage == IntensifierStage.RUN_CHALLENGER: break self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True) self.assertEqual(len(runs), 10) self.assertEqual(intensifier.num_run, 10) self.assertEqual(intensifier.num_chall_run, 10) def test_race_challenger_large_blocked_seed(self): """ test _race_challenger whether seeds are blocked for challenger runs """ def target(x): return 1 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), run_obj_time=False, deterministic=False, ) for i in range(10): self.rh.add( config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None, ) intensifier.stage = IntensifierStage.RUN_CHALLENGER # tie on first instances and then challenger should always win # and be returned as inc while True: if intensifier.continue_challenger: config = intensifier.current_challenger else: config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=config, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=config, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) # stop when challenger evaluation is over if not intensifier.stage == IntensifierStage.RUN_CHALLENGER: break self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True) self.assertEqual(len(runs), 10) seeds = sorted([r.seed for r in runs]) self.assertEqual(seeds, list(range(10)), seeds) self.assertEqual(intensifier.num_run, 10) self.assertEqual(intensifier.num_chall_run, 10) def test_add_inc_run_det(self): """ test _add_inc_run() """ def target(x): return (x["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True, ) instance, seed, cutoff = intensifier._get_next_inc_run( available_insts=intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh)) run_info = RunInfo( config=self.config1, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) intensifier.stage = IntensifierStage.PROCESS_FIRST_CONFIG_RUN inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(len(self.rh.data), 1, self.rh.data) # since we assume deterministic=1, # the second call should not add any more runs # given only one instance # So the returned seed/instance is None so that a new # run to be triggered is not launched available_insts = intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh) # Make sure that the list is empty, and hence no new call # of incumbent will be triggered self.assertFalse(available_insts) # The following two tests evaluate to zero because _next_iteration is triggered by _add_inc_run # as it is the first evaluation of this intensifier # After the above incumbent run, the stage is # IntensifierStage.RUN_CHALLENGER. Change it to test next iteration intensifier.stage = IntensifierStage.PROCESS_FIRST_CONFIG_RUN inc, perf = intensifier.process_results( run_info=run_info, incumbent=None, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(intensifier.num_run, 0) self.assertEqual(intensifier.num_chall_run, 0) def test_add_inc_run_nondet(self): """ test _add_inc_run() """ def target(x): return (x["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="solution_quality") intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], deterministic=False, ) instance, seed, cutoff = intensifier._get_next_inc_run( available_insts=intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh)) run_info = RunInfo( config=self.config1, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(len(self.rh.data), 1, self.rh.data) instance, seed, cutoff = intensifier._get_next_inc_run( available_insts=intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh)) run_info = RunInfo( config=self.config1, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(len(self.rh.data), 2, self.rh.data) runs = self.rh.get_runs_for_config(config=self.config1, only_max_observed_budget=True) # exactly one run on each instance self.assertIn(1, [runs[0].instance, runs[1].instance]) self.assertIn(2, [runs[0].instance, runs[1].instance]) instance, seed, cutoff = intensifier._get_next_inc_run( available_insts=intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh)) run_info = RunInfo( config=self.config1, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(len(self.rh.data), 3, self.rh.data) # The number of runs performed should be 3 # No Next iteration call as an incumbent is provided self.assertEqual(intensifier.num_run, 2) self.assertEqual(intensifier.num_chall_run, 0) def testget_next_challenger(self): """ test get_next_challenger() """ intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True, ) intensifier.stage = IntensifierStage.RUN_CHALLENGER # get a new challenger to evaluate config, new = intensifier.get_next_challenger( challengers=[self.config1, self.config2], chooser=None) self.assertEqual(config, self.config1, intensifier.current_challenger) self.assertEqual(intensifier._chall_indx, 1) self.assertEqual(intensifier.N, 1) self.assertTrue(new) # when already evaluating a challenger, return the same challenger intensifier.to_run = [(1, 1, 0)] config, new = intensifier.get_next_challenger( challengers=[self.config2], chooser=None) self.assertEqual(config, self.config1, intensifier.current_challenger) self.assertEqual(intensifier._chall_indx, 1) self.assertFalse(new) def test_generate_challenger(self): """ test generate_challenger() """ # test get generator from a list of challengers intensifier = Intensifier( stats=self.stats, traj_logger=None, rng=np.random.RandomState(12345), instances=[1], deterministic=True, ) gen = intensifier._generate_challengers( challengers=[self.config1, self.config2], chooser=None) self.assertEqual(next(gen), self.config1) self.assertEqual(next(gen), self.config2) self.assertRaises(StopIteration, next, gen) # test get generator from a chooser - would return only 1 configuration intensifier = Intensifier( stats=self.stats, traj_logger=None, rng=np.random.RandomState(12345), instances=[1], deterministic=True, ) chooser = SMAC4AC(self.scen, rng=1).solver.epm_chooser gen = intensifier._generate_challengers(challengers=None, chooser=chooser) self.assertEqual(next(gen).get_dictionary(), {"a": 24, "b": 68}) self.assertRaises(StopIteration, next, gen) # when both are none, raise error with self.assertRaisesRegex(ValueError, "No configurations/chooser provided"): intensifier._generate_challengers(challengers=None, chooser=None) def test_eval_challenger_1(self): """ test eval_challenger() - a complete intensification run with a `always_race_against` configuration """ print(self.rh) def target(x): if x["a"] == 100: time.sleep(1) return x["a"] taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], run_obj_time=True, cutoff=2, deterministic=False, always_race_against=self.config3, run_limit=1, ) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # intensification iteration #1 # run first config as incumbent if incumbent is None intent, run_info = intensifier.get_next_run( incumbent=None, run_history=self.rh, challengers=[self.config2], chooser=None, ) self.assertEqual(run_info.config, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.PROCESS_FIRST_CONFIG_RUN) # eval config 2 (=first run) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=None, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual(intensifier.n_iters, 1) # 1 intensification run complete! # intensification iteration #2 # regular intensification begins - run incumbent first intent, run_info = intensifier.get_next_run( challengers=None, # don't need a new list here as old one is cont'd incumbent=inc, run_history=self.rh, chooser=None, ) self.assertEqual(run_info.config, inc) self.assertEqual(intensifier.stage, IntensifierStage.PROCESS_INCUMBENT_RUN) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(self.stats.inc_changed, 1) # run challenger now that the incumbent has been executed intent, run_info = intensifier.get_next_run(challengers=[self.config1], incumbent=inc, run_history=self.rh, chooser=None) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(run_info.config, self.config1) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) # challenger has a better performance, but not run on all instances yet. so incumbent stays the same self.assertEqual(inc, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertTrue(intensifier.continue_challenger) # run challenger again on the other instance intent, run_info = intensifier.get_next_run( challengers=None, # don't need a new list here as old one is cont'd incumbent=inc, run_history=self.rh, chooser=None, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(run_info.config, self.config1) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) # challenger better than incumbent in both instances. so incumbent changed self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_BASIS) self.assertFalse(intensifier.continue_challenger) # run basis configuration (`always_race_against`) intent, run_info = intensifier.get_next_run( challengers=None, # don't need a new list here as old one is cont'd incumbent=inc, run_history=self.rh, chooser=None, ) self.assertEqual(run_info.config, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_BASIS) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) # the basis configuration (config3) not better than incumbent, so can move on self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT, msg=self.rh.data.items()) self.assertEqual(list(self.rh.data.values())[4][2], StatusType.CAPPED) self.assertEqual( intensifier.n_iters, 1) # iteration continues as `min_chall` condition is not met self.assertIsInstance(intensifier.configs_to_run, collections.abc.Iterator) # no configs should be left at the end with self.assertRaises(StopIteration): next(intensifier.configs_to_run) # intensification continues running incumbent again in same iteration... intent, run_info = intensifier.get_next_run( challengers=None, # don't need a new list here as old one is cont'd incumbent=inc, run_history=self.rh, chooser=None, ) self.assertEqual(intensifier.stage, IntensifierStage.PROCESS_INCUMBENT_RUN) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual( len( self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 3, ) self.assertEqual( len( self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True)), 2, ) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 0, ) # capped def test_eval_challenger_2(self): """ test eval_challenger() - a complete intensification run without a `always_race_against` configuration """ def target(x): return 2 * x["a"] + x["b"] taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="quality") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=True, always_race_against=None, run_limit=1, ) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # intensification iteration #1 # run first config as incumbent if incumbent is None intent, run_info = intensifier.get_next_run( challengers=[self.config3], incumbent=None, run_history=self.rh, chooser=None, ) self.assertEqual(run_info.config, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.PROCESS_FIRST_CONFIG_RUN) # eval config 2 (=first run) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=None, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual(intensifier.n_iters, 1) # 1 intensification run complete! # regular intensification begins - run incumbent # Normally a challenger will be given, which in this case is the incumbent # But no more instances are available. So to prevent cicles # where No iteration happens, provide the challengers intent, run_info = intensifier.get_next_run( challengers=[ self.config2, self.config1, ], # since incumbent is run, no configs required incumbent=inc, run_history=self.rh, chooser=None, ) # no new TA runs as there are no more instances to run self.assertEqual(inc, self.config3) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 1, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) # run challenger now that the incumbent has been executed # So this call happen above, to save one iteration self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(run_info.config, self.config2) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) # challenger has a better performance, so incumbent has changed self.assertEqual(inc, self.config2) self.assertEqual(self.stats.inc_changed, 2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT ) # since there is no `always_race_against` self.assertFalse(intensifier.continue_challenger) self.assertEqual( intensifier.n_iters, 1) # iteration continues as `min_chall` condition is not met # intensification continues running incumbent again in same iteration... # run incumbent # Same here, No further instance-seed pairs for incumbent available # so above call gets the new config to run self.assertEqual(run_info.config, self.config2) # There is a transition from: # IntensifierStage.RUN_FIRST_CONFIG-> IntensifierStage.RUN_INCUMBENT # Because after the first run, incumbent is run. # Nevertheless, there is now a transition: # IntensifierStage.RUN_INCUMBENT->IntensifierStage.RUN_CHALLENGER # because in add_inc_run, there are more available instance pairs # FROM: IntensifierStage.RUN_INCUMBENT TO: IntensifierStage.RUN_INCUMBENT WHY: no more to run # if all <instance, seed> have been run, compare challenger performance # self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) # run challenger intent, run_info = intensifier.get_next_run( challengers=None, # don't need a new list here as old one is cont'd incumbent=inc, run_history=self.rh, chooser=None, ) self.assertEqual(run_info.config, self.config1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(intensifier.n_iters, 2) # 2 intensification run complete! # no configs should be left at the end with self.assertRaises(StopIteration): next(intensifier.configs_to_run) self.assertEqual( len( self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 1, ) self.assertEqual( len( self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True)), 1, ) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 1, ) def test_eval_challenger_3(self): """ test eval_challenger for a resumed SMAC run (first run with incumbent) """ def target(x): return x["a"] taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="quality") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=False, always_race_against=None, run_limit=1, ) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # adding run for incumbent configuration self.rh.add( config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None, ) # intensification - incumbent will be run, but not as RUN_FIRST_CONFIG stage intent_, run_info = intensifier.get_next_run( challengers=[self.config2], incumbent=self.config1, run_history=self.rh, chooser=None, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual( len( self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 2, ) def test_no_new_intensification_wo_challenger_run(self): """ This test ensures that no new iteration is started if no challenger run was conducted """ def target(x): return 2 * x["a"] + x["b"] taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="quality") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=True, always_race_against=None, run_limit=1, min_chall=1, ) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) intent, run_info = intensifier.get_next_run( challengers=[self.config3], incumbent=None, run_history=self.rh, chooser=None, ) self.assertEqual(run_info.config, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.PROCESS_FIRST_CONFIG_RUN) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=None, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(intensifier.n_iters, 1) # 1 intensification run complete! # regular intensification begins - run incumbent # No further instance-seed pairs for incumbent available # Here None challenger is suggested. Code jumps to next iteration # This causes a transition from RUN_INCUMBENT->RUN_CHALLENGER # But then, the next configuration to run is the incumbent # We don't rerun the incumbent (see message): # Challenger was the same as the current incumbent; Skipping challenger # Then, we try to get more challengers, but below all challengers # Provided are config3, the incumbent which means nothing more to run intent, run_info = intensifier.get_next_run( challengers=[self.config3 ], # since incumbent is run, no configs required incumbent=inc, run_history=self.rh, chooser=None, ) self.assertEqual(run_info.config, None) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) intensifier._next_iteration() # Add a configuration, then try to execute it afterwards self.assertEqual(intensifier.n_iters, 2) self.rh.add( config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=0, additional_info=None, ) intensifier.stage = IntensifierStage.RUN_CHALLENGER # In the upcoming get next run, the stage is RUN_CHALLENGER # so the intensifier tries to run config1. Nevertheless, # there are no further instances for this configuration available. # In this scenario, the intensifier produces a SKIP intent as an indication # that a new iteration must be initiated, and for code simplicity, # relies on a new call to get_next_run to yield more configurations intent, run_info = intensifier.get_next_run(challengers=[self.config1], incumbent=inc, run_history=self.rh, chooser=None) self.assertEqual(intent, RunInfoIntent.SKIP) # This doesn't return a config because the array of configs is exhausted intensifier.stage = IntensifierStage.RUN_CHALLENGER config, _ = intensifier.get_next_challenger(challengers=None, chooser=None) self.assertIsNone(config) # This finally gives a runable configuration intent, run_info = intensifier.get_next_run(challengers=[self.config2], incumbent=inc, run_history=self.rh, chooser=None) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) # 4 Iterations due to the proactive runs # of get next challenger self.assertEqual(intensifier.n_iters, 3) self.assertEqual(intensifier.num_chall_run, 1)
class TestIntensify(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.rh = RunHistory(aggregate_func=average_cost) self.cs = get_config_space() self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100}) self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0}) self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100}) self.scen = Scenario({ "cutoff_time": 2, 'cs': self.cs, "output_dir": "" }) self.stats = Stats(scenario=self.scen) self.stats.start_timing() self.logger = logging.getLogger("Test") def test_compare_configs_chall(self): ''' challenger is better but has not enough runs ''' intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=2, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config2, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) # challenger has enough runs and is better self.assertEqual(conf, self.config2, "conf: %s" % (conf)) def test_compare_configs_inc(self): ''' incumbent is better ''' intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config2, cost=2, time=2, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) # challenger worse than inc self.assertEqual(conf, self.config1, "conf: %s" % (conf)) def test_compare_configs_unknow(self): ''' challenger is better but has less runs; -> no decision (None) ''' intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config1, cost=1, time=2, status=StatusType.SUCCESS, instance_id=2, seed=None, additional_info=None) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=2, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) # challenger worse than inc self.assertIsNone(conf, "conf: %s" % (conf)) def test_race_challenger(self): ''' test _race_challenger without adaptive capping ''' def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) self.assertEqual(inc, self.config2) def test_race_challenger(self): ''' test _race_challenger with adaptive capping ''' def target(x): time.sleep(1.5) return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1]) self.rh.add(config=self.config1, cost=.001, time=0.001, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None) # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # self.assertTrue(False) self.assertEqual(inc, self.config1) self.assertLess(self.rh.get_cost(self.config2), 2, self.rh.get_cost(self.config2)) # get data for config2 to check that the correct run was performed run = self.rh.get_runs_for_config(self.config2)[0] config_id = self.rh.config_ids[self.config2] self.assertEqual(run.instance, 1, run.instance) self.assertEqual(run.seed, 12345, run.seed) def test_race_challenger_large(self): ''' test _race_challenger using solution_quality ''' def target(x): return 1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), deterministic=True) for i in range(10): self.rh.add(config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=12345, additional_info=None) # tie on first instances and then challenger should always win # and be returned as inc inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # self.assertTrue(False) self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1, self.rh.get_cost(self.config2)) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2) self.assertEqual(len(runs), 10) def test_race_challenger_large_blocked_seed(self): ''' test _race_challenger whether seeds are blocked for challenger runs ''' def target(x): return 1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), deterministic=False) for i in range(10): self.rh.add(config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) # tie on first instances and then challenger should always win # and be returned as inc inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # self.assertTrue(False) self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1, self.rh.get_cost(self.config2)) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2) self.assertEqual(len(runs), 10) seeds = sorted([r.seed for r in runs]) self.assertEqual(seeds, list(range(10)), seeds) def test_add_inc_run_det(self): ''' test _add_inc_run() ''' def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) # since we assume deterministic=1, # the second call should not add any more runs # given only one instance intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) def test_add_inc_run_nondet(self): ''' test _add_inc_run() ''' def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], deterministic=False) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 2, self.rh.data) runs = self.rh.get_runs_for_config(config=self.config1) # exactly one run on each instance self.assertIn(1, [runs[0].instance, runs[1].instance]) self.assertIn(2, [runs[0].instance, runs[1].instance]) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 3, self.rh.data) def test_adaptive_capping(self): ''' test _adapt_cutoff() ''' intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(5)), deterministic=False) for i in range(5): self.rh.add(config=self.config1, cost=i + 1, time=i + 1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) for i in range(3): self.rh.add(config=self.config2, cost=i + 1, time=i + 1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) inst_seed_pairs = self.rh.get_runs_for_config(self.config1) # cost used by incumbent for going over all runs in inst_seed_pairs inc_sum_cost = sum_cost(config=self.config1, instance_seed_pairs=inst_seed_pairs, run_history=self.rh) cutoff = intensifier._adapt_cutoff(challenger=self.config2, incumbent=self.config1, run_history=self.rh, inc_sum_cost=inc_sum_cost) # 15*1.2 - 6 self.assertEqual(cutoff, 12) intensifier.cutoff = 5 cutoff = intensifier._adapt_cutoff(challenger=self.config2, incumbent=self.config1, run_history=self.rh, inc_sum_cost=inc_sum_cost) # scenario cutoff self.assertEqual(cutoff, 5)
class TestIntensify(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.rh = RunHistory(aggregate_func=average_cost) self.cs = get_config_space() self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100}) self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0}) self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100}) self.scen = Scenario({ "cutoff_time": 2, 'cs': self.cs, "run_obj": 'runtime', "output_dir": '' }) self.stats = Stats(scenario=self.scen) self.stats.start_timing() self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) def test_compare_configs_no_joint_set(self): intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) for i in range(2): self.rh.add(config=self.config1, cost=2, time=2, status=StatusType.SUCCESS, instance_id=1, seed=i, additional_info=None) for i in range(2, 5): self.rh.add(config=self.config2, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=i, additional_info=None) # The sets for the incumbent are completely disjoint. conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) self.assertIsNone(conf) # The incumbent has still one instance-seed pair left on which the # challenger was not run yet. self.rh.add(config=self.config2, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=1, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) self.assertIsNone(conf) def test_compare_configs_chall(self): ''' challenger is better ''' intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=2, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config2, cost=0, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) # challenger has enough runs and is better self.assertEqual(conf, self.config2, "conf: %s" % (conf)) def test_compare_configs_inc(self): ''' incumbent is better ''' intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config2, cost=2, time=2, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) # challenger worse than inc self.assertEqual(conf, self.config1, "conf: %s" % (conf)) def test_compare_configs_unknow(self): ''' challenger is better but has less runs; -> no decision (None) ''' intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config1, cost=1, time=2, status=StatusType.SUCCESS, instance_id=2, seed=None, additional_info=None) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=2, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) # challenger worse than inc self.assertIsNone(conf, "conf: %s" % (conf)) def test_race_challenger(self): ''' test _race_challenger without adaptive capping ''' def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) self.assertEqual(inc, self.config2) def test_race_challenger_2(self): ''' test _race_challenger with adaptive capping ''' def target(x): time.sleep(1.5) return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1]) self.rh.add(config=self.config1, cost=.001, time=0.001, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None) # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # self.assertTrue(False) self.assertEqual(inc, self.config1) def test_race_challenger_3(self): ''' test _race_challenger with adaptive capping on a previously capped configuration ''' def target(config: Configuration, seed: int, instance: str): if instance == 1: time.sleep(2.1) else: time.sleep(0.6) return (config['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime", par_factor=1) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), cutoff=2, instances=[1]) self.rh.add(config=self.config1, cost=0.5, time=.5, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None) # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # self.assertTrue(False) self.assertEqual(inc, self.config1) # further run for incumbent self.rh.add(config=self.config1, cost=2, time=2, status=StatusType.TIMEOUT, instance_id=2, seed=12345, additional_info=None) # give config2 a second chance inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # the incumbent should still be config1 because # config2 should get on inst 1 a full timeout # such that c(config1) = 1.25 and c(config2) close to 1.3 self.assertEqual(inc, self.config1) # the capped run should not be counted in runs_perf_config self.assertAlmostEqual(self.rh.runs_per_config[2], 2) def test_race_challenger_large(self): ''' test _race_challenger using solution_quality ''' def target(x): return 1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), deterministic=True) for i in range(10): self.rh.add(config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=12345, additional_info=None) # tie on first instances and then challenger should always win # and be returned as inc inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # self.assertTrue(False) self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1, self.rh.get_cost(self.config2)) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2) self.assertEqual(len(runs), 10) def test_race_challenger_large_blocked_seed(self): ''' test _race_challenger whether seeds are blocked for challenger runs ''' def target(x): return 1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), deterministic=False) for i in range(10): self.rh.add(config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) # tie on first instances and then challenger should always win # and be returned as inc inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # self.assertTrue(False) self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1, self.rh.get_cost(self.config2)) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2) self.assertEqual(len(runs), 10) seeds = sorted([r.seed for r in runs]) self.assertEqual(seeds, list(range(10)), seeds) def test_add_inc_run_det(self): ''' test _add_inc_run() ''' def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) # since we assume deterministic=1, # the second call should not add any more runs # given only one instance intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) def test_add_inc_run_nondet(self): ''' test _add_inc_run() ''' def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], deterministic=False) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 2, self.rh.data) runs = self.rh.get_runs_for_config(config=self.config1) # exactly one run on each instance self.assertIn(1, [runs[0].instance, runs[1].instance]) self.assertIn(2, [runs[0].instance, runs[1].instance]) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 3, self.rh.data) def test_adaptive_capping(self): ''' test _adapt_cutoff() ''' intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(5)), deterministic=False) for i in range(5): self.rh.add(config=self.config1, cost=i + 1, time=i + 1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) for i in range(3): self.rh.add(config=self.config2, cost=i + 1, time=i + 1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) inst_seed_pairs = self.rh.get_runs_for_config(self.config1) # cost used by incumbent for going over all runs in inst_seed_pairs inc_sum_cost = sum_cost(config=self.config1, instance_seed_pairs=inst_seed_pairs, run_history=self.rh) cutoff = intensifier._adapt_cutoff(challenger=self.config2, incumbent=self.config1, run_history=self.rh, inc_sum_cost=inc_sum_cost) # 15*1.2 - 6 self.assertEqual(cutoff, 12) intensifier.cutoff = 5 cutoff = intensifier._adapt_cutoff(challenger=self.config2, incumbent=self.config1, run_history=self.rh, inc_sum_cost=inc_sum_cost) # scenario cutoff self.assertEqual(cutoff, 5)
def process_results(self, run_info: RunInfo, incumbent: typing.Optional[Configuration], run_history: RunHistory, time_bound: float, result: RunValue, log_traj: bool = True, ) -> \ typing.Tuple[Configuration, float]: """ The intensifier stage will be updated based on the results/status of a configuration execution. During intensification, the following can happen: -> Challenger raced against incumbent -> Also, during a challenger run, a capped exception can be triggered, where no racer post processing is needed -> A run on the incumbent for more confidence needs to be processed, IntensifierStage.PROCESS_INCUMBENT_RUN -> The first run results need to be processed (PROCESS_FIRST_CONFIG_RUN) At the end of any run, checks are done to move to a new iteration. Parameters ---------- run_info : RunInfo A RunInfo containing the configuration that was evaluated incumbent : typing.Optional[Configuration] best configuration so far, None in 1st run run_history : RunHistory stores all runs we ran so far if False, an evaluated configuration will not be generated again time_bound : float time in [sec] available to perform intensify result: RunValue Contain the result (status and other methadata) of exercising a challenger/incumbent. log_traj: bool whether to log changes of incumbents in trajectory Returns ------- incumbent: Configuration() current (maybe new) incumbent configuration inc_perf: float empirical performance of incumbent configuration """ if self.stage == IntensifierStage.PROCESS_FIRST_CONFIG_RUN: if incumbent is None: self.logger.info("First run, no incumbent provided;" " challenger is assumed to be the incumbent") incumbent = run_info.config if self.stage in [ IntensifierStage.PROCESS_INCUMBENT_RUN, IntensifierStage.PROCESS_FIRST_CONFIG_RUN ]: self._ta_time += result.time self.num_run += 1 self._process_inc_run( incumbent=incumbent, run_history=run_history, log_traj=log_traj, ) else: self.num_run += 1 self.num_chall_run += 1 if result.status == StatusType.CAPPED: # move on to the next iteration self.logger.debug("Challenger itensification timed out due " "to adaptive capping.") self.stage = IntensifierStage.RUN_INCUMBENT else: self._ta_time += result.time incumbent = self._process_racer_results( challenger=run_info.config, incumbent=incumbent, run_history=run_history, log_traj=log_traj, ) self.elapsed_time += (result.endtime - result.starttime) # check if 1 intensification run is complete - line 18 # this is different to regular SMAC as it requires at least successful challenger run, # which is necessary to work on a fixed grid of configurations. if (self.stage == IntensifierStage.RUN_INCUMBENT and self._chall_indx >= self.min_chall and self.num_chall_run > 0): if self.num_run > self.run_limit: self.logger.info("Maximum #runs for intensification reached") self._next_iteration() if not self.use_ta_time_bound and self.elapsed_time - time_bound >= 0: self.logger.info( "Wallclock time limit for intensification reached " "(used: %f sec, available: %f sec)", self.elapsed_time, time_bound) self._next_iteration() elif self._ta_time - time_bound >= 0: self.logger.info( "TA time limit for intensification reached (used: %f sec, available: %f sec)", self._ta_time, time_bound) self._next_iteration() inc_perf = run_history.get_cost(incumbent) return incumbent, inc_perf
class CAVE(object): """ """ def __init__(self, folders: typing.List[str], output: str, ta_exec_dir: Union[str, None] = None, missing_data_method: str = 'epm', max_pimp_samples: int = -1, fanova_pairwise=True): """ Initialize CAVE facade to handle analyzing, plotting and building the report-page easily. During initialization, the analysis-infrastructure is built and the data is validated, meaning the overall best incumbent is found and default+incumbent are evaluated for all instances for all runs, by default using an EPM. The class holds two runhistories: self.original_rh -> only contains runs from the actual data self.validated_rh -> contains original runs and epm-predictions for all incumbents The analyze()-method performs an analysis and outputs a report.html. Arguments --------- folders: list<strings> paths to relevant SMAC runs output: string output for cave to write results (figures + report) ta_exec_dir: string execution directory for target algorithm (to find instance.txt, ..) missing_data_method: string from [validation, epm], how to estimate missing runs """ self.logger = logging.getLogger("cave.cavefacade") self.logger.debug("Folders: %s", str(folders)) self.ta_exec_dir = ta_exec_dir # Create output if necessary self.output = output self.logger.info("Saving results to %s", self.output) if not os.path.exists(output): self.logger.debug("Output-dir %s does not exist, creating", self.output) os.makedirs(output) if not os.path.exists(os.path.join(self.output, "debug")): os.makedirs(os.path.join(self.output, "debug")) # Log to file logger = logging.getLogger() handler = logging.FileHandler( os.path.join(self.output, "debug/debug.log"), "w") handler.setLevel(logging.DEBUG) logger.addHandler(handler) # Global runhistory combines all actual runs of individual SMAC-runs # We save the combined (unvalidated) runhistory to disk, so we can use it later on. # We keep the validated runhistory (with as many runs as possible) in # memory. The distinction is made to avoid using runs that are # only estimated using an EPM for further EPMs or to handle runs # validated on different hardware (depending on validation-method). self.original_rh = RunHistory(average_cost) self.validated_rh = RunHistory(average_cost) # Save all relevant SMAC-runs in a list self.runs = [] for folder in folders: try: self.logger.debug("Collecting data from %s.", folder) self.runs.append(SMACrun(folder, ta_exec_dir)) except Exception as err: self.logger.warning( "Folder %s could not be loaded, failed " "with error message: %s", folder, err) continue if not len(self.runs): raise ValueError( "None of the specified SMAC-folders could be loaded.") # Use scenario of first run for general purposes (expecting they are all the same anyway!) self.scenario = self.runs[0].solver.scenario # Update global runhistory with all available runhistories self.logger.debug("Update original rh with all available rhs!") runhistory_fns = [ os.path.join(run.folder, "runhistory.json") for run in self.runs ] for rh_file in runhistory_fns: self.original_rh.update_from_json(rh_file, self.scenario.cs) self.logger.debug( 'Combined number of Runhistory data points: %d. ' '# Configurations: %d. # Runhistories: %d', len(self.original_rh.data), len(self.original_rh.get_all_configs()), len(runhistory_fns)) self.original_rh.save_json( os.path.join(self.output, "combined_rh.json")) # Validator for a) validating with epm, b) plot over time # Initialize without trajectory self.validator = Validator(self.scenario, None, None) # Estimate missing costs for [def, inc1, inc2, ...] self.complete_data(method=missing_data_method) self.best_run = min( self.runs, key=lambda run: self.validated_rh.get_cost(run.solver.incumbent)) self.default = self.scenario.cs.get_default_configuration() self.incumbent = self.best_run.solver.incumbent self.logger.debug("Overall best run: %s, with incumbent: %s", self.best_run.folder, self.incumbent) # Following variable determines whether a distinction is made # between train and test-instances (e.g. in plotting) self.train_test = bool(self.scenario.train_insts != [None] and self.scenario.test_insts != [None]) self.analyzer = Analyzer(self.original_rh, self.validated_rh, self.default, self.incumbent, self.train_test, self.scenario, self.validator, self.output, max_pimp_samples, fanova_pairwise) self.builder = HTMLBuilder(self.output, "CAVE") # Builder for html-website self.website = OrderedDict([]) def complete_data(self, method="epm"): """Complete missing data of runs to be analyzed. Either using validation or EPM. """ with changedir(self.ta_exec_dir if self.ta_exec_dir else '.'): self.logger.info("Completing data using %s.", method) path_for_validated_rhs = os.path.join(self.output, "validated_rhs") for run in self.runs: self.validator.traj = run.traj if method == "validation": # TODO determine # repetitions new_rh = self.validator.validate( 'def+inc', 'train+test', 1, -1, runhistory=self.original_rh) elif method == "epm": new_rh = self.validator.validate_epm( 'def+inc', 'train+test', 1, runhistory=self.original_rh) else: raise ValueError("Missing data method illegal (%s)", method) self.validator.traj = None # Avoid usage-mistakes self.validated_rh.update(new_rh) def analyze(self, performance=True, cdf=True, scatter=True, confviz=True, param_importance=['forward_selection', 'ablation', 'fanova'], feature_analysis=[ "box_violin", "correlation", "feat_importance", "clustering", "feature_cdf" ], parallel_coordinates=True, cost_over_time=True, algo_footprint=True): """Analyze the available data and build HTML-webpage as dict. Save webpage in 'self.output/CAVE/report.html'. Analyzing is performed with the analyzer-instance that is initialized in the __init__ Parameters ---------- performance: bool whether to calculate par10-values cdf: bool whether to plot cdf scatter: bool whether to plot scatter confviz: bool whether to perform configuration visualization param_importance: List[str] containing methods for parameter importance feature_analysis: List[str] containing methods for feature analysis parallel_coordinates: bool whether to plot parallel coordinates cost_over_time: bool whether to plot cost over time algo_footprint: bool whether to plot algorithm footprints """ # Check arguments for p in param_importance: if p not in [ 'forward_selection', 'ablation', 'fanova', 'incneighbor' ]: raise ValueError( "%s not a valid option for parameter " "importance!", p) for f in feature_analysis: if f not in [ "box_violin", "correlation", "importance", "clustering", "feature_cdf" ]: raise ValueError("%s not a valid option for feature analysis!", f) # Start analysis overview = self.analyzer.create_overview_table(self.best_run.folder) self.website["Meta Data"] = {"table": overview} compare_config = self.analyzer.config_to_html(self.default, self.incumbent) self.website["Best configuration"] = {"table": compare_config} ########## PERFORMANCE ANALYSIS self.website["Performance Analysis"] = OrderedDict() if performance: performance_table = self.analyzer.create_performance_table( self.default, self.incumbent) self.website["Performance Analysis"]["Performance Table"] = { "table": performance_table } if cdf: cdf_path = self.analyzer.plot_cdf() self.website["Performance Analysis"][ "empirical Cumulative Distribution Function (eCDF)"] = { "figure": cdf_path } if scatter and (self.scenario.train_insts != [[None]]): scatter_path = self.analyzer.plot_scatter() self.website["Performance Analysis"]["Scatterplot"] = { "figure": scatter_path } elif scatter: self.logger.info( "Scatter plot desired, but no instances available.") # Build report before time-consuming analysis self.build_website() if algo_footprint and self.scenario.feature_dict: algorithms = {self.default: "default", self.incumbent: "incumbent"} # Add all available incumbents to test portfolio strategy #for r in self.runs: # if not r.get_incumbent() in algorithms: # algorithms[r.get_incumbent()] = str(self.runs.index(r)) algo_footprint_plots = self.analyzer.plot_algorithm_footprint( algorithms) self.website["Performance Analysis"][ "Algorithm Footprints"] = OrderedDict() for p in algo_footprint_plots: header = os.path.splitext(os.path.split(p)[1])[0] # algo name self.website["Performance Analysis"]["Algorithm Footprints"][ header] = { "figure": p, "tooltip": get_tooltip("Algorithm Footprints") + ": " + header } self.build_website() ########### Configurator's behavior self.website["Configurator's behavior"] = OrderedDict() if confviz: if self.scenario.feature_array is None: self.scenario.feature_array = np.array([[]]) # Sort runhistories and incs wrt cost incumbents = [r.solver.incumbent for r in self.runs] trajectories = [r.traj for r in self.runs] runhistories = [r.runhistory for r in self.runs] costs = [self.validated_rh.get_cost(i) for i in incumbents] costs, incumbents, runhistories, trajectories = ( list(t) for t in zip( *sorted(zip(costs, incumbents, runhistories, trajectories), key=lambda x: x[0]))) incumbents = list(map(lambda x: x['incumbent'], trajectories[0])) confviz_script = self.analyzer.plot_confviz( incumbents, runhistories) self.website["Configurator's behavior"][ "Configurator Footprint"] = { "table": confviz_script } elif confviz: self.logger.info("Configuration visualization desired, but no " "instance-features available.") self.build_website() if cost_over_time: cost_over_time_path = self.analyzer.plot_cost_over_time( self.best_run.traj, self.validator) self.website["Configurator's behavior"]["Cost over time"] = { "figure": cost_over_time_path } self.build_website() self.parameter_importance(ablation='ablation' in param_importance, fanova='fanova' in param_importance, forward_selection='forward_selection' in param_importance, incneighbor='incneighbor' in param_importance) self.build_website() if parallel_coordinates: # Should be after parameter importance, if performed. n_params = 6 parallel_path = self.analyzer.plot_parallel_coordinates(n_params) self.website["Configurator's behavior"]["Parallel Coordinates"] = { "figure": parallel_path } self.build_website() if self.scenario.feature_dict: self.feature_analysis(box_violin='box_violin' in feature_analysis, correlation='correlation' in feature_analysis, clustering='clustering' in feature_analysis, importance='importance' in feature_analysis) else: self.logger.info('No feature analysis possible') self.logger.info("CAVE finished. Report is located in %s", os.path.join(self.output, 'report.html')) self.build_website() def parameter_importance(self, ablation=False, fanova=False, forward_selection=False, incneighbor=False): """Perform the specified parameter importance procedures. """ # PARAMETER IMPORTANCE if (ablation or forward_selection or fanova or incneighbor): self.website["Parameter Importance"] = OrderedDict() sum_ = 0 if fanova: sum_ += 1 table, plots, pair_plots = self.analyzer.fanova(self.incumbent) self.website["Parameter Importance"]["fANOVA"] = OrderedDict() self.website["Parameter Importance"]["fANOVA"]["Importance"] = { "table": table } # Insert plots (the received plots is a dict, mapping param -> path) self.website["Parameter Importance"]["fANOVA"][ "Marginals"] = OrderedDict([]) for param, plot in plots.items(): self.website["Parameter Importance"]["fANOVA"]["Marginals"][ param] = { "figure": plot } if pair_plots: self.website["Parameter Importance"]["fANOVA"][ "PairwiseMarginals"] = OrderedDict([]) for param, plot in pair_plots.items(): self.website["Parameter Importance"]["fANOVA"][ "PairwiseMarginals"][param] = { "figure": plot } if ablation: sum_ += 1 self.logger.info("Ablation...") self.analyzer.parameter_importance("ablation", self.incumbent, self.output) ablationpercentage_path = os.path.join(self.output, "ablationpercentage.png") ablationperformance_path = os.path.join(self.output, "ablationperformance.png") self.website["Parameter Importance"]["Ablation"] = { "figure": [ablationpercentage_path, ablationperformance_path] } if forward_selection: sum_ += 1 self.logger.info("Forward Selection...") self.analyzer.parameter_importance("forward-selection", self.incumbent, self.output) f_s_barplot_path = os.path.join(self.output, "forward selection-barplot.png") f_s_chng_path = os.path.join(self.output, "forward selection-chng.png") self.website["Parameter Importance"]["Forward Selection"] = { "figure": [f_s_barplot_path, f_s_chng_path] } if incneighbor: sum_ += 1 self.logger.info("Local EPM-predictions around incumbent...") plots = self.analyzer.local_epm_plots() self.website["Parameter Importance"][ "Local Parameter Importance (LPI)"] = OrderedDict([]) for param, plot in plots.items(): self.website["Parameter Importance"][ "Local Parameter Importance (LPI)"][param] = { "figure": plot } if sum_: of = os.path.join(self.output, 'pimp.tex') self.logger.info('Creating pimp latex table at %s' % of) self.analyzer.pimp.table_for_comparison(self.analyzer.evaluators, of, style='latex') def feature_analysis(self, box_violin=False, correlation=False, clustering=False, importance=False): if not (box_violin or correlation or clustering or importance): self.logger.debug("No feature analysis.") return # FEATURE ANALYSIS (ASAPY) # TODO make the following line prettier # TODO feat-names from scenario? in_reader = InputReader() feat_fn = self.scenario.feature_fn if not self.scenario.feature_names: with changedir(self.ta_exec_dir if self.ta_exec_dir else '.'): if not feat_fn or not os.path.exists(feat_fn): self.logger.warning( "Feature Analysis needs valid feature " "file! Either {} is not a valid " "filename or features are not saved in " "the scenario.") self.logger.error("Skipping Feature Analysis.") return else: feat_names = in_reader.read_instance_features_file( self.scenario.feature_fn)[0] else: feat_names = copy.deepcopy(self.scenario.feature_names) self.website["Feature Analysis"] = OrderedDict([]) # feature importance using forward selection if importance: self.website["Feature Analysis"][ "Feature Importance"] = OrderedDict() imp, plots = self.analyzer.feature_importance() imp = DataFrame(data=list(imp.values()), index=list(imp.keys()), columns=["Error"]) imp = imp.to_html() # this is a table with the values in html self.website["Feature Analysis"]["Feature Importance"]["Table"] = { "table": imp } for p in plots: name = os.path.splitext(os.path.basename(p))[0] self.website["Feature Analysis"]["Feature Importance"][ name] = { "figure": p } # box and violin plots if box_violin: name_plots = self.analyzer.feature_analysis( 'box_violin', feat_names) self.website["Feature Analysis"][ "Violin and Box Plots"] = OrderedDict() for plot_tuple in name_plots: key = "%s" % (plot_tuple[0]) self.website["Feature Analysis"]["Violin and Box Plots"][ key] = { "figure": plot_tuple[1] } # correlation plot if correlation: correlation_plot = self.analyzer.feature_analysis( 'correlation', feat_names) if correlation_plot: self.website["Feature Analysis"]["Correlation"] = { "figure": correlation_plot } # cluster instances in feature space if clustering: cluster_plot = self.analyzer.feature_analysis( 'clustering', feat_names) self.website["Feature Analysis"]["Clustering"] = { "figure": cluster_plot } self.build_website() def build_website(self): self.builder.generate_html(self.website)
def _process_racer_results( self, challenger: Configuration, incumbent: Configuration, run_history: RunHistory, log_traj: bool = True, ) -> typing.Optional[Configuration]: """Process the result of a racing configuration against the current incumbent. Might propose a new incumbent. Parameters ---------- challenger : Configuration Configuration which challenges incumbent incumbent : Configuration Best configuration so far run_history : RunHistory Stores all runs we ran so far Returns ------- new_incumbent: typing.Optional[Configuration] Either challenger or incumbent """ chal_runs = run_history.get_runs_for_config( challenger, only_max_observed_budget=True) chal_perf = run_history.get_cost(challenger) # if all <instance, seed> have been run, compare challenger performance if not self.to_run: new_incumbent = self._compare_configs(incumbent=incumbent, challenger=challenger, run_history=run_history, log_traj=log_traj) # update intensification stage if new_incumbent == incumbent: # move on to the next iteration self.stage = IntensifierStage.RUN_INCUMBENT self.continue_challenger = False self.logger.debug( 'Estimated cost of challenger on %d runs: %.4f, but worse than incumbent', len(chal_runs), chal_perf) elif new_incumbent == challenger: # New incumbent found incumbent = challenger self.continue_challenger = False # compare against basis configuration if provided, else go to next iteration if self.always_race_against and \ self.always_race_against != challenger: self.stage = IntensifierStage.RUN_BASIS else: self.stage = IntensifierStage.RUN_INCUMBENT self.logger.debug( 'Estimated cost of challenger on %d runs: %.4f, becomes new incumbent', len(chal_runs), chal_perf) else: # Line 17 # challenger is not worse, continue self.N = 2 * self.N self.continue_challenger = True self.logger.debug( 'Estimated cost of challenger on %d runs: %.4f, adding %d runs to the queue', len(chal_runs), chal_perf, self.N / 2) else: self.logger.debug( 'Estimated cost of challenger on %d runs: %.4f, still %d runs to go (continue racing)', len(chal_runs), chal_perf, len(self.to_run)) return incumbent
def _race_challenger(self, challenger: Configuration, incumbent: Configuration, run_history: RunHistory, aggregate_func: typing.Callable): ''' aggressively race challenger against incumbent Parameters ---------- challenger : Configuration configuration which challenges incumbent incumbent : Configuration best configuration so far run_history : RunHistory stores all runs we ran so far aggregate_func: typing.Callable aggregate performance across instances Returns ------- new_incumbent: Configuration either challenger or incumbent ''' # at least one run of challenger # to increase chall_indx counter first_run = False inc_perf = run_history.get_cost(incumbent) learning_curve = [] self._num_run += 1 self._chall_indx += 1 pc = None for epoch in range(self.max_epochs): status, cost, time, add_info = self.tae_runner.start( config=challenger, instance=None, seed=0, cutoff=2**32 - 1, instance_specific=None, pc=pc) try: pc = add_info["model"] except KeyError: # model building failed, e.g. because of nan break learning_curve.append(cost) if len(self.learning_curves) > 10 and epoch > self.max_epochs / 4: seen_curves = np.array(self.learning_curves)[:, epoch] if cost > np.median(seen_curves): self.logger.info("Abort run (%f vs %f)" % (cost, np.median(seen_curves))) break # delete model in runhistory to be more memory efficient chall_id = run_history.config_ids[challenger] runkey = RunKey(chall_id, None, 0) runvalue = run_history.data[runkey] try: del runvalue.additional_info["model"] except KeyError: pass if epoch == self.max_epochs - 1: self.learning_curves.append(learning_curve) chal_perf = cost if cost < inc_perf: self.logger.info( "Challenger (%.4f) is better than incumbent (%.4f)" % (chal_perf, inc_perf)) # Show changes in the configuration params = sorted([(param, incumbent[param], challenger[param]) for param in challenger.keys()]) self.logger.info("Changes in incumbent:") for param in params: if param[1] != param[2]: self.logger.info(" %s : %r -> %r" % (param)) else: self.logger.debug(" %s remains unchanged: %r" % (param[0], param[1])) incumbent = challenger self.stats.inc_changed += 1 self.traj_logger.add_entry(train_perf=chal_perf, incumbent_id=self.stats.inc_changed, incumbent=challenger) else: self.logger.debug( "Incumbent (%.4f) is better than challenger (%.4f)" % (inc_perf, chal_perf)) return incumbent
def intensify(self, challengers: typing.List[Configuration], incumbent: Configuration, run_history: RunHistory, aggregate_func: typing.Callable, time_bound: int = MAXINT): ''' running intensification to determine the incumbent configuration Side effect: adds runs to run_history Implementation of Procedure 2 in Hutter et al. (2011). Parameters ---------- challengers : typing.List[Configuration] promising configurations incumbent : Configuration best configuration so far run_history : RunHistory stores all runs we ran so far aggregate_func: typing.Callable aggregate performance across instances time_bound : int, optional (default=2 ** 31 - 1) time in [sec] available to perform intensify Returns ------- incumbent: Configuration() current (maybe new) incumbent configuration inc_perf: float empirical performance of incumbent configuration ''' self.start_time = time.time() if time_bound < 0.01: raise ValueError("time_bound must be >= 0.01") self._num_run = 0 self._chall_indx = 0 # Line 1 + 2 for challenger in challengers: if challenger == incumbent: self.logger.warning( "Challenger was the same as the current incumbent; Skipping challenger" ) continue self.logger.debug("Intensify on %s", challenger) if hasattr(challenger, 'origin'): self.logger.debug("Configuration origin: %s", challenger.origin) try: # Lines 3-7 self._add_inc_run(incumbent=incumbent, run_history=run_history) # Lines 8-17 incumbent = self._race_challenger( challenger=challenger, incumbent=incumbent, run_history=run_history, aggregate_func=aggregate_func) except BudgetExhaustedException: # We return incumbent, SMBO stops due to its own budget checks inc_perf = run_history.get_cost(incumbent) self.logger.debug("Budget exhausted; Return incumbent") return incumbent, inc_perf if self._chall_indx > 1 and self._num_run > self.run_limit: self.logger.debug("Maximum #runs for intensification reached") break elif self._chall_indx > 1 and time.time( ) - self.start_time - time_bound >= 0: self.logger.debug("Timelimit for intensification reached (" "used: %f sec, available: %f sec)" % (time.time() - self.start_time, time_bound)) break # output estimated performance of incumbent inc_runs = run_history.get_runs_for_config(incumbent) inc_perf = aggregate_func(incumbent, run_history, inc_runs) self.logger.info( "Updated estimated performance of incumbent on %d runs: %.4f" % (len(inc_runs), inc_perf)) self.stats.update_average_configs_per_intensify( n_configs=self._chall_indx) return incumbent, inc_perf
class SMBO(BaseSolver): def __init__(self, scenario, tae_runner=None, acquisition_function=None, model=None, runhistory2epm=None, stats=None, rng=None): ''' Interface that contains the main Bayesian optimization loop Parameters ---------- scenario: smac.scenario.scenario.Scenario Scenario object tae_runner: object object that implements the following method to call the target algorithm (or any other arbitrary function): run(self, config) If not set, it will be initialized with the tae.ExecuteTARunOld() acquisition_function : AcquisitionFunction Object that implements the AbstractAcquisitionFunction. Will use EI if not set. model : object Model that implements train() and predict(). Will use a RandomForest if not set. runhistory2epm : RunHistory2EMP Object that implements the AbstractRunHistory2EPM. If None, will use RunHistory2EPM4Cost if objective is cost or RunHistory2EPM4LogCost if objective is runtime. stats: Stats optional stats object rng: numpy.random.RandomState Random number generator ''' if stats: self.stats = stats else: self.stats = Stats(scenario) self.runhistory = RunHistory() self.logger = logging.getLogger("smbo") if rng is None: self.num_run = np.random.randint(1234567980) self.rng = np.random.RandomState(seed=self.num_run) elif isinstance(rng, int): self.num_run = rng self.rng = np.random.RandomState(seed=rng) elif isinstance(rng, np.random.RandomState): self.num_run = rng.randint(1234567980) self.rng = rng else: raise TypeError('Unknown type %s for argument rng. Only accepts ' 'None, int or np.random.RandomState' % str(type(rng))) self.scenario = scenario self.config_space = scenario.cs self.traj_logger = TrajLogger(output_dir=self.scenario.output_dir, stats=self.stats) self.types = get_types(self.config_space, scenario.feature_array) if model is None: self.model = RandomForestWithInstances( self.types, scenario.feature_array, seed=self.rng.randint(1234567980)) else: self.model = model if acquisition_function is None: self.acquisition_func = EI(self.model) else: self.acquisition_func = acquisition_function self.local_search = LocalSearch(self.acquisition_func, self.config_space) self.incumbent = None if tae_runner is None: self.executor = ExecuteTARunOld(ta=scenario.ta, stats=self.stats, run_obj=scenario.run_obj, par_factor=scenario.par_factor) else: self.executor = tae_runner self.inten = Intensifier( executor=self.executor, stats=self.stats, traj_logger=self.traj_logger, instances=self.scenario.train_insts, cutoff=self.scenario.cutoff, deterministic=self.scenario.deterministic, run_obj_time=self.scenario.run_obj == "runtime", instance_specifics=self.scenario.instance_specific) num_params = len(self.config_space.get_hyperparameters()) self.objective = average_cost if self.scenario.run_obj == "runtime": if runhistory2epm is None: # if we log the performance data, # the RFRImputator will already get # log transform data from the runhistory cutoff = np.log10(self.scenario.cutoff) threshold = np.log10(self.scenario.cutoff * self.scenario.par_factor) imputor = RFRImputator(cs=self.config_space, rs=self.rng, cutoff=cutoff, threshold=threshold, model=self.model, change_threshold=0.01, max_iter=10) self.rh2EPM = RunHistory2EPM4LogCost(scenario=self.scenario, num_params=num_params, success_states=[ StatusType.SUCCESS, ], impute_censored_data=True, impute_state=[ StatusType.TIMEOUT, ], imputor=imputor) else: self.rh2EPM = runhistory2epm elif self.scenario.run_obj == 'quality': if runhistory2epm is None: self.rh2EPM = RunHistory2EPM4Cost\ (scenario=self.scenario, num_params=num_params, success_states=[StatusType.SUCCESS, ], impute_censored_data=False, impute_state=None) else: self.rh2EPM = runhistory2epm else: raise ValueError('Unknown run objective: %s. Should be either ' 'quality or runtime.' % self.scenario.run_obj) def run_initial_design(self): ''' runs algorithm runs for a initial design; default implementation: running the default configuration on a random instance-seed pair Side effect: adds runs to self.runhistory Returns ------- incumbent: Configuration() initial incumbent configuration ''' default_conf = self.config_space.get_default_configuration() self.incumbent = default_conf # add this incumbent right away to have an entry to time point 0 self.traj_logger.add_entry(train_perf=2**31, incumbent_id=1, incumbent=self.incumbent) rand_inst_id = self.rng.randint(0, len(self.scenario.train_insts)) # ignore instance specific values rand_inst = self.scenario.train_insts[rand_inst_id] if self.scenario.deterministic: initial_seed = 0 else: initial_seed = random.randint(0, MAXINT) status, cost, runtime, additional_info = self.executor.start( default_conf, instance=rand_inst, cutoff=self.scenario.cutoff, seed=initial_seed, instance_specific=self.scenario.instance_specific.get( rand_inst, "0")) if status in [StatusType.CRASHED or StatusType.ABORT]: self.logger.critical("First run crashed -- Abort") sys.exit(1) self.runhistory.add(config=default_conf, cost=cost, time=runtime, status=status, instance_id=rand_inst, seed=initial_seed, additional_info=additional_info) defaul_inst_seeds = set( self.runhistory.get_runs_for_config(default_conf)) default_perf = self.objective(default_conf, self.runhistory, defaul_inst_seeds) self.runhistory.update_cost(default_conf, default_perf) self.stats.inc_changed += 1 # first incumbent self.traj_logger.add_entry(train_perf=default_perf, incumbent_id=self.stats.inc_changed, incumbent=self.incumbent) return default_conf def run(self, max_iters=10): ''' Runs the Bayesian optimization loop for max_iters iterations Parameters ---------- max_iters: int The maximum number of iterations Returns ---------- incumbent: np.array(1, H) The best found configuration ''' self.stats.start_timing() #self.runhistory = RunHisory() self.incumbent = self.run_initial_design() # Main BO loop iteration = 1 while True: if self.scenario.shared_model: pSMAC.read(run_history=self.runhistory, output_directory=self.scenario.output_dir, configuration_space=self.config_space, logger=self.logger) start_time = time.time() X, Y = self.rh2EPM.transform(self.runhistory) self.logger.debug("Search for next configuration") # get all found configurations sorted according to acq challengers = self.choose_next(X, Y) time_spend = time.time() - start_time logging.debug( "Time spend to choose next configurations: %.2f sec" % (time_spend)) self.logger.debug("Intensify") self.incumbent, inc_perf = self.inten.intensify( challengers=challengers, incumbent=self.incumbent, run_history=self.runhistory, objective=self.objective, time_bound=max(0.01, time_spend)) # TODO: Write run history into database if self.scenario.shared_model: pSMAC.write(run_history=self.runhistory, output_directory=self.scenario.output_dir, num_run=self.num_run) if iteration == max_iters: break iteration += 1 logging.debug( "Remaining budget: %f (wallclock), %f (ta costs), %f (target runs)" % (self.stats.get_remaing_time_budget(), self.stats.get_remaining_ta_budget(), self.stats.get_remaining_ta_runs())) if self.stats.is_budget_exhausted(): break self.stats.print_stats(debug_out=True) return self.incumbent def choose_next(self, X, Y, num_interleaved_random=1010, num_configurations_by_random_search_sorted=1000, num_configurations_by_local_search=10): """Choose next candidate solution with Bayesian optimization. Parameters ---------- X : (N, D) numpy array Each row contains a configuration and one set of instance features. Y : (N, O) numpy array The function values for each configuration instance pair. Returns ------- list List of 2020 suggested configurations to evaluate. """ self.model.train(X, Y) if self.runhistory.empty(): incumbent_value = 0.0 elif self.incumbent is None: # TODO try to calculate an incumbent from the runhistory! incumbent_value = 0.0 else: incumbent_value = self.runhistory.get_cost(self.incumbent) self.acquisition_func.update(model=self.model, eta=incumbent_value) # Remove dummy acquisition function value next_configs_by_random_search = [ x[1] for x in self._get_next_by_random_search( num_points=num_interleaved_random) ] # Get configurations sorted by EI next_configs_by_random_search_sorted = \ self._get_next_by_random_search( num_configurations_by_random_search_sorted, _sorted=True) next_configs_by_local_search = \ self._get_next_by_local_search(num_configurations_by_local_search) next_configs_by_acq_value = next_configs_by_random_search_sorted + \ next_configs_by_local_search next_configs_by_acq_value.sort(reverse=True, key=lambda x: x[0]) self.logger.debug( "First 10 acq func values of selected configurations: %s" % (str([_[0] for _ in next_configs_by_acq_value[:10]]))) next_configs_by_acq_value = [_[1] for _ in next_configs_by_acq_value] challengers = list( itertools.chain(*zip(next_configs_by_acq_value, next_configs_by_random_search))) return challengers def _get_next_by_random_search(self, num_points=1000, _sorted=False): """Get candidate solutions via local search. Parameters ---------- num_points : int, optional (default=10) Number of local searches and returned values. _sorted : bool, optional (default=True) Whether to sort the candidate solutions by acquisition function value. Returns ------- list : (acquisition value, Candidate solutions) """ rand_configs = self.config_space.sample_configuration(size=num_points) if _sorted: imputed_rand_configs = map(ConfigSpace.util.impute_inactive_values, rand_configs) imputed_rand_configs = [ x.get_array() for x in imputed_rand_configs ] imputed_rand_configs = np.array(imputed_rand_configs, dtype=np.float64) acq_values = self.acquisition_func(imputed_rand_configs) # From here # http://stackoverflow.com/questions/20197990/how-to-make-argsort-result-to-be-random-between-equal-values random = self.rng.rand(len(acq_values)) # Last column is primary sort key! indices = np.lexsort((random.flatten(), acq_values.flatten())) for i in range(len(rand_configs)): rand_configs[i].origin = 'Random Search (sorted)' # Cannot use zip here because the indices array cannot index the # rand_configs list, because the second is a pure python list return [(acq_values[ind][0], rand_configs[ind]) for ind in indices[::-1]] else: for i in range(len(rand_configs)): rand_configs[i].origin = 'Random Search' return [(0, rand_configs[i]) for i in range(len(rand_configs))] def _get_next_by_local_search(self, num_points=10): """Get candidate solutions via local search. In case acquisition function values tie, these will be broken randomly. Parameters ---------- num_points : int, optional (default=10) Number of local searches and returned values. Returns ------- list : (acquisition value, Candidate solutions), ordered by their acquisition function value """ configs_acq = [] # Start N local search from different random start points for i in range(num_points): if i == 0 and self.incumbent is not None: start_point = self.incumbent else: start_point = self.config_space.sample_configuration() configuration, acq_val = self.local_search.maximize(start_point) configuration.origin = 'Local Search' configs_acq.append((acq_val[0][0], configuration)) # shuffle for random tie-break random.shuffle(configs_acq, self.rng.rand) # sort according to acq value # and return n best configurations configs_acq.sort(reverse=True, key=lambda x: x[0]) return configs_acq
def __init__( self, original_rh: RunHistory, validated_rh: RunHistory, validator: Validator, scenario: Scenario, default: Configuration, incumbent: Configuration, param_imp: Union[None, Dict[str, float]], params: Union[int, List[str]], n_configs: int, pc_sort_by: str, output_dir: str, cs: ConfigurationSpace, runtime: bool = False, max_runs_epm: int = 3000000, ): """This function prepares the data from a SMAC-related format (using runhistories and parameters) to a more general format (using a dataframe). The resulting dataframe is passed to the parallel_coordinates-routine Parameters ---------- original_rh: RunHistory runhistory that should contain only runs that were executed during search validated_rh: RunHistory runhistory that may contain as many runs as possible, also external runs. this runhistory will be used to build the EPM validator: Validator validator to be used to estimate costs for configurations scenario: Scenario scenario object to take instances from default, incumbent: Configuration default and incumbent, they will surely be displayed param_imp: Union[None, Dict[str->float] if given, maps parameter-names to importance params: Union[int, List[str]] either directly the parameters to displayed or the number of parameters (will try to define the most important ones n_configs: int number of configs to be plotted pc_sort_by: str defines the pimp-method by which to choose the plotted parameters max_runs_epm: int maximum number of runs to train the epm with. this should prevent MemoryErrors output_dir: str output directory for plots cs: ConfigurationSpace parameter configuration space to be visualized runtime: boolean runtime will be on logscale """ self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) self.error = None self.default = default self.param_imp = param_imp self.cs = cs # Sorting by importance, if possible (choose first executed parameter-importance) self.method, self.importance = "", {} if pc_sort_by == 'all': self.logger.debug("Sorting by average importance") self.method = 'average' for m, i in self.param_imp.items(): if i: for p, imp in i.items(): if p in self.importance: self.importance[p].append(imp) else: self.importance[p] = [imp] self.importance = { k: sum(v) / len(v) for k, v in self.importance.items() } elif pc_sort_by in self.param_imp: self.method, self.importance = pc_sort_by, self.param_imp[ pc_sort_by] else: self.logger.debug("%s not evaluated.. choosing at random from: %s", pc_sort_by, str(list(self.param_imp.keys()))) for m, i in self.param_imp.items(): if i: self.method, self.importance = m, i break self.hp_names = sorted( [hp for hp in self.cs.get_hyperparameter_names()], key=lambda x: self.importance.get(x, 0), reverse=True) self.logger.debug("Sorted hp's by method \'%s\': %s", self.method, str(self.hp_names)) # To be set self.plots = [] # Define set of configurations (limiting to max and choosing most interesting ones) all_configs = original_rh.get_all_configs() max_runs_epm = 300000 # Maximum total number of runs considered for epm to limit maximum possible number configs max_configs = int( max_runs_epm / (len(scenario.train_insts) + len(scenario.test_insts))) if len(all_configs) > max_configs: self.logger.debug( "Limiting number of configs to train epm from %d to %d (based on max runs %d) and choosing " "the ones with the most runs (for parallel coordinates)", len(all_configs), max_configs, max_runs_epm) all_configs = sorted( all_configs, key=lambda c: len(original_rh.get_runs_for_config(c) ))[:max_configs] if not default in all_configs: all_configs = [default] + all_configs if not incumbent in all_configs: all_configs.append(incumbent) # Get costs for those configurations epm_rh = RunHistory(average_cost) epm_rh.update(validated_rh) if scenario.feature_dict: # if instances are available epm_rh.update( timing(validator.validate_epm)(all_configs, 'train+test', 1, runhistory=validated_rh)) self.config_to_cost = {c: epm_rh.get_cost(c) for c in all_configs} self.params = self.get_params(params) self.n_configs = n_configs self.pcp = ParallelCoordinatesPlotter(self.config_to_cost, output_dir, cs, runtime)
def _preprocess_budget( self, original_rh: RunHistory, validated_rh: RunHistory, validator: Validator, scenario: Scenario, default: Configuration, incumbent: Configuration, param_imp: Union[None, Dict[str, float]], output_dir: str, cs: ConfigurationSpace, runtime: bool = False, ): """ Preprocess data and save in self.data to enable fast replots Parameters: ----------- original_rh: RunHistory runhistory that should contain only runs that were executed during search validated_rh: RunHistory runhistory that may contain as many runs as possible, also external runs. this runhistory will be used to build the EPM validator: Validator validator to be used to estimate costs for configurations scenario: Scenario scenario object to take instances from default, incumbent: Configuration default and incumbent, they will surely be displayed param_imp: Union[None, Dict[str->float] if given, maps parameter-names to importance output_dir: str output directory for plots cs: ConfigurationSpace parameter configuration space to be visualized runtime: boolean runtime will be on logscale """ # Sorting parameters by importance, if possible (choose first executed parameter-importance) method, importance = "", {} if self.pc_sort_by == 'all': self.logger.debug("Sorting by average importance") method = 'average' for m, i in param_imp.items(): if i: for p, imp in i.items(): if p in importance: importance[p].append(imp) else: importance[p] = [imp] importance = {k: sum(v) / len(v) for k, v in importance.items()} elif self.pc_sort_by in param_imp: method, importance = self.pc_sort_by, param_imp[self.pc_sort_by] else: self.logger.debug("%s not evaluated.. choosing at random from: %s", self.pc_sort_by, str(list(param_imp.keys()))) for m, i in param_imp.items(): if i: method, importance = m, i self.logger.debug("Chose %s", method) break hp_names = sorted([p for p in cs.get_hyperparameter_names()], key=lambda x: importance.get(x, 0), reverse=True) self.logger.debug("Sorted hyperparameters by method \'%s\': %s", method, str(hp_names)) # Define set of configurations (limiting to max and choosing most interesting ones) all_configs = original_rh.get_all_configs() # max_runs_epm is the maximum total number of runs considered for epm to limit maximum possible number configs max_configs = int( self.max_runs_epm / (len(scenario.train_insts) + len(scenario.test_insts))) if len(all_configs) > max_configs: self.logger.debug( "Limiting number of configs to train epm from %d to %d (based on max runs %d) and " "choosing the ones with the most runs (for parallel coordinates)", len(all_configs), max_configs, self.max_runs_epm) all_configs = sorted(all_configs, key=lambda c: len( original_rh.get_runs_for_config( c, only_max_observed_budget=False))) all_configs = all_configs[:max_configs] if default not in all_configs: all_configs = [default] + all_configs if incumbent not in all_configs: all_configs.append(incumbent) # Get costs for those configurations epm_rh = RunHistory() epm_rh.update(validated_rh) if scenario.feature_dict: # if instances are available epm_rh.update( timing(validator.validate_epm)(all_configs, 'train+test', 1, runhistory=validated_rh)) config_to_cost = OrderedDict( {c: epm_rh.get_cost(c) for c in all_configs}) data = OrderedDict() data['cost'] = list(config_to_cost.values()) for hp in self.runscontainer.scenario.cs.get_hyperparameter_names(): data[hp] = np.array([ c[hp] # if hp in c.get_dictionary() and not isinstance(c[hp], str) else np.nan for c in config_to_cost.keys() ]) df = pd.DataFrame(data=data) return df