def optimize(optimizer, scenario, trajectory=None): then = time.time() best_conf = optimizer.optimize() print(best_conf) print('training time:', time.time() - then) traj_logger = TrajLogger(None, Stats(scenario)) trajectory = trajectory or traj_logger.read_traj_aclib_format( "smac-output/run_1/traj_aclib2.json", scenario.cs) validator = Validator(scenario, trajectory, rng=np.random.RandomState(42)) # evaluate on test instances and calculate cpu time then = time.time() runhis_dev = validator.validate(config_mode="def", instance_mode="test") runhis_inc = validator.validate(config_mode="inc", instance_mode="test") print('validating time:', time.time() - then) default_conf = runhis_dev.ids_config[1] incumbent_conf = runhis_inc.ids_config[1] dev_vals = get_instance_costs_for_config(runhis_dev, default_conf) inc_vals = get_instance_costs_for_config(runhis_inc, incumbent_conf) # ###### Filter runs for plotting ####### dev_x = [] inc_x = [] for key in set(dev_vals.keys()) & set(inc_vals.keys()): dev_x.append(dev_vals[key]) inc_x.append(inc_vals[key]) # print(dev_vals) # print(inc_vals) print(dev_x) print(inc_x) print('PAR10:', np.mean(inc_x), '/', np.mean(dev_x)) max_x = 1000.0 par1er = lambda xx: np.mean([(x / 10 if x == max_x else x) for x in xx]) print('PAR1 :', par1er(inc_x), '/', par1er(dev_x)) to_counter = lambda xx: len([x for x in xx if x == max_x]) print('TOs :', to_counter(inc_x), '/', to_counter(dev_x)) print('wins :', len([i for i in range(len(dev_x)) if dev_x[i] > inc_x[i]]), '/', len(dev_x)) fig, ax = plt.subplots() ax.scatter(dev_x, inc_x, marker="x") ax.set_xlabel("Default Configuration") ax.set_ylabel("Incumbent Configuration") lims = [ np.min([ax.get_xlim(), ax.get_ylim()]), np.max([ax.get_xlim(), ax.get_ylim()]) ] ax.plot(lims, lims, 'k-', alpha=0.75, zorder=0) # ax.set_xlim(lims) # ax.set_ylim(lims) ax.set_xscale('log') ax.set_yscale('log') fig.savefig("fig-smac.png")
def test_parallel_same_as_serial_HB(self): """Makes sure we behave the same as a serial run at the end""" # Get the run_history for a_HB instance run: rh = RunHistory() stats = Stats(scenario=self.scen) stats.start_timing() _HB = _Hyperband( stats=stats, traj_logger=TrajLogger(output_dir=None, stats=stats), rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False, instances=[1, 2, 3, 4, 5], initial_budget=2, max_budget=5, eta=2, ) incumbent, inc_perf = self._exhaust_run_and_get_incumbent( _HB, rh, num_workers=1) # Just to make sure nothing has changed from the_HB instance side to make # this check invalid: # We add config values, so config 3 with 0 and 7 should be the lesser cost self.assertEqual(incumbent, self.config3) self.assertEqual(inc_perf, 7.0) # Do the same for HB, but have multiple_HB instance in there # This_HB instance will be created via num_workers==2 # in self._exhaust_run_and_get_incumbent HB = Hyperband( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False, instances=[1, 2, 3, 4, 5], initial_budget=2, max_budget=5, eta=2, ) incumbent_phb, inc_perf_phb = self._exhaust_run_and_get_incumbent( HB, self.rh) self.assertEqual(incumbent, incumbent_phb) # This makes sure there is a single incumbent in HB self.assertEqual(inc_perf, inc_perf_phb) # We don't want to loose any configuration, and particularly # we want to make sure the values of_HB instance to HB match self.assertEqual(len(self.rh.data), len(rh.data)) # Because it is a deterministic run, the run histories must be the # same on exhaustion self.assertDictEqual(self.rh.data, rh.data)
def get_trajectory(self, cs): def alternative_configuration_recovery(config_list: typing.List[str], cs: ConfigurationSpace): """ Used to recover ints and bools as categoricals or constants from trajectory """ config_dict = {} for param in config_list: k,v = param.split("=") v = v.strip("'") hp = cs.get_hyperparameter(k) if isinstance(hp, FloatHyperparameter): v = float(v) elif isinstance(hp, IntegerHyperparameter): v = int(v) ################# DIFFERENCE: ################ elif isinstance(hp, CategoricalHyperparameter) or isinstance(hp, Constant): if isinstance(hp.default_value, bool): v = True if v == 'True' else False elif isinstance(hp.default_value, int): v = int(v) elif isinstance(hp.default_value, float): v = float(v) else: v = v ############################################## config_dict[k] = v config = Configuration(configuration_space=cs, values=config_dict) config.origin = "External Trajectory" return config TrajLogger._convert_dict_to_config = alternative_configuration_recovery # Try to find trajectory in "alljson"-format todo instead just convert "old" smac data to new smac data traj_fn = os.path.join(self.folder, 'traj.json') if os.path.isfile(traj_fn): self.logger.debug("Found trajectory file in alljson-format at %s", traj_fn) return TrajLogger.read_traj_alljson_format(fn=traj_fn, cs=cs) self.logger.debug("%s not found. Trying to find in subfolders.") try: return TrajLogger.read_traj_alljson_format(fn=self.get_glob_file(self.folder, 'traj.json'), cs=cs) except FileNotFoundError: self.logger.info("Globbed approach failed. Trying old format.") old_traj_fn = os.path.join(self.folder, 'traj_aclib2.json') if os.path.isfile(old_traj_fn): self.logger.debug("Found trajectory file in aclib2-format (deprecated) at %s", old_traj_fn) return TrajLogger.read_traj_aclib_format(fn=old_traj_fn, cs=cs) try: return TrajLogger.read_traj_aclib_format(fn=self.get_glob_file(self.folder, 'traj_aclib2.json'), cs=cs) except FileNotFoundError: raise FileNotFoundError("Neither 'traj.json' nor 'traj_aclib2.json in %s or subdirectories.", self.folder) traj = TrajLogger.read_traj_aclib_format(fn=traj_fn, cs=cs) return traj
def test_add_entry(self, mock_stats): tl = TrajLogger(output_dir='./tmp_test_folder', stats=mock_stats) test_config = {'param_a': 0.5, 'param_b': 1, 'param_c': 'value'} mock_stats.ta_time_used = .5 mock_stats.get_used_wallclock_time = self.mocked_get_used_wallclock_time mock_stats.ta_runs = 1 tl.add_entry(0.9, 1, test_config) self.assertTrue(os.path.exists('tmp_test_folder/traj_old.csv')) self.assertTrue(os.path.exists('tmp_test_folder/traj_aclib2.json')) with open('tmp_test_folder/traj_old.csv') as to: data = to.read().split('\n')[:-1] header = data[0].split(',') self.assertEquals(header[0], '"CPU Time Used"') self.assertEquals(header[-1], '"Configuration..."') data = list(map(lambda x: x.split(', '), data[1:])) frmt_str = '%1.6f' self.assertEquals(frmt_str % .5, data[0][0]) self.assertEquals(frmt_str % .9, data[0][1]) self.assertEquals(frmt_str % 0.5, data[0][-4]) with open('tmp_test_folder/traj_aclib2.json') as js: json_dict = json.load(js) self.assertEquals(json_dict['cpu_time'], .5) self.assertEquals(json_dict['cost'], 0.9) self.assertEquals(len(json_dict['incumbent']), 3) self.assertTrue("param_a='0.5'" in json_dict['incumbent']) # And finally, test the list that's added to the trajectory class self.assertEqual( tl.trajectory[0], TrajEntry(0.9, 1, { 'param_c': 'value', 'param_b': 1, 'param_a': 0.5 }, 1, 0.5, 1)) # Test named-tuple-access: self.assertEqual(tl.trajectory[0].train_perf, 0.9) self.assertEqual(tl.trajectory[0].incumbent_id, 1) self.assertEqual(tl.trajectory[0].ta_runs, 1) self.assertEqual(tl.trajectory[0].ta_time_used, 0.5) self.assertEqual(tl.trajectory[0].wallclock_time, 1) self.assertEqual(len(tl.trajectory), 1)
def restore_state_after_output_dir( self, scen: Scenario, stats: Stats, traj_list_aclib: typing.List, traj_list_old: typing.List, ) -> Configuration: """Finish processing files for state-restoration. Trajectory is read in, but needs to be written to new output-folder. Therefore, the output-dir is created. This needs to be considered in the SMAC-facade.""" # write trajectory-list traj_path_aclib = os.path.join( scen.output_dir, "traj_aclib2.json") # type: ignore[attr-defined] # noqa F821 traj_path_old = os.path.join( scen.output_dir, "traj_old.csv") # type: ignore[attr-defined] # noqa F821 with open(traj_path_aclib, 'w') as traj_fn: traj_fn.writelines(traj_list_aclib) with open(traj_path_old, 'w') as traj_fn: traj_fn.writelines(traj_list_old) # read trajectory to retrieve incumbent # TODO replace this with simple traj_path_aclib? trajectory = TrajLogger.read_traj_aclib_format( fn=traj_path_aclib, cs=scen.cs) # type: ignore[attr-defined] # noqa F821 incumbent = trajectory[-1]["incumbent"] self.logger.debug("Restored incumbent %s from %s", incumbent, traj_path_aclib) return incumbent
def main(): # Initialize scenario, using runcount_limit as budget. orig_scen_dict = { 'algo': 'python cmdline_wrapper.py', 'paramfile': 'param_config_space.pcs', 'run_obj': 'quality', 'runcount_limit': 25, 'deterministic': True, 'output_dir': 'restore_me' } original_scenario = Scenario(orig_scen_dict) smac = SMAC(scenario=original_scenario) smac.optimize() print( "\n########## BUDGET EXHAUSTED! Restoring optimization: ##########\n") # Now the output is in the folder 'restore_me' # # We could simply modify the scenario-object, stored in # 'smac.solver.scenario' and start optimization again: #smac.solver.scenario.ta_run_limit = 50 #smac.optimize() # Or, to show the whole process of recovering a SMAC-run from the output # directory, create a new scenario with an extended budget: new_scenario = Scenario( orig_scen_dict, cmd_args={ 'runcount_limit': 50, # overwrite these args 'output_dir': 'restored' }) # We load the runhistory, ... rh_path = os.path.join(original_scenario.output_dir, "runhistory.json") runhistory = RunHistory(aggregate_func=None) runhistory.load_json(rh_path, new_scenario.cs) # ... stats, ... stats_path = os.path.join(original_scenario.output_dir, "stats.json") stats = Stats(new_scenario) stats.load(stats_path) # ... and trajectory. traj_path = os.path.join(original_scenario.output_dir, "traj_aclib2.json") trajectory = TrajLogger.read_traj_aclib_format(fn=traj_path, cs=new_scenario.cs) incumbent = trajectory[-1]["incumbent"] # Because we changed the output_dir, we might want to copy the old # trajectory-file (runhistory and stats will be complete) new_traj_path = os.path.join(new_scenario.output_dir, "traj_aclib2.json") shutil.copy(traj_path, new_traj_path) # Now we can initialize SMAC with the recovered objects and restore the # state where we left off. By providing stats and a restore_incumbent, SMAC # automatically detects the intention of restoring a state. smac = SMAC(scenario=new_scenario, runhistory=runhistory, stats=stats, restore_incumbent=incumbent) smac.optimize()
def test_add_inc_run_det(self): ''' test _add_inc_run() ''' def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) # since we assume deterministic=1, # the second call should not add any more runs # given only one instance intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data)
def test_race_challenger(self): ''' test _race_challenger without adaptive capping ''' def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) self.assertEqual(inc, self.config2)
def _set_param_space(self, param_space): self.param_space = param_space from ConfigSpace.hyperparameters import UniformFloatHyperparameter from smac.configspace import ConfigurationSpace from smac.optimizer.objective import average_cost from smac.runhistory.runhistory import RunHistory from smac.scenario.scenario import Scenario from smac.stats.stats import Stats from smac.utils.io.traj_logging import TrajLogger self.cs = ConfigurationSpace() for param in param_space: if param.type == 'continuous': var = UniformFloatHyperparameter(param.name, param.low, param.high) self.cs.add_hyperparameter(var) self.runhistory = RunHistory(aggregate_func=average_cost) self.scenario = Scenario({ 'run_obj': 'quality', 'runcount-limit': self.budget, 'cs': self.cs }) self.stats = Stats(self.scenario) self.traj_logger = TrajLogger(output_dir=__scratch__, stats=self.stats)
def restore_state(self, args_, scen, root_logger): # Check for folder and files rh_path = os.path.join(args_.restore_state, "runhistory.json") stats_path = os.path.join(args_.restore_state, "stats.json") traj_path = os.path.join(args_.restore_state, "traj_aclib2.json") scen_path = os.path.join(args_.restore_state, "scenario.txt") if not os.path.isdir(args_.restore_state): raise FileNotFoundError( "Could not find folder from which to restore.") # Load runhistory and stats rh = RunHistory(aggregate_func=None) rh.load_json(rh_path, scen.cs) root_logger.debug("Restored runhistory from %s", rh_path) stats = Stats(scen) stats.load(stats_path) root_logger.debug("Restored stats from %s", stats_path) trajectory = TrajLogger.read_traj_aclib_format(fn=traj_path, cs=scen.cs) incumbent = trajectory[-1]["incumbent"] root_logger.debug("Restored incumbent %s from %s", incumbent, traj_path) # Copy traj if output_dir of specified scenario-file is different than # the output_dir of the scenario-file in the folder from which to restore. if scen.output_dir != InputReader().read_scenario_file( scen_path)['output_dir']: new_traj_path = os.path.join(scen.output_dir, "traj_aclib2.json") shutil.copy(traj_path, new_traj_path) root_logger.debug("Copied traj %s", rh_path) return rh, stats, incumbent
def setUp(self): unittest.TestCase.setUp(self) self.rh = RunHistory() self.cs = get_config_space() self.config1 = Configuration(self.cs, values={'a': 7, 'b': 11}) self.config2 = Configuration(self.cs, values={'a': 13, 'b': 17}) self.config3 = Configuration(self.cs, values={'a': 0, 'b': 7}) self.config4 = Configuration(self.cs, values={'a': 29, 'b': 31}) self.config5 = Configuration(self.cs, values={'a': 31, 'b': 33}) self.scen = Scenario({ "cutoff_time": 2, 'cs': self.cs, "run_obj": 'runtime', "output_dir": '' }) self.stats = Stats(scenario=self.scen) self.stats.start_timing() # Create the base object self.HB = Hyperband( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), deterministic=False, run_obj_time=False, instances=[1, 2, 3, 4, 5], n_seeds=2, initial_budget=2, max_budget=5, eta=2, )
def test_compare_configs_no_joint_set(self): intensifier = Intensifier( tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) for i in range(2): self.rh.add(config=self.config1, cost=2, time=2, status=StatusType.SUCCESS, instance_id=1, seed=i, additional_info=None) for i in range(2, 5): self.rh.add(config=self.config2, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=i, additional_info=None) # The sets for the incumbent are completely disjoint. conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) self.assertIsNone(conf) # The incumbent has still one instance-seed pair left on which the # challenger was not run yet. self.rh.add(config=self.config2, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=1, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) self.assertIsNone(conf)
def test_add_inc_run_det(self): """ test _add_inc_run() """ def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) # since we assume deterministic=1, # the second call should not add any more runs # given only one instance intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) # The following two tests evaluate to zero because _next_iteration is triggered by _add_inc_run # as it is the first evaluation of this intensifier self.assertEqual(intensifier.num_run, 0) self.assertEqual(intensifier.num_chall_run, 0)
def test_adaptive_capping(self): """ test _adapt_cutoff() """ intensifier = AbstractRacer( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(5)), deterministic=False) for i in range(5): self.rh.add(config=self.config1, cost=i + 1, time=i + 1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) for i in range(3): self.rh.add(config=self.config2, cost=i + 1, time=i + 1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) inst_seed_pairs = self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True) # cost used by incumbent for going over all runs in inst_seed_pairs inc_sum_cost = self.rh.sum_cost(config=self.config1, instance_seed_budget_keys=inst_seed_pairs) cutoff = intensifier._adapt_cutoff(challenger=self.config2, run_history=self.rh, inc_sum_cost=inc_sum_cost) # 15*1.2 - 6 self.assertEqual(cutoff, 12) intensifier.cutoff = 5 cutoff = intensifier._adapt_cutoff(challenger=self.config2, run_history=self.rh, inc_sum_cost=inc_sum_cost) # scenario cutoff self.assertEqual(cutoff, 5)
def test_compare_configs_unknow(self): """ challenger is better but has less runs; -> no decision (None) """ intensifier = AbstractRacer( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config1, cost=1, time=2, status=StatusType.SUCCESS, instance_id=2, seed=None, additional_info=None) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=2, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh) # challenger worse than inc self.assertIsNone(conf, "conf: %s" % (conf))
def setUp(self): base_directory = os.path.split(__file__)[0] base_directory = os.path.abspath( os.path.join(base_directory, '..', '..')) self.current_dir = os.getcwd() os.chdir(base_directory) logging.basicConfig() self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) self.logger.setLevel(logging.DEBUG) self.rng = np.random.RandomState(seed=42) self.scen_fn = 'test/test_files/validation/scenario.txt' self.train_insts = ['0', '1', '2'] self.test_insts = ['3', '4', '5'] self.inst_specs = {'0': 'null', '1': 'one', '2': 'two', '3': 'three', '4': 'four', '5': 'five'} self.feature_dict = {'0': np.array((1, 2, 3)), '1': np.array((1, 2, 3)), '2': np.array((1, 2, 3)), '3': np.array((1, 2, 3)), '4': np.array((1, 2, 3)), '5': np.array((1, 2, 3))} self.output_rh = 'test/test_files/validation/' scen = Scenario(self.scen_fn, cmd_options={'run_obj': 'quality'}) self.stats = Stats(scen) self.trajectory = TrajLogger.read_traj_aclib_format( fn='test/test_files/validation/test_validation_traj.json', cs=scen.cs) self.output_dirs = [self.output_rh + 'test'] self.output_files = [self.output_rh + 'validated_runhistory_EPM.json', self.output_rh + 'validated_runhistory.json'] self.maxDiff = None
def test_get_next_challenger(self): """ test get_next_challenger() """ intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True) intensifier.stage = IntensifierStage.RUN_CHALLENGER # get a new challenger to evaluate config, new = intensifier.get_next_challenger( challengers=[self.config1, self.config2], chooser=None) self.assertEqual(config, self.config1, intensifier.current_challenger) self.assertEqual(intensifier._chall_indx, 1) self.assertEqual(intensifier.N, 1) self.assertTrue(new) # when already evaluating a challenger, return the same challenger intensifier.to_run = [(1, 1, 0)] config, new = intensifier.get_next_challenger( challengers=[self.config2], chooser=None) self.assertEqual(config, self.config1, intensifier.current_challenger) self.assertEqual(intensifier._chall_indx, 1) self.assertFalse(new)
def test_compare_configs_chall(self): """ challenger is better """ intensifier = AbstractRacer(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=2, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config2, cost=0, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh) # challenger has enough runs and is better self.assertEqual(conf, self.config2, "conf: %s" % (conf))
def setUp(self): logging.basicConfig() self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) self.logger.setLevel(logging.DEBUG) self.rng = np.random.RandomState(seed=42) self.scen_fn = 'test/test_files/validation/scenario.txt' self.train_insts = ['0', '1', '2'] self.test_insts = ['3', '4', '5'] self.inst_specs = { '0': 'null', '1': 'one', '2': 'two', '3': 'three', '4': 'four', '5': 'five' } self.feature_dict = { '0': np.array((1, 2, 3)), '1': np.array((1, 2, 3)), '2': np.array((1, 2, 3)), '3': np.array((1, 2, 3)), '4': np.array((1, 2, 3)), '5': np.array((1, 2, 3)) } self.output_rh = 'test/test_files/validation/' scen = Scenario(self.scen_fn, cmd_args={'run_obj': 'quality'}) self.trajectory = TrajLogger.read_traj_aclib_format( fn='test/test_files/validation/test_validation_traj.json', cs=scen.cs)
def setUp(self): logging.basicConfig() self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) self.logger.setLevel(logging.DEBUG) self.rng = np.random.RandomState(seed=42) self.scen = Scenario({ 'run_obj': 'quality', 'param_file': 'test/test_files/validation/test_validation_pcs.pcs', 'algo': 'python -u test/test_files/example_ta.py' }) self.train_insts = ['0', '1', '2'] self.test_insts = ['3', '4', '5'] self.scen.instance_specific = { '0': 'null', '1': 'one', '2': 'two', '3': 'three', '4': 'four', '5': 'five' } self.output_rh = 'test/test_files/validation/test_validation_rh.json' self.trajectory = TrajLogger.read_traj_aclib_format( fn='test/test_files/validation/test_validation_traj.json', cs=self.scen.cs)
def test_multi_config_design(self): stats = Stats(scenario=self.scenario) stats.start_timing() self.ta.stats = stats tj = TrajLogger(output_dir=None, stats=stats) rh = RunHistory(aggregate_func=average_cost) self.ta.runhistory = rh rng = np.random.RandomState(seed=12345) intensifier = Intensifier(tae_runner=self.ta, stats=stats, traj_logger=tj, rng=rng, instances=[None], run_obj_time=False) configs = [ Configuration(configuration_space=self.cs, values={"x1": 4}), Configuration(configuration_space=self.cs, values={"x1": 2}) ] dc = MultiConfigInitialDesign(tae_runner=self.ta, scenario=self.scenario, stats=stats, traj_logger=tj, runhistory=rh, rng=rng, configs=configs, intensifier=intensifier, aggregate_func=average_cost) inc = dc.run() self.assertTrue(stats.ta_runs == 2) self.assertTrue(len(rh.data) == 2) self.assertTrue(rh.get_cost(inc) == 4)
def get_trajectory(self, cs): def alternative_configuration_recovery(config_list: typing.List[str], cs: ConfigurationSpace): """ Used to recover ints and bools as categoricals or constants from trajectory """ config_dict = {} for param in config_list: k, v = param.split("=") v = v.strip("'") hp = cs.get_hyperparameter(k) if isinstance(hp, FloatHyperparameter): v = float(v) elif isinstance(hp, IntegerHyperparameter): v = int(v) ################# DIFFERENCE: ################ elif isinstance(hp, CategoricalHyperparameter) or isinstance( hp, Constant): if isinstance(hp.default_value, bool): v = True if v == 'True' else False elif isinstance(hp.default_value, int): v = int(v) else: v = v ############################################## config_dict[k] = v config = Configuration(configuration_space=cs, values=config_dict) config.origin = "External Trajectory" return config TrajLogger._convert_dict_to_config = alternative_configuration_recovery traj_fn = os.path.join(self.folder, 'traj_aclib2.json') traj = TrajLogger.read_traj_aclib_format(fn=traj_fn, cs=cs) return traj
def setUp(self): unittest.TestCase.setUp(self) self.rh = RunHistory() self.cs = get_config_space() self.config1 = Configuration(self.cs, values={'a': 7, 'b': 11}) self.config2 = Configuration(self.cs, values={'a': 13, 'b': 17}) self.config3 = Configuration(self.cs, values={'a': 0, 'b': 7}) self.config4 = Configuration(self.cs, values={'a': 29, 'b': 31}) self.scen = Scenario({"cutoff_time": 2, 'cs': self.cs, "run_obj": 'runtime', "output_dir": ''}) self.stats = Stats(scenario=self.scen) self.stats.start_timing() # Create the base object self.intensifier = SimpleIntensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False, instances=[1], )
def test_race_challenger_2(self): """ Makes sure that a racing configuration with better performance, that is capped, doesn't substitute the incumbent. """ def target(x): time.sleep(1.5) return (x["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], ) self.rh.add( config=self.config1, cost=0.001, time=0.001, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None, ) intensifier.N = 1 # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=self.config2, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=self.config2, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=True, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config1) self.assertEqual(intensifier.num_run, 1) self.assertEqual(intensifier.num_chall_run, 1)
def test_compare_configs_inc(self): ''' incumbent is better ''' intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config2, cost=2, time=2, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) # challenger worse than inc self.assertEqual(conf, self.config1, "conf: %s" % (conf))
def restore_state(scenario: typing.Union[Scenario, ScenarioProperties]): r"""Read in files for state-restoration: runhistory, stats, trajectory. :param scenario: Scenario whose state shall be loaded. :return: (RunHistory, Stats, dict)-tuple """ # Check for folder and files rh_path = os.path.join(scenario.output_dir_for_this_run, 'runhistory.json') stats_path = os.path.join(scenario.output_dir_for_this_run, 'stats.json') traj_path_aclib = os.path.join(scenario.output_dir_for_this_run, 'traj_aclib2.json') if not os.path.isdir(scenario.output_dir_for_this_run): raise FileNotFoundError('Could not find folder from which to restore.') # Load runhistory and stats rh = RunHistory(aggregate_func=None) rh.load_json(rh_path, scenario.cs) log.debug('Restored runhistory from %s', rh_path) stats = Stats(scenario) stats.load(stats_path) log.debug('Restored stats from %s', stats_path) trajectory = TrajLogger.read_traj_aclib_format(fn=traj_path_aclib, cs=scenario.cs) incumbent = trajectory[-1]['incumbent'] log.debug('Restored incumbent %s from %s', incumbent, traj_path_aclib) return rh, stats, incumbent
def test_race_challenger_2(self): ''' test _race_challenger with adaptive capping ''' def target(x): time.sleep(1.5) return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1]) self.rh.add(config=self.config1, cost=.001, time=0.001, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None) # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # self.assertTrue(False) self.assertEqual(inc, self.config1)
def test_init(self): scen = Scenario(scenario={'run_obj': 'quality', 'cs': self.cs, 'output_dir': ''}) stats = Stats(scen) with tempfile.TemporaryDirectory() as tmpdir: path = os.path.join(tmpdir, 'tmp_test_folder') TrajLogger(output_dir=path, stats=stats) self.assertTrue(os.path.exists(path))
def test_add_inc_run_nondet(self): ''' test _add_inc_run() ''' def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], deterministic=False) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 2, self.rh.data) runs = self.rh.get_runs_for_config(config=self.config1) # exactly one run on each instance self.assertIn(1, [runs[0].instance, runs[1].instance]) self.assertIn(2, [runs[0].instance, runs[1].instance]) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 3, self.rh.data)
def validate(self, config_mode='inc', instance_mode='train+test', repetitions=1, n_jobs=-1, backend='threading'): """Create validator-object and run validation, using scenario-information, runhistory from smbo and tae_runner from intensify Parameters ---------- config_mode: string what configurations to validate from [def, inc, def+inc, time, all], time means evaluation at timesteps 2^-4, 2^-3, 2^-2, 2^-1, 2^0, 2^1, ... instance_mode: string what instances to use for validation, from [train, test, train+test] repetitions: int number of repetitions in nondeterministic algorithms (in deterministic will be fixed to 1) n_jobs: int number of parallel processes used by joblib Returns ------- runhistory: RunHistory runhistory containing all specified runs """ traj_fn = os.path.join(self.scenario.output_dir, "traj_aclib2.json") trajectory = TrajLogger.read_traj_aclib_format(fn=traj_fn, cs=self.scenario.cs) new_rh_path = os.path.join(self.scenario.output_dir, "validated_runhistory.json") validator = Validator(self.scenario, trajectory, new_rh_path, self.rng) new_rh = validator.validate(config_mode, instance_mode, repetitions, n_jobs, backend, self.runhistory, self.intensifier.tae_runner) return new_rh