def test_multi_config_design(self): stats = Stats(scenario=self.scenario) stats.start_timing() self.ta.stats = stats tj = TrajLogger(output_dir=None, stats=stats) rh = RunHistory(aggregate_func=average_cost) self.ta.runhistory = rh rng = np.random.RandomState(seed=12345) intensifier = Intensifier(tae_runner=self.ta, stats=stats, traj_logger=tj, rng=rng, instances=[None], run_obj_time=False) configs = [ Configuration(configuration_space=self.cs, values={"x1": 4}), Configuration(configuration_space=self.cs, values={"x1": 2}) ] dc = MultiConfigInitialDesign(tae_runner=self.ta, scenario=self.scenario, stats=stats, traj_logger=tj, runhistory=rh, rng=rng, configs=configs, intensifier=intensifier, aggregate_func=average_cost) inc = dc.run() self.assertTrue(stats.ta_runs == 2) self.assertTrue(len(rh.data) == 2) self.assertTrue(rh.get_cost(inc) == 4)
def test_crashed_cost_value(self, test_run): ''' test cost on crashed runs ''' # Patch run-function for custom-return scen = Scenario(scenario={ 'cs': ConfigurationSpace(), 'run_obj': 'quality' }, cmd_options=None) stats = Stats(scen) stats.start_timing() stats.ta_runs += 1 # Check quality test_run.return_value = StatusType.CRASHED, np.nan, np.nan, {} eta = ExecuteTARun(ta=lambda *args: None, stats=stats, run_obj='quality', cost_for_crash=100) self.assertEqual(100, eta.start(config={}, instance=1)[1]) # Check runtime eta = ExecuteTARun(ta=lambda *args: None, stats=stats, run_obj='runtime', cost_for_crash=10.7) self.assertEqual(20.0, eta.start(config={}, instance=1, cutoff=20)[1])
def test_start_tae_return_abort(self, test_run): ''' testing abort ''' # Patch run-function for custom-return test_run.return_value = StatusType.ABORT, 12345.0, 1.2345, {} scen = Scenario( scenario={ 'cs': ConfigurationSpace(), 'run_obj': 'quality', 'output_dir': '', }, cmd_options=None, ) stats = Stats(scen) stats.start_timing() eta = SerialRunner(ta=lambda *args: None, stats=stats) _, run_value = eta.run_wrapper( RunInfo(config=None, instance=1, instance_specific=None, cutoff=30, seed=None, capped=False, budget=0.0)) self.assertEqual(run_value.status, StatusType.ABORT)
def test_parallel_same_as_serial_HB(self): """Makes sure we behave the same as a serial run at the end""" # Get the run_history for a_HB instance run: rh = RunHistory() stats = Stats(scenario=self.scen) stats.start_timing() _HB = _Hyperband( stats=stats, traj_logger=TrajLogger(output_dir=None, stats=stats), rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False, instances=[1, 2, 3, 4, 5], initial_budget=2, max_budget=5, eta=2, ) incumbent, inc_perf = self._exhaust_run_and_get_incumbent( _HB, rh, num_workers=1) # Just to make sure nothing has changed from the_HB instance side to make # this check invalid: # We add config values, so config 3 with 0 and 7 should be the lesser cost self.assertEqual(incumbent, self.config3) self.assertEqual(inc_perf, 7.0) # Do the same for HB, but have multiple_HB instance in there # This_HB instance will be created via num_workers==2 # in self._exhaust_run_and_get_incumbent HB = Hyperband( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False, instances=[1, 2, 3, 4, 5], initial_budget=2, max_budget=5, eta=2, ) incumbent_phb, inc_perf_phb = self._exhaust_run_and_get_incumbent( HB, self.rh) self.assertEqual(incumbent, incumbent_phb) # This makes sure there is a single incumbent in HB self.assertEqual(inc_perf, inc_perf_phb) # We don't want to loose any configuration, and particularly # we want to make sure the values of_HB instance to HB match self.assertEqual(len(self.rh.data), len(rh.data)) # Because it is a deterministic run, the run histories must be the # same on exhaustion self.assertDictEqual(self.rh.data, rh.data)
def test_single_default_config_design(self): stats = Stats(scenario=self.scenario) stats.start_timing() self.ta.stats = stats tj = TrajLogger(output_dir=None, stats=stats) rh = RunHistory(aggregate_func=average_cost) dc = DefaultConfiguration(tae_runner=self.ta, scenario=self.scenario, stats=stats, traj_logger=tj, rng=np.random.RandomState(seed=12345)) inc = dc.run() self.assertTrue(stats.ta_runs==1) self.assertTrue(len(rh.data)==0)
def get_tae(obj): """ Create SerialRunner-object for testing. """ scen = Scenario(scenario={ 'cs': ConfigurationSpace(), 'run_obj': obj, 'cutoff_time': '10' }, cmd_options=None) stats = Stats(scen) stats.start_timing() # Add first run to not trigger FirstRunCrashedException stats.submitted_ta_runs += 1 eta = SerialRunner(ta=lambda *args: None, stats=stats, run_obj=obj) return eta
def test_memory_limit_usage(self, run_mock): run_mock.return_value = StatusType.SUCCESS, 12345.0, 1.2345, {} stats = Stats(Scenario({})) stats.start_timing() tae = ExecuteTARun(lambda x: x**2, stats) self.assertRaisesRegex( ValueError, 'Target algorithm executor ' 'ExecuteTARun does not support restricting the memory usage.', tae.start, {'x': 2}, 'a', memory_limit=123) tae._supports_memory_limit = True rval = tae.start({'x': 2}, 'a', memory_limit=10) self.assertEqual(rval, run_mock.return_value)
def setUp(self): self.datamanager = get_multiclass_classification_datamanager() self.tmp = os.path.join(os.getcwd(), '.test_evaluation') self.logger = logging.getLogger() scenario_mock = unittest.mock.Mock() scenario_mock.wallclock_limit = 10 scenario_mock.algo_runs_timelimit = 1000 scenario_mock.ta_run_limit = 100 self.scenario = scenario_mock stats = Stats(scenario_mock) stats.start_timing() self.stats = stats try: shutil.rmtree(self.tmp) except: pass
def test_start_tae_return_abort(self, test_run): ''' testing abort ''' # Patch run-function for custom-return test_run.return_value = StatusType.ABORT, 12345.0, 1.2345, {} scen = Scenario(scenario={ 'cs': ConfigurationSpace(), 'output_dir': '' }, cmd_args=None) stats = Stats(scen) stats.start_timing() eta = ExecuteTARun(ta=lambda *args: None, stats=stats) self.assertRaises(TAEAbortException, eta.start, config={}, instance=1)
def setUp(self): self.datamanager = get_multiclass_classification_datamanager() self.tmp = os.path.join(os.getcwd(), '.test_evaluation') os.mkdir(self.tmp) self.logger_port = logging.handlers.DEFAULT_TCP_LOGGING_PORT scenario_mock = unittest.mock.Mock() scenario_mock.wallclock_limit = 10 scenario_mock.algo_runs_timelimit = 1000 scenario_mock.ta_run_limit = 100 self.scenario = scenario_mock stats = Stats(scenario_mock) stats.start_timing() self.stats = stats try: shutil.rmtree(self.tmp) except Exception: pass
def test_crashed_cost_value(self, test_run): ''' test cost on crashed runs ''' # Patch run-function for custom-return scen = Scenario(scenario={ 'cs': ConfigurationSpace(), 'run_obj': 'quality' }, cmd_options=None) stats = Stats(scen) stats.start_timing() stats.submitted_ta_runs += 1 # Check quality test_run.return_value = StatusType.CRASHED, np.nan, np.nan, {} eta = SerialRunner(ta=lambda *args: None, stats=stats, run_obj='quality', cost_for_crash=100) run_info, result = eta.run_wrapper( RunInfo(config={}, instance=1, instance_specific="0", cutoff=None, seed=None, capped=False, budget=0.0)) self.assertEqual(100, result.cost) # Check runtime eta = SerialRunner(ta=lambda *args: None, stats=stats, run_obj='runtime', cost_for_crash=10.7) run_info, result = eta.run_wrapper( RunInfo(config={}, instance=1, instance_specific="0", cutoff=20, seed=None, capped=False, budget=0.0)) self.assertEqual(20.0, result.cost)
def test_start_crash_first_run(self, test_run): ''' testing crash-on-first-run ''' # Patch run-function for custom-return test_run.return_value = StatusType.CRASHED, 12345.0, 1.2345, {} scen = Scenario(scenario={ 'cs': ConfigurationSpace(), 'output_dir': '' }, cmd_args=None) stats = Stats(scen) stats.start_timing() eta = ExecuteTARun(ta=lambda *args: None, stats=stats) self.assertRaises(FirstRunCrashedException, eta.start, config={}, instance=1)
def test_start_exhausted_budget(self): ''' testing exhausted budget ''' # Set time-limit negative in scenario-options to trigger exception scen = Scenario(scenario={ 'wallclock_limit': -1, 'cs': ConfigurationSpace(), 'output_dir': '' }, cmd_args=None) stats = Stats(scen) stats.start_timing() eta = ExecuteTARun( ta=lambda *args: None, # Dummy-function stats=stats) self.assertRaises(BudgetExhaustedException, eta.start, config={}, instance=1)
class TestSimpleIntensifier(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.rh = RunHistory() self.cs = get_config_space() self.config1 = Configuration(self.cs, values={'a': 7, 'b': 11}) self.config2 = Configuration(self.cs, values={'a': 13, 'b': 17}) self.config3 = Configuration(self.cs, values={'a': 0, 'b': 7}) self.config4 = Configuration(self.cs, values={'a': 29, 'b': 31}) self.scen = Scenario({"cutoff_time": 2, 'cs': self.cs, "run_obj": 'runtime', "output_dir": ''}) self.stats = Stats(scenario=self.scen) self.stats.start_timing() # Create the base object self.intensifier = SimpleIntensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False, instances=[1], ) def test_get_next_run(self): """ Makes sure that sampling a configuration returns a valid configuration """ intent, run_info = self.intensifier.get_next_run( challengers=[self.config1], incumbent=None, run_history=self.rh, num_workers=1, chooser=None, ) self.assertEqual(intent, RunInfoIntent.RUN) self.assertEqual(run_info, RunInfo( config=self.config1, instance=1, instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0, )) def test_get_next_run_waits_if_no_workers(self): """ In the case all workers are busy, we wait so that we do not saturate the process with configurations that will not finish in time """ intent, run_info = self.intensifier.get_next_run( challengers=[self.config1, self.config2], incumbent=None, run_history=self.rh, num_workers=1, chooser=None, ) # We can get the configuration 1 self.assertEqual(intent, RunInfoIntent.RUN) self.assertEqual(run_info, RunInfo( config=self.config1, instance=1, instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0, )) # We should not get configuration 2 # As there is just 1 worker intent, run_info = self.intensifier.get_next_run( challengers=[self.config2], incumbent=None, run_history=self.rh, num_workers=1, chooser=None, ) self.assertEqual(intent, RunInfoIntent.WAIT) self.assertEqual(run_info, RunInfo( config=None, instance=None, instance_specific="0", seed=0, cutoff=None, capped=False, budget=0.0, )) def test_process_results(self): """ Makes sure that we can process the results of a completed configuration """ intent, run_info = self.intensifier.get_next_run( challengers=[self.config1, self.config2], incumbent=None, run_history=self.rh, num_workers=1, chooser=None, ) result = RunValue( cost=1, time=0.5, status=StatusType.SUCCESS, starttime=1, endtime=2, additional_info=None, ) self.rh.add( config=run_info.config, cost=1, time=0.5, status=StatusType.SUCCESS, instance_id=run_info.instance, seed=run_info.seed, additional_info=None) incumbent, inc_perf = self.intensifier.process_results( run_info=run_info, incumbent=None, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(incumbent, run_info.config) self.assertEqual(inc_perf, 1)
class TestAbstractIntensifier(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.rh = RunHistory() self.cs = get_config_space() self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100}) self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0}) self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100}) self.scen = Scenario({ "cutoff_time": 2, 'cs': self.cs, "run_obj": 'runtime', "output_dir": '' }) self.stats = Stats(scenario=self.scen) self.stats.start_timing() self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) def test_get_next_challenger(self): """ test get_next_challenger - pick from list/chooser """ intensifier = AbstractRacer(tae_runner=None, stats=self.stats, traj_logger=None, rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False, cutoff=1, instances=[1]) # Error when nothing to choose from with self.assertRaisesRegex(ValueError, "No configurations/chooser provided"): intensifier.get_next_challenger(challengers=None, chooser=None, run_history=self.rh) # next challenger from a list config, _ = intensifier.get_next_challenger( challengers=[self.config1, self.config2], chooser=None, run_history=self.rh) self.assertEqual(config, self.config1) config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None, run_history=self.rh) self.assertEqual(config, self.config2) # next challenger from a chooser intensifier = AbstractRacer(tae_runner=None, stats=self.stats, traj_logger=None, rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False, cutoff=1, instances=[1]) chooser = SMAC4AC(self.scen, rng=1).solver.epm_chooser config, _ = intensifier.get_next_challenger(challengers=None, chooser=chooser, run_history=self.rh) self.assertEqual(len(list(config.get_dictionary().values())), 2) self.assertTrue(24 in config.get_dictionary().values()) self.assertTrue(68 in config.get_dictionary().values()) config, _ = intensifier.get_next_challenger(challengers=None, chooser=chooser, run_history=self.rh) self.assertEqual(len(list(config.get_dictionary().values())), 2) self.assertTrue(95 in config.get_dictionary().values()) self.assertTrue(38 in config.get_dictionary().values()) def test_get_next_challenger_repeat(self): """ test get_next_challenger - repeat configurations """ intensifier = AbstractRacer(tae_runner=None, stats=self.stats, traj_logger=None, rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False, cutoff=1, instances=[1]) # should not repeat configurations self.rh.add(self.config1, 1, 1, StatusType.SUCCESS) config, _ = intensifier.get_next_challenger( challengers=[self.config1, self.config2], chooser=None, run_history=self.rh, repeat_configs=False) self.assertEqual(config, self.config2) # should repeat configurations config, _ = intensifier.get_next_challenger( challengers=[self.config1, self.config2], chooser=None, run_history=self.rh, repeat_configs=True) self.assertEqual(config, self.config1) def test_compare_configs_no_joint_set(self): intensifier = AbstractRacer(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) for i in range(2): self.rh.add(config=self.config1, cost=2, time=2, status=StatusType.SUCCESS, instance_id=1, seed=i, additional_info=None) for i in range(2, 5): self.rh.add(config=self.config2, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=i, additional_info=None) # The sets for the incumbent are completely disjoint. conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh) self.assertIsNone(conf) # The incumbent has still one instance-seed pair left on which the # challenger was not run yet. self.rh.add(config=self.config2, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=1, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh) self.assertIsNone(conf) def test_compare_configs_chall(self): """ challenger is better """ intensifier = AbstractRacer(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=2, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config2, cost=0, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh) # challenger has enough runs and is better self.assertEqual(conf, self.config2, "conf: %s" % (conf)) def test_compare_configs_inc(self): """ incumbent is better """ intensifier = AbstractRacer(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config2, cost=2, time=2, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh) # challenger worse than inc self.assertEqual(conf, self.config1, "conf: %s" % (conf)) def test_compare_configs_unknow(self): """ challenger is better but has less runs; -> no decision (None) """ intensifier = AbstractRacer(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config1, cost=1, time=2, status=StatusType.SUCCESS, instance_id=2, seed=None, additional_info=None) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=2, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh) # challenger worse than inc self.assertIsNone(conf, "conf: %s" % (conf)) def test_adaptive_capping(self): """ test _adapt_cutoff() """ intensifier = AbstractRacer(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(5)), deterministic=False) for i in range(5): self.rh.add(config=self.config1, cost=i + 1, time=i + 1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) for i in range(3): self.rh.add(config=self.config2, cost=i + 1, time=i + 1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) inst_seed_pairs = self.rh.get_runs_for_config( self.config1, only_max_observed_budget=True) # cost used by incumbent for going over all runs in inst_seed_pairs inc_sum_cost = self.rh.sum_cost( config=self.config1, instance_seed_budget_keys=inst_seed_pairs) cutoff = intensifier._adapt_cutoff(challenger=self.config2, run_history=self.rh, inc_sum_cost=inc_sum_cost) # 15*1.2 - 6 self.assertEqual(cutoff, 12) intensifier.cutoff = 5 cutoff = intensifier._adapt_cutoff(challenger=self.config2, run_history=self.rh, inc_sum_cost=inc_sum_cost) # scenario cutoff self.assertEqual(cutoff, 5)
class TestIntensify(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.rh = RunHistory() self.cs = get_config_space() self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100}) self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0}) self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100}) self.scen = Scenario({ "cutoff_time": 2, 'cs': self.cs, "run_obj": 'runtime', "output_dir": '' }) self.stats = Stats(scenario=self.scen) self.stats.start_timing() self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) def test_race_challenger(self): """ test _race_challenger without adaptive capping """ def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) intensifier.N = 1 inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.num_run, 1) self.assertEqual(intensifier.num_chall_run, 1) def test_race_challenger_2(self): """ test _race_challenger with adaptive capping """ def target(x): time.sleep(1.5) return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1]) self.rh.add(config=self.config1, cost=.001, time=0.001, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None) intensifier.N = 1 # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc = intensifier._race_challenger( challenger=self.config2, incumbent=self.config1, run_history=self.rh, ) self.assertEqual(inc, self.config1) self.assertEqual(intensifier.num_run, 1) self.assertEqual(intensifier.num_chall_run, 1) def test_race_challenger_3(self): """ test _race_challenger with adaptive capping on a previously capped configuration """ def target(config: Configuration, seed: int, instance: str): if instance == 1: time.sleep(2.1) else: time.sleep(0.6) return (config['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime", par_factor=1) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), cutoff=2, instances=[1]) self.rh.add(config=self.config1, cost=0.5, time=.5, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None) # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) self.assertEqual(inc, self.config1) # further run for incumbent self.rh.add(config=self.config1, cost=2, time=2, status=StatusType.TIMEOUT, instance_id=2, seed=12345, additional_info=None) # give config2 a second chance - now it should run on both instances # run on instance 1 config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) # run on instance 2 config, _ = intensifier.get_next_challenger(challengers=[self.config3], chooser=None) self.assertEqual(config, self.config2) self.assertTrue(intensifier.continue_challenger) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) # the incumbent should still be config1 because # config2 should get on inst 1 a full timeout # such that c(config1) = 1.25 and c(config2) close to 1.3 self.assertEqual(inc, self.config1) # the capped run should not be counted in runs_perf_config self.assertAlmostEqual(self.rh.num_runs_per_config[2], 2) self.assertFalse(intensifier.continue_challenger) self.assertEqual(intensifier.num_run, 3) self.assertEqual(intensifier.num_chall_run, 3) def test_race_challenger_large(self): """ test _race_challenger using solution_quality """ def target(x): return 1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), run_obj_time=False, deterministic=True) for i in range(10): self.rh.add(config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=12345, additional_info=None) intensifier.stage = IntensifierStage.RUN_CHALLENGER # tie on first instances and then challenger should always win # and be returned as inc while True: config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) # stop when challenger evaluation is over if not intensifier.stage == IntensifierStage.RUN_CHALLENGER: break self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True) self.assertEqual(len(runs), 10) self.assertEqual(intensifier.num_run, 10) self.assertEqual(intensifier.num_chall_run, 10) def test_race_challenger_large_blocked_seed(self): """ test _race_challenger whether seeds are blocked for challenger runs """ def target(x): return 1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), run_obj_time=False, deterministic=False) for i in range(10): self.rh.add(config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) intensifier.stage = IntensifierStage.RUN_CHALLENGER # tie on first instances and then challenger should always win # and be returned as inc while True: config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) # stop when challenger evaluation is over if not intensifier.stage == IntensifierStage.RUN_CHALLENGER: break self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True) self.assertEqual(len(runs), 10) seeds = sorted([r.seed for r in runs]) self.assertEqual(seeds, list(range(10)), seeds) self.assertEqual(intensifier.num_run, 10) self.assertEqual(intensifier.num_chall_run, 10) def test_add_inc_run_det(self): """ test _add_inc_run() """ def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) # since we assume deterministic=1, # the second call should not add any more runs # given only one instance intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) # The following two tests evaluate to zero because _next_iteration is triggered by _add_inc_run # as it is the first evaluation of this intensifier self.assertEqual(intensifier.num_run, 0) self.assertEqual(intensifier.num_chall_run, 0) def test_add_inc_run_nondet(self): """ test _add_inc_run() """ def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, runhistory=self.rh, run_obj="solution_quality") intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], deterministic=False) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 2, self.rh.data) runs = self.rh.get_runs_for_config(config=self.config1, only_max_observed_budget=True) # exactly one run on each instance self.assertIn(1, [runs[0].instance, runs[1].instance]) self.assertIn(2, [runs[0].instance, runs[1].instance]) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 3, self.rh.data) self.assertEqual(intensifier.num_run, 2) self.assertEqual(intensifier.num_chall_run, 0) def test_get_next_challenger(self): """ test get_next_challenger() """ intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True) intensifier.stage = IntensifierStage.RUN_CHALLENGER # get a new challenger to evaluate config, new = intensifier.get_next_challenger( challengers=[self.config1, self.config2], chooser=None) self.assertEqual(config, self.config1, intensifier.current_challenger) self.assertEqual(intensifier._chall_indx, 1) self.assertEqual(intensifier.N, 1) self.assertTrue(new) # when already evaluating a challenger, return the same challenger intensifier.to_run = [(1, 1, 0)] config, new = intensifier.get_next_challenger( challengers=[self.config2], chooser=None) self.assertEqual(config, self.config1, intensifier.current_challenger) self.assertEqual(intensifier._chall_indx, 1) self.assertFalse(new) def test_generate_challenger(self): """ test generate_challenger() """ # test get generator from a list of challengers intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=None, rng=np.random.RandomState(12345), instances=[1], deterministic=True) gen = intensifier._generate_challengers( challengers=[self.config1, self.config2], chooser=None) self.assertEqual(next(gen), self.config1) self.assertEqual(next(gen), self.config2) self.assertRaises(StopIteration, next, gen) # test get generator from a chooser - would return only 1 configuration intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=None, rng=np.random.RandomState(12345), instances=[1], deterministic=True) chooser = SMAC4AC(self.scen, rng=1).solver.epm_chooser gen = intensifier._generate_challengers(challengers=None, chooser=chooser) self.assertEqual(next(gen).get_dictionary(), {'a': 24, 'b': 68}) self.assertRaises(StopIteration, next, gen) # when both are none, raise error with self.assertRaisesRegex(ValueError, "No configurations/chooser provided"): intensifier._generate_challengers(challengers=None, chooser=None) def test_eval_challenger_1(self): """ test eval_challenger() - a complete intensification run with a `always_race_against` configuration """ def target(x): if x['a'] == 100: time.sleep(1) return x['a'] taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], run_obj_time=True, cutoff=2, deterministic=False, always_race_against=self.config3, run_limit=1) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # intensification iteration #1 # run first config as incumbent if incumbent is None config, _ = intensifier.get_next_challenger(challengers=[self.config2], chooser=None) self.assertEqual(config, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # eval config 2 (=first run) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=None, run_history=self.rh, ) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual(intensifier.n_iters, 1) # 1 intensification run complete! # intensification iteration #2 # regular intensification begins - run incumbent first config, _ = intensifier.get_next_challenger( challengers=None, # don't need a new list here as old one is cont'd chooser=None) self.assertEqual(config, inc) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(self.stats.ta_runs, 2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(self.stats.inc_changed, 1) # run challenger now that the incumbent has been executed config, _ = intensifier.get_next_challenger(challengers=[self.config1], chooser=None) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(config, self.config1) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # challenger has a better performance, but not run on all instances yet. so incumbent stays the same self.assertEqual(inc, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertTrue(intensifier.continue_challenger) # run challenger again on the other instance config, _ = intensifier.get_next_challenger( challengers=None, # don't need a new list here as old one is cont'd chooser=None) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(config, self.config1) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # challenger better than incumbent in both instances. so incumbent changed self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_BASIS) self.assertFalse(intensifier.continue_challenger) # run basis configuration (`always_race_against`) config, _ = intensifier.get_next_challenger( challengers=None, # don't need a new list here as old one is cont'd chooser=None) self.assertEqual(config, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_BASIS) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # the basis configuration (config3) not better than incumbent, so can move on self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 2) self.assertEqual(self.stats.ta_runs, 5) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(list(self.rh.data.values())[4][2], StatusType.CAPPED) self.assertEqual( intensifier.n_iters, 1) # iteration continues as `min_chall` condition is not met self.assertIsInstance(intensifier.configs_to_run, collections.abc.Iterator) # no configs should be left at the end with self.assertRaises(StopIteration): next(intensifier.configs_to_run) # intensification continues running incumbent again in same iteration... config, _ = intensifier.get_next_challenger( challengers=None, # don't need a new list here as old one is cont'd chooser=None) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(inc, self.config1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual( len( self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 3) self.assertEqual( len( self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True)), 2) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 0) # capped def test_eval_challenger_2(self): """ test eval_challenger() - a complete intensification run without a `always_race_against` configuration """ def target(x): return 2 * x['a'] + x['b'] taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=True, always_race_against=None, run_limit=1) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # intensification iteration #1 # run first config as incumbent if incumbent is None config, _ = intensifier.get_next_challenger(challengers=[self.config3], chooser=None) self.assertEqual(config, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # eval config 2 (=first run) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=None, run_history=self.rh, ) self.assertEqual(inc, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual(intensifier.n_iters, 1) # 1 intensification run complete! # regular intensification begins - run incumbent config, _ = intensifier.get_next_challenger( challengers=None, # since incumbent is run, no configs required chooser=None) self.assertEqual(config, inc) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # no new TA runs as there are no more instances to run self.assertEqual(inc, self.config3) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual(self.stats.ta_runs, 1) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) # run challenger now that the incumbent has been executed config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config1], chooser=None) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(config, self.config2) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # challenger has a better performance, so incumbent has changed self.assertEqual(inc, self.config2) self.assertEqual(self.stats.inc_changed, 2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT ) # since there is no `always_race_against` self.assertFalse(intensifier.continue_challenger) self.assertEqual( intensifier.n_iters, 1) # iteration continues as `min_chall` condition is not met # intensification continues running incumbent again in same iteration... # run incumbent config, _ = intensifier.get_next_challenger( challengers=None, # don't need a new list here as old one is cont'd chooser=None) self.assertEqual(config, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # run challenger config, _ = intensifier.get_next_challenger( challengers=None, # don't need a new list here as old one is cont'd chooser=None) self.assertEqual(config, self.config1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(intensifier.n_iters, 2) # 2 intensification run complete! # no configs should be left at the end with self.assertRaises(StopIteration): next(intensifier.configs_to_run) self.assertEqual( len( self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 1) self.assertEqual( len( self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True)), 1) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 1) def test_eval_challenger_3(self): """ test eval_challenger for a resumed SMAC run (first run with incumbent) """ def target(x): return x['a'] taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=False, always_race_against=None, run_limit=1) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # adding run for incumbent configuration self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) # intensification - incumbent will be run, but not as RUN_FIRST_CONFIG stage config, _ = intensifier.get_next_challenger(challengers=[self.config2], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual( len( self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 2) def test_no_new_intensification_wo_challenger_run(self): """ This test ensures that no new iteration is started if no challenger run was conducted """ def target(x): return 2 * x['a'] + x['b'] taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="quality") taf.runhistory = self.rh intensifier = Intensifier( tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=True, always_race_against=None, run_limit=1, min_chall=1, ) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) config, _ = intensifier.get_next_challenger(challengers=[self.config3], chooser=None) self.assertEqual(config, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=None, run_history=self.rh, ) self.assertEqual(inc, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(intensifier.n_iters, 1) # 1 intensification run complete! # regular intensification begins - run incumbent config, _ = intensifier.get_next_challenger( challengers=None, # since incumbent is run, no configs required chooser=None) self.assertEqual(config, inc) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(intensifier.n_iters, 1) # Check that we don't walk into the next iteration if the challenger is passed again config, _ = intensifier.get_next_challenger(challengers=[self.config3], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(intensifier.n_iters, 1) intensifier._next_iteration() # Add a configuration, then try to execute it afterwards self.assertEqual(intensifier.n_iters, 2) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=0, additional_info=None) intensifier.stage = IntensifierStage.RUN_CHALLENGER config, _ = intensifier.get_next_challenger(challengers=[self.config1], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(intensifier.n_iters, 2) self.assertEqual(intensifier.num_chall_run, 0) # This returns the config evaluating the incumbent again config, _ = intensifier.get_next_challenger(challengers=None, chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # This doesn't return a config because the array of configs is exhausted config, _ = intensifier.get_next_challenger(challengers=None, chooser=None) self.assertIsNone(config) # This finally gives a runable configuration config, _ = intensifier.get_next_challenger(challengers=[self.config2], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(intensifier.n_iters, 3) self.assertEqual(intensifier.num_chall_run, 1)
class TestAbstractRacer(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.rh = RunHistory() self.cs = get_config_space() self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100}) self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0}) self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100}) self.scen = Scenario({ "cutoff_time": 2, 'cs': self.cs, "run_obj": 'runtime', "output_dir": '' }) self.stats = Stats(scenario=self.scen) self.stats.start_timing() self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) def test_compare_configs_no_joint_set(self): intensifier = AbstractRacer(stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) for i in range(2): self.rh.add(config=self.config1, cost=2, time=2, status=StatusType.SUCCESS, instance_id=1, seed=i, additional_info=None) for i in range(2, 5): self.rh.add(config=self.config2, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=i, additional_info=None) # The sets for the incumbent are completely disjoint. conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh) self.assertIsNone(conf) # The incumbent has still one instance-seed pair left on which the # challenger was not run yet. self.rh.add(config=self.config2, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=1, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh) self.assertIsNone(conf) def test_compare_configs_chall(self): """ challenger is better """ intensifier = AbstractRacer(stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=2, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config2, cost=0, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh) # challenger has enough runs and is better self.assertEqual(conf, self.config2, "conf: %s" % (conf)) def test_compare_configs_inc(self): """ incumbent is better """ intensifier = AbstractRacer(stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config2, cost=2, time=2, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh) # challenger worse than inc self.assertEqual(conf, self.config1, "conf: %s" % (conf)) def test_compare_configs_unknow(self): """ challenger is better but has less runs; -> no decision (None) """ intensifier = AbstractRacer(stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config1, cost=1, time=2, status=StatusType.SUCCESS, instance_id=2, seed=None, additional_info=None) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=2, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh) # challenger worse than inc self.assertIsNone(conf, "conf: %s" % (conf)) def test_adaptive_capping(self): """ test _adapt_cutoff() """ intensifier = AbstractRacer(stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(5)), deterministic=False) for i in range(5): self.rh.add(config=self.config1, cost=i + 1, time=i + 1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) for i in range(3): self.rh.add(config=self.config2, cost=i + 1, time=i + 1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) inst_seed_pairs = self.rh.get_runs_for_config( self.config1, only_max_observed_budget=True) # cost used by incumbent for going over all runs in inst_seed_pairs inc_sum_cost = self.rh.sum_cost( config=self.config1, instance_seed_budget_keys=inst_seed_pairs) cutoff = intensifier._adapt_cutoff(challenger=self.config2, run_history=self.rh, inc_sum_cost=inc_sum_cost) # 15*1.2 - 6 self.assertEqual(cutoff, 12) intensifier.cutoff = 5 cutoff = intensifier._adapt_cutoff(challenger=self.config2, run_history=self.rh, inc_sum_cost=inc_sum_cost) # scenario cutoff self.assertEqual(cutoff, 5)
class TestIntensify(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.rh = RunHistory(aggregate_func=average_cost) self.cs = get_config_space() self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100}) self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0}) self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100}) self.scen = Scenario({ "cutoff_time": 2, 'cs': self.cs, "output_dir": "" }) self.stats = Stats(scenario=self.scen) self.stats.start_timing() self.logger = logging.getLogger("Test") def test_compare_configs_chall(self): ''' challenger is better but has not enough runs ''' intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=2, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config2, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) # challenger has enough runs and is better self.assertEqual(conf, self.config2, "conf: %s" % (conf)) def test_compare_configs_inc(self): ''' incumbent is better ''' intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config2, cost=2, time=2, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) # challenger worse than inc self.assertEqual(conf, self.config1, "conf: %s" % (conf)) def test_compare_configs_unknow(self): ''' challenger is better but has less runs; -> no decision (None) ''' intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config1, cost=1, time=2, status=StatusType.SUCCESS, instance_id=2, seed=None, additional_info=None) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=2, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) # challenger worse than inc self.assertIsNone(conf, "conf: %s" % (conf)) def test_race_challenger(self): ''' test _race_challenger without adaptive capping ''' def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) self.assertEqual(inc, self.config2) def test_race_challenger(self): ''' test _race_challenger with adaptive capping ''' def target(x): time.sleep(1.5) return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1]) self.rh.add(config=self.config1, cost=.001, time=0.001, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None) # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # self.assertTrue(False) self.assertEqual(inc, self.config1) self.assertLess(self.rh.get_cost(self.config2), 2, self.rh.get_cost(self.config2)) # get data for config2 to check that the correct run was performed run = self.rh.get_runs_for_config(self.config2)[0] config_id = self.rh.config_ids[self.config2] self.assertEqual(run.instance, 1, run.instance) self.assertEqual(run.seed, 12345, run.seed) def test_race_challenger_large(self): ''' test _race_challenger using solution_quality ''' def target(x): return 1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), deterministic=True) for i in range(10): self.rh.add(config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=12345, additional_info=None) # tie on first instances and then challenger should always win # and be returned as inc inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # self.assertTrue(False) self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1, self.rh.get_cost(self.config2)) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2) self.assertEqual(len(runs), 10) def test_race_challenger_large_blocked_seed(self): ''' test _race_challenger whether seeds are blocked for challenger runs ''' def target(x): return 1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), deterministic=False) for i in range(10): self.rh.add(config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) # tie on first instances and then challenger should always win # and be returned as inc inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # self.assertTrue(False) self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1, self.rh.get_cost(self.config2)) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2) self.assertEqual(len(runs), 10) seeds = sorted([r.seed for r in runs]) self.assertEqual(seeds, list(range(10)), seeds) def test_add_inc_run_det(self): ''' test _add_inc_run() ''' def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) # since we assume deterministic=1, # the second call should not add any more runs # given only one instance intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) def test_add_inc_run_nondet(self): ''' test _add_inc_run() ''' def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], deterministic=False) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 2, self.rh.data) runs = self.rh.get_runs_for_config(config=self.config1) # exactly one run on each instance self.assertIn(1, [runs[0].instance, runs[1].instance]) self.assertIn(2, [runs[0].instance, runs[1].instance]) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 3, self.rh.data) def test_adaptive_capping(self): ''' test _adapt_cutoff() ''' intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(5)), deterministic=False) for i in range(5): self.rh.add(config=self.config1, cost=i + 1, time=i + 1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) for i in range(3): self.rh.add(config=self.config2, cost=i + 1, time=i + 1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) inst_seed_pairs = self.rh.get_runs_for_config(self.config1) # cost used by incumbent for going over all runs in inst_seed_pairs inc_sum_cost = sum_cost(config=self.config1, instance_seed_pairs=inst_seed_pairs, run_history=self.rh) cutoff = intensifier._adapt_cutoff(challenger=self.config2, incumbent=self.config1, run_history=self.rh, inc_sum_cost=inc_sum_cost) # 15*1.2 - 6 self.assertEqual(cutoff, 12) intensifier.cutoff = 5 cutoff = intensifier._adapt_cutoff(challenger=self.config2, incumbent=self.config1, run_history=self.rh, inc_sum_cost=inc_sum_cost) # scenario cutoff self.assertEqual(cutoff, 5)
def run_configuration(backend, config_id, task_id, configuration, run_args, memory_limit, per_run_time_limit): evaluation, iterative_fit, early_stopping, N_FOLDS, searchspace = run_args # TODO make this an argument from the command line! scenario_mock = unittest.mock.Mock() scenario_mock.wallclock_limit = per_run_time_limit * 100 scenario_mock.algo_runs_timelimit = per_run_time_limit * 100 scenario_mock.ta_run_limit = np.inf stats = Stats(scenario_mock) stats.ta_runs = 2 # Resampling strategies kwargs = {} if evaluation == "holdout" and iterative_fit: resampling_strategy = 'holdout-iterative-fit' elif evaluation == "holdout" and not iterative_fit: resampling_strategy = 'holdout' elif evaluation == "CV" and not iterative_fit: resampling_strategy = 'cv' kwargs = {'folds': N_FOLDS} elif evaluation == "CV" and iterative_fit: resampling_strategy = 'cv-iterative-fit' kwargs = {'folds': N_FOLDS} else: raise ValueError("Unknown resampling strategy", evaluation) iterative_wo_early_stopping = [ 'extra_trees', 'PassiveAggressiveWOEarlyStopping', 'random_forest', 'SGDWOEarlyStopping', 'GradientBoostingClassifierWOEarlyStopping' ] iterative_w_early_stopping = [ 'extra_trees', 'passive_aggressive', 'random_forest', 'sgd', 'gradient_boosting' ] if not early_stopping: add_classifier_wo_early_stopping() if searchspace == "iterative": include_estimator = iterative_w_early_stopping if early_stopping else iterative_wo_early_stopping include_preprocessor = [ "no_preprocessing", ] elif searchspace == "full": assert early_stopping is True include_estimator = None include_preprocessor = None # elif searchspace == 'only-iterative-nopreproc': # include_estimator = iterative_w_early_stopping if early_stopping else iterative_wo_early_stopping # include_preprocessor = ["no_preprocessing", ] # elif searchspace == 'only-iterative-cheappreproc': # include_estimator = iterative_w_early_stopping if early_stopping else iterative_wo_early_stopping # include_preprocessor = ["no_preprocessing", 'kitchen_sinks', 'polynomial', 'select_percentile_classification', 'select_rates'] # elif searchspace == 'only-iterative': # include_estimator = iterative_w_early_stopping if early_stopping else iterative_wo_early_stopping # include_preprocessor = None # elif searchspace == "gb": # include_estimator = ['GradientBoostingClassifierWOEarlyStopping'] if early_stopping else ['GradientBoostingClassifierWEarlyStopping'] # include_preprocessor = None else: raise ValueError(searchspace) stats.start_timing() tae = ExecuteTaFuncWithQueue( backend=backend, autosklearn_seed=3, resampling_strategy=resampling_strategy, metric=balanced_accuracy, logger=logging.getLogger(name="%s_%s" % (task_id, config_id)), initial_num_run=2, stats=stats, runhistory=None, run_obj='quality', par_factor=1, all_scoring_functions=False, output_y_hat_optimization=True, include={ "classifier": include_estimator, "feature_preprocessor": include_preprocessor }, exclude=None, memory_limit=memory_limit, disable_file_output=True, init_params=None, **kwargs) # Finally run configuration status, cost, runtime, additional_run_info = tae.start( config=configuration, instance=None, cutoff=per_run_time_limit, instance_specific=None, capped=False, ) return status, cost, runtime, additional_run_info
class TestIntensify(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.rh = RunHistory() self.cs = get_config_space() self.config1 = Configuration(self.cs, values={"a": 0, "b": 100}) self.config2 = Configuration(self.cs, values={"a": 100, "b": 0}) self.config3 = Configuration(self.cs, values={"a": 100, "b": 100}) self.scen = Scenario({ "cutoff_time": 2, "cs": self.cs, "run_obj": "runtime", "output_dir": "", "deterministic": False, "limit_resources": True, }) self.stats = Stats(scenario=self.scen) self.stats.start_timing() self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) def test_race_challenger_1(self): """ Makes sure that a racing configuration with better performance, is selected as incumbent No adaptive capping """ def target(x): return (x["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, ) self.rh.add( config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None, ) intensifier.N = 1 inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=self.config2, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=self.config2, instance=instance, instance_specific="0", cutoff=cutoff, seed=seed, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.num_run, 1) self.assertEqual(intensifier.num_chall_run, 1) def test_race_challenger_2(self): """ Makes sure that a racing configuration with better performance, that is capped, doesn't substitute the incumbent. """ def target(x): time.sleep(1.5) return (x["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], ) self.rh.add( config=self.config1, cost=0.001, time=0.001, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None, ) intensifier.N = 1 # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=self.config2, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=self.config2, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=True, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config1) self.assertEqual(intensifier.num_run, 1) self.assertEqual(intensifier.num_chall_run, 1) def test_race_challenger_3(self): """ test _race_challenger with adaptive capping on a previously capped configuration """ def target(config: Configuration, seed: int, instance: str): if instance == 1: time.sleep(2.1) else: time.sleep(0.6) return (config["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict( use_pynisher=False, ta=target, stats=self.stats, run_obj="runtime", par_factor=1, ) taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), cutoff=2, instances=[1], ) self.rh.add( config=self.config1, cost=0.5, time=0.5, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None, ) # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=config, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=config, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=True, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config1) # further run for incumbent self.rh.add( config=self.config1, cost=2, time=2, status=StatusType.TIMEOUT, instance_id=2, seed=12345, additional_info=None, ) # give config2 a second chance - now it should run on both instances # run on instance 1 config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=config, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=config, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) # run on instance 2 config, _ = intensifier.get_next_challenger(challengers=[self.config3], chooser=None) self.assertEqual(config, self.config2) self.assertTrue(intensifier.continue_challenger) inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=config, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=config, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) # the incumbent should still be config1 because # config2 should get on inst 1 a full timeout # such that c(config1) = 1.25 and c(config2) close to 1.3 self.assertEqual(inc, self.config1) # the capped run should not be counted in runs_perf_config self.assertAlmostEqual(self.rh.num_runs_per_config[2], 2) self.assertFalse(intensifier.continue_challenger) self.assertEqual(intensifier.num_run, 3) self.assertEqual(intensifier.num_chall_run, 3) def test_race_challenger_large(self): """ test _race_challenger using solution_quality """ def target(x): return 1 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), run_obj_time=False, deterministic=True, ) for i in range(10): self.rh.add( config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=12345, additional_info=None, ) intensifier.stage = IntensifierStage.RUN_CHALLENGER # tie on first instances and then challenger should always win # and be returned as inc while True: if intensifier.continue_challenger: config = intensifier.current_challenger else: config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=config, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=config, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) # stop when challenger evaluation is over if not intensifier.stage == IntensifierStage.RUN_CHALLENGER: break self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True) self.assertEqual(len(runs), 10) self.assertEqual(intensifier.num_run, 10) self.assertEqual(intensifier.num_chall_run, 10) def test_race_challenger_large_blocked_seed(self): """ test _race_challenger whether seeds are blocked for challenger runs """ def target(x): return 1 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), run_obj_time=False, deterministic=False, ) for i in range(10): self.rh.add( config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None, ) intensifier.stage = IntensifierStage.RUN_CHALLENGER # tie on first instances and then challenger should always win # and be returned as inc while True: if intensifier.continue_challenger: config = intensifier.current_challenger else: config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=config, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=config, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) # stop when challenger evaluation is over if not intensifier.stage == IntensifierStage.RUN_CHALLENGER: break self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True) self.assertEqual(len(runs), 10) seeds = sorted([r.seed for r in runs]) self.assertEqual(seeds, list(range(10)), seeds) self.assertEqual(intensifier.num_run, 10) self.assertEqual(intensifier.num_chall_run, 10) def test_add_inc_run_det(self): """ test _add_inc_run() """ def target(x): return (x["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True, ) instance, seed, cutoff = intensifier._get_next_inc_run( available_insts=intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh)) run_info = RunInfo( config=self.config1, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) intensifier.stage = IntensifierStage.PROCESS_FIRST_CONFIG_RUN inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(len(self.rh.data), 1, self.rh.data) # since we assume deterministic=1, # the second call should not add any more runs # given only one instance # So the returned seed/instance is None so that a new # run to be triggered is not launched available_insts = intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh) # Make sure that the list is empty, and hence no new call # of incumbent will be triggered self.assertFalse(available_insts) # The following two tests evaluate to zero because _next_iteration is triggered by _add_inc_run # as it is the first evaluation of this intensifier # After the above incumbent run, the stage is # IntensifierStage.RUN_CHALLENGER. Change it to test next iteration intensifier.stage = IntensifierStage.PROCESS_FIRST_CONFIG_RUN inc, perf = intensifier.process_results( run_info=run_info, incumbent=None, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(intensifier.num_run, 0) self.assertEqual(intensifier.num_chall_run, 0) def test_add_inc_run_nondet(self): """ test _add_inc_run() """ def target(x): return (x["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="solution_quality") intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], deterministic=False, ) instance, seed, cutoff = intensifier._get_next_inc_run( available_insts=intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh)) run_info = RunInfo( config=self.config1, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(len(self.rh.data), 1, self.rh.data) instance, seed, cutoff = intensifier._get_next_inc_run( available_insts=intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh)) run_info = RunInfo( config=self.config1, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(len(self.rh.data), 2, self.rh.data) runs = self.rh.get_runs_for_config(config=self.config1, only_max_observed_budget=True) # exactly one run on each instance self.assertIn(1, [runs[0].instance, runs[1].instance]) self.assertIn(2, [runs[0].instance, runs[1].instance]) instance, seed, cutoff = intensifier._get_next_inc_run( available_insts=intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh)) run_info = RunInfo( config=self.config1, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(len(self.rh.data), 3, self.rh.data) # The number of runs performed should be 3 # No Next iteration call as an incumbent is provided self.assertEqual(intensifier.num_run, 2) self.assertEqual(intensifier.num_chall_run, 0) def testget_next_challenger(self): """ test get_next_challenger() """ intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True, ) intensifier.stage = IntensifierStage.RUN_CHALLENGER # get a new challenger to evaluate config, new = intensifier.get_next_challenger( challengers=[self.config1, self.config2], chooser=None) self.assertEqual(config, self.config1, intensifier.current_challenger) self.assertEqual(intensifier._chall_indx, 1) self.assertEqual(intensifier.N, 1) self.assertTrue(new) # when already evaluating a challenger, return the same challenger intensifier.to_run = [(1, 1, 0)] config, new = intensifier.get_next_challenger( challengers=[self.config2], chooser=None) self.assertEqual(config, self.config1, intensifier.current_challenger) self.assertEqual(intensifier._chall_indx, 1) self.assertFalse(new) def test_generate_challenger(self): """ test generate_challenger() """ # test get generator from a list of challengers intensifier = Intensifier( stats=self.stats, traj_logger=None, rng=np.random.RandomState(12345), instances=[1], deterministic=True, ) gen = intensifier._generate_challengers( challengers=[self.config1, self.config2], chooser=None) self.assertEqual(next(gen), self.config1) self.assertEqual(next(gen), self.config2) self.assertRaises(StopIteration, next, gen) # test get generator from a chooser - would return only 1 configuration intensifier = Intensifier( stats=self.stats, traj_logger=None, rng=np.random.RandomState(12345), instances=[1], deterministic=True, ) chooser = SMAC4AC(self.scen, rng=1).solver.epm_chooser gen = intensifier._generate_challengers(challengers=None, chooser=chooser) self.assertEqual(next(gen).get_dictionary(), {"a": 24, "b": 68}) self.assertRaises(StopIteration, next, gen) # when both are none, raise error with self.assertRaisesRegex(ValueError, "No configurations/chooser provided"): intensifier._generate_challengers(challengers=None, chooser=None) def test_eval_challenger_1(self): """ test eval_challenger() - a complete intensification run with a `always_race_against` configuration """ print(self.rh) def target(x): if x["a"] == 100: time.sleep(1) return x["a"] taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], run_obj_time=True, cutoff=2, deterministic=False, always_race_against=self.config3, run_limit=1, ) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # intensification iteration #1 # run first config as incumbent if incumbent is None intent, run_info = intensifier.get_next_run( incumbent=None, run_history=self.rh, challengers=[self.config2], chooser=None, ) self.assertEqual(run_info.config, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.PROCESS_FIRST_CONFIG_RUN) # eval config 2 (=first run) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=None, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual(intensifier.n_iters, 1) # 1 intensification run complete! # intensification iteration #2 # regular intensification begins - run incumbent first intent, run_info = intensifier.get_next_run( challengers=None, # don't need a new list here as old one is cont'd incumbent=inc, run_history=self.rh, chooser=None, ) self.assertEqual(run_info.config, inc) self.assertEqual(intensifier.stage, IntensifierStage.PROCESS_INCUMBENT_RUN) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(self.stats.inc_changed, 1) # run challenger now that the incumbent has been executed intent, run_info = intensifier.get_next_run(challengers=[self.config1], incumbent=inc, run_history=self.rh, chooser=None) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(run_info.config, self.config1) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) # challenger has a better performance, but not run on all instances yet. so incumbent stays the same self.assertEqual(inc, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertTrue(intensifier.continue_challenger) # run challenger again on the other instance intent, run_info = intensifier.get_next_run( challengers=None, # don't need a new list here as old one is cont'd incumbent=inc, run_history=self.rh, chooser=None, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(run_info.config, self.config1) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) # challenger better than incumbent in both instances. so incumbent changed self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_BASIS) self.assertFalse(intensifier.continue_challenger) # run basis configuration (`always_race_against`) intent, run_info = intensifier.get_next_run( challengers=None, # don't need a new list here as old one is cont'd incumbent=inc, run_history=self.rh, chooser=None, ) self.assertEqual(run_info.config, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_BASIS) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) # the basis configuration (config3) not better than incumbent, so can move on self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT, msg=self.rh.data.items()) self.assertEqual(list(self.rh.data.values())[4][2], StatusType.CAPPED) self.assertEqual( intensifier.n_iters, 1) # iteration continues as `min_chall` condition is not met self.assertIsInstance(intensifier.configs_to_run, collections.abc.Iterator) # no configs should be left at the end with self.assertRaises(StopIteration): next(intensifier.configs_to_run) # intensification continues running incumbent again in same iteration... intent, run_info = intensifier.get_next_run( challengers=None, # don't need a new list here as old one is cont'd incumbent=inc, run_history=self.rh, chooser=None, ) self.assertEqual(intensifier.stage, IntensifierStage.PROCESS_INCUMBENT_RUN) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual( len( self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 3, ) self.assertEqual( len( self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True)), 2, ) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 0, ) # capped def test_eval_challenger_2(self): """ test eval_challenger() - a complete intensification run without a `always_race_against` configuration """ def target(x): return 2 * x["a"] + x["b"] taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="quality") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=True, always_race_against=None, run_limit=1, ) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # intensification iteration #1 # run first config as incumbent if incumbent is None intent, run_info = intensifier.get_next_run( challengers=[self.config3], incumbent=None, run_history=self.rh, chooser=None, ) self.assertEqual(run_info.config, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.PROCESS_FIRST_CONFIG_RUN) # eval config 2 (=first run) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=None, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual(intensifier.n_iters, 1) # 1 intensification run complete! # regular intensification begins - run incumbent # Normally a challenger will be given, which in this case is the incumbent # But no more instances are available. So to prevent cicles # where No iteration happens, provide the challengers intent, run_info = intensifier.get_next_run( challengers=[ self.config2, self.config1, ], # since incumbent is run, no configs required incumbent=inc, run_history=self.rh, chooser=None, ) # no new TA runs as there are no more instances to run self.assertEqual(inc, self.config3) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 1, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) # run challenger now that the incumbent has been executed # So this call happen above, to save one iteration self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(run_info.config, self.config2) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) # challenger has a better performance, so incumbent has changed self.assertEqual(inc, self.config2) self.assertEqual(self.stats.inc_changed, 2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT ) # since there is no `always_race_against` self.assertFalse(intensifier.continue_challenger) self.assertEqual( intensifier.n_iters, 1) # iteration continues as `min_chall` condition is not met # intensification continues running incumbent again in same iteration... # run incumbent # Same here, No further instance-seed pairs for incumbent available # so above call gets the new config to run self.assertEqual(run_info.config, self.config2) # There is a transition from: # IntensifierStage.RUN_FIRST_CONFIG-> IntensifierStage.RUN_INCUMBENT # Because after the first run, incumbent is run. # Nevertheless, there is now a transition: # IntensifierStage.RUN_INCUMBENT->IntensifierStage.RUN_CHALLENGER # because in add_inc_run, there are more available instance pairs # FROM: IntensifierStage.RUN_INCUMBENT TO: IntensifierStage.RUN_INCUMBENT WHY: no more to run # if all <instance, seed> have been run, compare challenger performance # self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) # run challenger intent, run_info = intensifier.get_next_run( challengers=None, # don't need a new list here as old one is cont'd incumbent=inc, run_history=self.rh, chooser=None, ) self.assertEqual(run_info.config, self.config1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(intensifier.n_iters, 2) # 2 intensification run complete! # no configs should be left at the end with self.assertRaises(StopIteration): next(intensifier.configs_to_run) self.assertEqual( len( self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 1, ) self.assertEqual( len( self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True)), 1, ) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 1, ) def test_eval_challenger_3(self): """ test eval_challenger for a resumed SMAC run (first run with incumbent) """ def target(x): return x["a"] taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="quality") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=False, always_race_against=None, run_limit=1, ) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # adding run for incumbent configuration self.rh.add( config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None, ) # intensification - incumbent will be run, but not as RUN_FIRST_CONFIG stage intent_, run_info = intensifier.get_next_run( challengers=[self.config2], incumbent=self.config1, run_history=self.rh, chooser=None, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual( len( self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 2, ) def test_no_new_intensification_wo_challenger_run(self): """ This test ensures that no new iteration is started if no challenger run was conducted """ def target(x): return 2 * x["a"] + x["b"] taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="quality") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=True, always_race_against=None, run_limit=1, min_chall=1, ) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) intent, run_info = intensifier.get_next_run( challengers=[self.config3], incumbent=None, run_history=self.rh, chooser=None, ) self.assertEqual(run_info.config, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.PROCESS_FIRST_CONFIG_RUN) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=None, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(intensifier.n_iters, 1) # 1 intensification run complete! # regular intensification begins - run incumbent # No further instance-seed pairs for incumbent available # Here None challenger is suggested. Code jumps to next iteration # This causes a transition from RUN_INCUMBENT->RUN_CHALLENGER # But then, the next configuration to run is the incumbent # We don't rerun the incumbent (see message): # Challenger was the same as the current incumbent; Skipping challenger # Then, we try to get more challengers, but below all challengers # Provided are config3, the incumbent which means nothing more to run intent, run_info = intensifier.get_next_run( challengers=[self.config3 ], # since incumbent is run, no configs required incumbent=inc, run_history=self.rh, chooser=None, ) self.assertEqual(run_info.config, None) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) intensifier._next_iteration() # Add a configuration, then try to execute it afterwards self.assertEqual(intensifier.n_iters, 2) self.rh.add( config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=0, additional_info=None, ) intensifier.stage = IntensifierStage.RUN_CHALLENGER # In the upcoming get next run, the stage is RUN_CHALLENGER # so the intensifier tries to run config1. Nevertheless, # there are no further instances for this configuration available. # In this scenario, the intensifier produces a SKIP intent as an indication # that a new iteration must be initiated, and for code simplicity, # relies on a new call to get_next_run to yield more configurations intent, run_info = intensifier.get_next_run(challengers=[self.config1], incumbent=inc, run_history=self.rh, chooser=None) self.assertEqual(intent, RunInfoIntent.SKIP) # This doesn't return a config because the array of configs is exhausted intensifier.stage = IntensifierStage.RUN_CHALLENGER config, _ = intensifier.get_next_challenger(challengers=None, chooser=None) self.assertIsNone(config) # This finally gives a runable configuration intent, run_info = intensifier.get_next_run(challengers=[self.config2], incumbent=inc, run_history=self.rh, chooser=None) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) # 4 Iterations due to the proactive runs # of get next challenger self.assertEqual(intensifier.n_iters, 3) self.assertEqual(intensifier.num_chall_run, 1)
class Test_Hyperband(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.rh = RunHistory() self.cs = get_config_space() self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100}) self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0}) self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100}) self.scen = Scenario({ "cutoff_time": 2, 'cs': self.cs, "run_obj": 'runtime', "output_dir": '' }) self.stats = Stats(scenario=self.scen) self.stats.start_timing() self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) def test_update_stage(self): """ test initialization of all parameters and tracking variables """ intensifier = _Hyperband(stats=self.stats, traj_logger=None, rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False, instances=[1], initial_budget=0.1, max_budget=1, eta=2) intensifier._update_stage() self.assertEqual(intensifier.s, 3) self.assertEqual(intensifier.s_max, 3) self.assertEqual(intensifier.hb_iters, 0) self.assertIsInstance(intensifier.sh_intensifier, _SuccessiveHalving) self.assertEqual(intensifier.sh_intensifier.initial_budget, 0.125) self.assertEqual(intensifier.sh_intensifier.n_configs_in_stage, [8.0, 4.0, 2.0, 1.0]) # next HB stage intensifier._update_stage() self.assertEqual(intensifier.s, 2) self.assertEqual(intensifier.hb_iters, 0) self.assertEqual(intensifier.sh_intensifier.initial_budget, 0.25) self.assertEqual(intensifier.sh_intensifier.n_configs_in_stage, [4.0, 2.0, 1.0]) intensifier._update_stage() # s = 1 intensifier._update_stage() # s = 0 # HB iteration completed intensifier._update_stage() self.assertEqual(intensifier.s, intensifier.s_max) self.assertEqual(intensifier.hb_iters, 1) self.assertEqual(intensifier.sh_intensifier.initial_budget, 0.125) self.assertEqual(intensifier.sh_intensifier.n_configs_in_stage, [8.0, 4.0, 2.0, 1.0]) def test_eval_challenger(self): """ since hyperband uses eval_challenger and get_next_run of the internal successive halving, we don't test these method extensively """ def target(x): return 0.1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = _Hyperband(stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False, instances=[None], initial_budget=0.5, max_budget=1, eta=2) self.assertFalse(hasattr(intensifier, 's')) # Testing get_next_run - get next configuration intent, run_info = intensifier.get_next_run( challengers=[self.config2, self.config3], chooser=None, incumbent=None, run_history=self.rh) self.assertEqual(intensifier.s, intensifier.s_max) self.assertEqual(run_info.config, self.config2) # update to the last SH iteration of the given HB stage self.assertEqual(intensifier.s, 1) self.assertEqual(intensifier.s_max, 1) # We assume now that process results was called with below successes. # We track closely run execution through run_tracker, so this also # has to be update -- the fact that the succesive halving inside hyperband # processed the given configurations self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, seed=0, budget=1) intensifier.sh_intensifier.run_tracker[(self.config1, None, 0, 1)] = True self.rh.add(config=self.config2, cost=2, time=2, status=StatusType.SUCCESS, seed=0, budget=0.5) intensifier.sh_intensifier.run_tracker[(self.config2, None, 0, 0.5)] = True self.rh.add(config=self.config3, cost=3, time=2, status=StatusType.SUCCESS, seed=0, budget=0.5) intensifier.sh_intensifier.run_tracker[(self.config3, None, 0, 0.5)] = True intensifier.sh_intensifier.success_challengers = { self.config2, self.config3 } intensifier.sh_intensifier._update_stage(self.rh) intent, run_info = intensifier.get_next_run( challengers=[self.config2, self.config3], chooser=None, incumbent=None, run_history=self.rh) # evaluation should change the incumbent to config2 self.assertIsNotNone(run_info.config) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, inc_value = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.s, 0) self.assertEqual(inc_value, 0.1) self.assertEqual( list(self.rh.data.keys())[-1][0], self.rh.config_ids[self.config2]) self.assertEqual(self.stats.inc_changed, 1)
def runhistory_builder(ta,scenario_dic,rng): tae_runner = ExecuteTARun(ta=ta) scenario = Scenario(scenario_dic) stats = Stats(scenario=scenario) traj_logger = TrajLogger(stats=stats,output_dir="/home/dfki/Desktop/temp") # if tae_runner.stats is None: # new_smac =SMAC(scenario=scenario,tae_runner=tae_runner) # tae_runner.stats = new_smac.stats stats.start_timing() deful_config_builder = DefaultConfiguration(tae_runner,scenario,stats,traj_logger,rng) config_milad =deful_config_builder._select_configuration() config_milad._values = None config_milad._values = {'balancing:strategy': 'none', 'categorical_encoding:__choice__': 'one_hot_encoding', 'classifier:__choice__': 'random_forest', 'imputation:strategy': 'mean', 'preprocessor:__choice__': 'no_preprocessing', 'rescaling:__choice__': 'standardize', 'categorical_encoding:one_hot_encoding:use_minimum_fraction': 'True', 'classifier:random_forest:bootstrap': 'True', 'classifier:random_forest:criterion': 'gini', 'classifier:random_forest:max_depth': 10, 'classifier:random_forest:max_features': 0.5, 'classifier:random_forest:max_leaf_nodes': 'None', 'classifier:random_forest:min_impurity_decrease': 0.0, 'classifier:random_forest:min_samples_leaf': 1, 'classifier:random_forest:min_samples_split': 2, 'classifier:random_forest:min_weight_fraction_leaf': 0.0, 'classifier:random_forest:n_estimators': 100, 'categorical_encoding:one_hot_encoding:minimum_fraction': 0.01} # config_milad._values = {'balancing:strategy': 'none', # 'categorical_encoding:__choice__': 'no_encoding', # 'classifier:__choice__': 'random_forest', # 'imputation:strategy': 'mean', # 'preprocessor:__choice__': 'pca', # 'preprocessor:copy':True, # 'preprocessor:iterated_power':'auto', # 'preprocessor:n_components':'None', # 'preprocessor:random_state':'None', # 'preprocessor:svd_solver':'auto', # 'preprocessor:tol':0.0, # 'preprocessor:whiten':'False', # 'rescaling:__choice__': 'None', # 'classifier:random_forest:bootstrap': 'True', # 'classifier:random_forest:class_weight': 'None', # 'classifier:random_forest:criterion': 'gini', # 'classifier:random_forest:max_depth': 'None', # 'classifier:random_forest:max_features': 'auto', # 'classifier:random_forest:max_leaf_nodes': 'None', # 'classifier:random_forest:min_impurity_decrease': 0.0, # 'classifier:random_forest:min_impurity_split': '1e-07', # 'classifier:random_forest:min_samples_leaf': 1, # 'classifier:random_forest:min_samples_split': 2, # 'classifier:random_forest:min_weight_fraction_leaf': 0.0, # 'classifier:random_forest:n_estimators': 10, # 'classifier:random_forest:n_jobs': 1, # 'classifier:random_forest:oob_score': 'False', # 'classifier:random_forest:random_state': 'None', # 'classifier:random_forest:verbose': 0, # 'classifier:random_forest:warm_start': 'False', # } # config_milad._vector =None status, cost, runtime, additional_info = tae_runner.start(config=config_milad,instance=None) print(status, cost, runtime, additional_info) runhistory = RunHistory(aggregate_func=average_cost) runhistory.add( config=config_milad, cost=cost, time=runtime, status=status, instance_id=None, additional_info=additional_info) return runhistory
print('Starting to validate configurations') for i, entry in enumerate(trajectory): print('Starting to validate configuration %d/%d' % (i + 1, len(trajectory))) incumbent_id = entry.incumbent_id train_performance = entry.train_perf if incumbent_id not in incumbent_id_to_model: config = entry.incumbent logger = logging.getLogger('Testing:)') stats = Stats( Scenario({ 'cutoff_time': per_run_time_limit * 2, 'run_obj': 'quality', })) stats.start_timing() # To avoid the output "first run crashed"... stats.submitted_ta_runs += 1 stats.finished_ta_runs += 1 memory_lim = memory_limit_factor * automl_arguments['memory_limit'] ta = ExecuteTaFuncWithQueue( backend=automl.automl_._backend, autosklearn_seed=seed, resampling_strategy='test', memory_limit=memory_lim, disable_file_output=True, logger=logger, stats=stats, scoring_functions=scoring_functions, include=include, metric=automl_arguments['metric'],
class TestHyperband(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.rh = RunHistory() self.cs = get_config_space() self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100}) self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0}) self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100}) self.scen = Scenario({ "cutoff_time": 2, 'cs': self.cs, "run_obj": 'runtime', "output_dir": '' }) self.stats = Stats(scenario=self.scen) self.stats.start_timing() self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) def test_update_stage(self): """ test initialization of all parameters and tracking variables """ intensifier = Hyperband(tae_runner=None, stats=self.stats, traj_logger=None, rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False, instances=[1], initial_budget=0.1, max_budget=1, eta=2) intensifier._update_stage() self.assertEqual(intensifier.s, 3) self.assertEqual(intensifier.s_max, 3) self.assertEqual(intensifier.hb_iters, 0) self.assertIsInstance(intensifier.sh_intensifier, SuccessiveHalving) self.assertEqual(intensifier.sh_intensifier.initial_budget, 0.125) self.assertEqual(intensifier.sh_intensifier.n_configs_in_stage, [8.0, 4.0, 2.0, 1.0]) # next HB stage intensifier._update_stage() self.assertEqual(intensifier.s, 2) self.assertEqual(intensifier.hb_iters, 0) self.assertEqual(intensifier.sh_intensifier.initial_budget, 0.25) self.assertEqual(intensifier.sh_intensifier.n_configs_in_stage, [4.0, 2.0, 1.0]) intensifier._update_stage() # s = 1 intensifier._update_stage() # s = 0 # HB iteration completed intensifier._update_stage() self.assertEqual(intensifier.s, intensifier.s_max) self.assertEqual(intensifier.hb_iters, 1) self.assertEqual(intensifier.sh_intensifier.initial_budget, 0.125) self.assertEqual(intensifier.sh_intensifier.n_configs_in_stage, [8.0, 4.0, 2.0, 1.0]) def test_eval_challenger(self): """ since hyperband uses eval_challenger and get_next_challenger of the internal successive halving, we don't test these method extensively """ def target(x): return 0.1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Hyperband(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False, instances=[None], initial_budget=0.5, max_budget=1, eta=2) self.assertFalse(hasattr(intensifier, 's')) # Testing get_next_challenger - get next configuration config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None, run_history=self.rh) self.assertEqual(intensifier.s, intensifier.s_max) self.assertEqual(config, self.config2) # update to the last SH iteration of the given HB stage self.assertEqual(intensifier.s, 1) self.assertEqual(intensifier.s_max, 1) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, seed=0, budget=1) self.rh.add(config=self.config2, cost=2, time=2, status=StatusType.SUCCESS, seed=0, budget=0.5) self.rh.add(config=self.config3, cost=3, time=2, status=StatusType.SUCCESS, seed=0, budget=0.5) intensifier.sh_intensifier.curr_challengers = { self.config2, self.config3 } intensifier.sh_intensifier._update_stage(self.rh) # evaluation should change the incumbent to config2 inc, inc_value = intensifier.eval_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.s, 0) self.assertEqual(inc_value, 0.1) self.assertEqual(self.stats.ta_runs, 1) self.assertEqual( list(self.rh.data.keys())[-1][0], self.rh.config_ids[self.config2]) self.assertEqual(self.stats.inc_changed, 1)
def validate(self, config_mode: str = 'def', instance_mode: str = 'test', repetitions: int = 1, n_jobs: int = 1, backend: str = 'threading', runhistory: RunHistory = None, tae: ExecuteTARun = None): """ Validate configs on instances and save result in runhistory. Parameters ---------- config_mode: string what configurations to validate from [def, inc, def+inc, time, all], time means evaluation at timesteps 2^-4, 2^-3, 2^-2, 2^-1, 2^0, 2^1, ... instance_mode: string what instances to use for validation, from [train, test, train+test] repetitions: int number of repetitions in nondeterministic algorithms n_jobs: int number of parallel processes used by joblib runhistory: RunHistory or string or None runhistory to take data from tae: ExecuteTARun tae to be used. if none, will initialize ExecuteTARunOld Returns ------- runhistory: RunHistory runhistory with validated runs """ self.logger.debug( "Validating configs '%s' on instances '%s', repeating %d times" " with %d parallel runs on backend '%s'.", config_mode, instance_mode, repetitions, n_jobs, backend) # Reset runhistory self.rh = RunHistory(average_cost) # Get relevant configurations and instances configs = self._get_configs(config_mode) instances = self._get_instances(instance_mode) # If runhistory is given as string, load into memory if isinstance(runhistory, str): fn = runhistory runhistory = RunHistory(average_cost) runhistory.load_json(fn, self.scen.cs) # Get all runs needed as list runs = self.get_runs(configs, instances, repetitions=repetitions, runhistory=runhistory) # Create new Stats without limits inf_scen = Scenario({ 'run_obj': self.scen.run_obj, 'cutoff_time': self.scen.cutoff, 'output_dir': None }) inf_stats = Stats(inf_scen) inf_stats.start_timing() # Create TAE if not tae: tae = ExecuteTARunOld(ta=self.scen.ta, stats=inf_stats, run_obj=self.scen.run_obj, par_factor=self.scen.par_factor, cost_for_crash=self.scen.cost_for_crash) else: # Inject endless-stats tae.stats = inf_stats # Validate! run_results = self._validate_parallel(tae, runs, n_jobs, backend) # tae returns (status, cost, runtime, additional_info) # Add runs to RunHistory idx = 0 for result in run_results: self.rh.add(config=runs[idx]['config'], cost=result[1], time=result[2], status=result[0], instance_id=runs[idx]['inst'], seed=runs[idx]['seed'], additional_info=result[3]) idx += 1 # Save runhistory if not self.output.endswith('.json'): old = self.output self.output = os.path.join(self.output, 'validated_runhistory.json') self.logger.debug("Output is \"%s\", changing to \"%s\"!", old, self.output) base = os.path.split(self.output)[0] if not os.path.exists(base): self.logger.debug("Folder (\"%s\") doesn't exist, creating.", base) os.makedirs(base) self.logger.info("Saving validation-results in %s", self.output) self.rh.save_json(self.output) return self.rh
class TestHyperband(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.rh = RunHistory() self.cs = get_config_space() self.config1 = Configuration(self.cs, values={'a': 7, 'b': 11}) self.config2 = Configuration(self.cs, values={'a': 13, 'b': 17}) self.config3 = Configuration(self.cs, values={'a': 0, 'b': 7}) self.config4 = Configuration(self.cs, values={'a': 29, 'b': 31}) self.config5 = Configuration(self.cs, values={'a': 31, 'b': 33}) self.scen = Scenario({ "cutoff_time": 2, 'cs': self.cs, "run_obj": 'runtime', "output_dir": '' }) self.stats = Stats(scenario=self.scen) self.stats.start_timing() # Create the base object self.HB = Hyperband( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), deterministic=False, run_obj_time=False, instances=[1, 2, 3, 4, 5], n_seeds=2, initial_budget=2, max_budget=5, eta=2, ) def test_initialization(self): """Makes sure that a proper_HB is created""" # We initialize the HB with zero intensifier_instances self.assertEqual(len(self.HB.intensifier_instances), 0) # Add an instance to check the_HB initialization self.assertTrue(self.HB._add_new_instance(num_workers=1)) # Some default init self.assertEqual(self.HB.intensifier_instances[0].hb_iters, 0) self.assertEqual(self.HB.intensifier_instances[0].max_budget, 5) self.assertEqual(self.HB.intensifier_instances[0].initial_budget, 2) # Update the stage (self.HB.intensifier_instances[0]._update_stage(self.rh)) # Parameters properly passed to SH self.assertEqual( len(self.HB.intensifier_instances[0].sh_intensifier.inst_seed_pairs ), 10) self.assertEqual( self.HB.intensifier_instances[0].sh_intensifier.initial_budget, 2) self.assertEqual( self.HB.intensifier_instances[0].sh_intensifier.max_budget, 5) def test_process_results_via_sourceid(self): """Makes sure source id is honored when deciding which_HB instance will consume the result/run_info """ # Mock the_HB instance so we can make sure the correct item is passed for i in range(10): self.HB.intensifier_instances[i] = mock.Mock() self.HB.intensifier_instances[i].process_results.return_value = ( self.config1, 0.5) # make iter false so the mock object is not overwritten self.HB.intensifier_instances[i].iteration_done = False # randomly create run_infos and push into HB. Then we will make # sure they got properly allocated for i in np.random.choice(list(range(10)), 30): run_info = RunInfo( config=self.config1, instance=0, instance_specific="0", cutoff=None, seed=0, capped=False, budget=0.0, source_id=i, ) # make sure results aren't messed up via magic variable # That is we check only the proper_HB instance has this magic = time.time() result = RunValue(cost=1, time=0.5, status=StatusType.SUCCESS, starttime=1, endtime=2, additional_info=magic) self.HB.process_results(run_info=run_info, incumbent=None, run_history=self.rh, time_bound=None, result=result, log_traj=False) # Check the call arguments of each sh instance and make sure # it is the correct one # First the expected one self.assertEqual( self.HB.intensifier_instances[i].process_results.call_args[1] ['run_info'], run_info) self.assertEqual( self.HB.intensifier_instances[i].process_results.call_args[1] ['result'], result) all_other_run_infos, all_other_results = [], [] for j in range(len(self.HB.intensifier_instances)): # Skip the expected_HB instance if i == j: continue if self.HB.intensifier_instances[ j].process_results.call_args is None: all_other_run_infos.append(None) else: all_other_run_infos.append( self.HB.intensifier_instances[j].process_results. call_args[1]['run_info']) all_other_results.append( self.HB.intensifier_instances[j].process_results. call_args[1]['result']) self.assertNotIn(run_info, all_other_run_infos) self.assertNotIn(result, all_other_results) def test_get_next_run_single_HB_instance(self): """Makes sure that a single_HB instance returns a valid config""" challengers = [self.config1, self.config2, self.config3, self.config4] for i in range(30): intent, run_info = self.HB.get_next_run( challengers=challengers, incumbent=None, chooser=None, run_history=self.rh, num_workers=1, ) # Regenerate challenger list challengers = [c for c in challengers if c != run_info.config] if intent == RunInfoIntent.WAIT: break # Add the config to self.rh in order to make HB aware that this # config/instance was launched self.rh.add( config=run_info.config, cost=10, time=0.0, status=StatusType.RUNNING, instance_id=run_info.instance, seed=run_info.seed, budget=run_info.budget, ) # smax==1 (int(np.floor(np.log(self.max_budget / self.initial_budget) / np.log(self. eta)))) self.assertEqual(self.HB.intensifier_instances[0].s_max, 1) # And we do not even complete 1 iteration, so s has to be 1 self.assertEqual(self.HB.intensifier_instances[0].s, 1) # We should not create more_HB instance intensifier_instances self.assertEqual(len(self.HB.intensifier_instances), 1) # We are running with: # 'all_budgets': array([2.5, 5. ]) -> 2 intensifier_instances per config top # 'n_configs_in_stage': [2.0, 1.0], # This means we run int(2.5) + 2.0 = 4 runs before waiting self.assertEqual(i, 4) # Also, check the internals of the unique sh instance # sh_initial_budget==2.5 (self.eta ** -self.s * self.max_budget) self.assertEqual( self.HB.intensifier_instances[0].sh_intensifier.initial_budget, 2) # n_challengers=2 (int(np.floor((self.s_max + 1) / (self.s + 1)) * self.eta ** self.s)) self.assertEqual( len(self.HB.intensifier_instances[0].sh_intensifier. n_configs_in_stage), 2) def test_get_next_run_multiple_HB_instances(self): """Makes sure that two _HB instance can properly coexist and tag run_info properly""" # We allow 2_HB instance to be created. This means, we have a newer iteration # to expect in hyperband challengers = [self.config1, self.config2, self.config3, self.config4] run_infos = [] for i in range(30): intent, run_info = self.HB.get_next_run( challengers=challengers, incumbent=None, chooser=None, run_history=self.rh, num_workers=2, ) run_infos.append(run_info) # Regenerate challenger list challengers = [c for c in challengers if c != run_info.config] # Add the config to self.rh in order to make HB aware that this # config/instance was launched if intent == RunInfoIntent.WAIT: break self.rh.add( config=run_info.config, cost=10, time=0.0, status=StatusType.RUNNING, instance_id=run_info.instance, seed=run_info.seed, budget=run_info.budget, ) # We have not completed an iteration self.assertEqual(self.HB.intensifier_instances[0].hb_iters, 0) # Because n workers is now 2, we expect 2 sh intensifier_instances self.assertEqual(len(self.HB.intensifier_instances), 2) # Each of the intensifier_instances should have s equal to 1 # As no iteration has been completed self.assertEqual(self.HB.intensifier_instances[0].s_max, 1) self.assertEqual(self.HB.intensifier_instances[0].s, 1) self.assertEqual(self.HB.intensifier_instances[1].s_max, 1) self.assertEqual(self.HB.intensifier_instances[1].s, 1) # First let us check everything makes sense in_HB-SH-0 HB-SH-0 self.assertEqual( self.HB.intensifier_instances[0].sh_intensifier.initial_budget, 2) self.assertEqual( self.HB.intensifier_instances[0].sh_intensifier.max_budget, 5) self.assertEqual( len(self.HB.intensifier_instances[0].sh_intensifier. n_configs_in_stage), 2) self.assertEqual( self.HB.intensifier_instances[1].sh_intensifier.initial_budget, 2) self.assertEqual( self.HB.intensifier_instances[1].sh_intensifier.max_budget, 5) self.assertEqual( len(self.HB.intensifier_instances[1].sh_intensifier. n_configs_in_stage), 2) # We are running with: # + 4 runs for sh instance 0 ('all_budgets': array([2.5, 5. ]), 'n_configs_in_stage': [2.0, 1.0]) # that is, for SH0 we run in stage==0 int(2.5) intensifier_instances * 2.0 configs # And this times 2 because we have 2_HB intensifier_instances self.assertEqual(i, 8) # Adding a new worker is not possible as we already have 2 intensifier_instances # and n_workers==2 intent, run_info = self.HB.get_next_run( challengers=challengers, incumbent=None, chooser=None, run_history=self.rh, num_workers=2, ) self.assertEqual(intent, RunInfoIntent.WAIT) def test_add_new_instance(self): """Test whether we can add a instance and when we should not""" # By default we do not create a_HB # test adding the first instance! self.assertEqual(len(self.HB.intensifier_instances), 0) self.assertTrue(self.HB._add_new_instance(num_workers=1)) self.assertEqual(len(self.HB.intensifier_instances), 1) self.assertIsInstance(self.HB.intensifier_instances[0], _Hyperband) # A second call should not add a new_HB instance self.assertFalse(self.HB._add_new_instance(num_workers=1)) # We try with 2_HB instance active # We effectively return true because we added a new_HB instance self.assertTrue(self.HB._add_new_instance(num_workers=2)) self.assertEqual(len(self.HB.intensifier_instances), 2) self.assertIsInstance(self.HB.intensifier_instances[1], _Hyperband) # Trying to add a third one should return false self.assertFalse(self.HB._add_new_instance(num_workers=2)) self.assertEqual(len(self.HB.intensifier_instances), 2) def _exhaust_run_and_get_incumbent(self, sh, rh, num_workers=2): """ Runs all provided configs on all intensifier_instances and return the incumbent as a nice side effect runhistory/stats are properly filled """ challengers = [self.config1, self.config2, self.config3, self.config4] incumbent = None for i in range(100): try: intent, run_info = sh.get_next_run( challengers=challengers, incumbent=None, chooser=None, run_history=rh, num_workers=num_workers, ) except ValueError as e: # Get configurations until you run out of them print(e) break # Regenerate challenger list challengers = [c for c in challengers if c != run_info.config] if intent == RunInfoIntent.WAIT: break result = target_from_run_info(run_info) rh.add( config=run_info.config, cost=result.cost, time=result.time, status=result.status, instance_id=run_info.instance, seed=run_info.seed, budget=run_info.budget, ) incumbent, inc_perf = sh.process_results( run_info=run_info, incumbent=incumbent, run_history=rh, time_bound=100.0, result=result, log_traj=False, ) return incumbent, inc_perf def test_parallel_same_as_serial_HB(self): """Makes sure we behave the same as a serial run at the end""" # Get the run_history for a_HB instance run: rh = RunHistory() stats = Stats(scenario=self.scen) stats.start_timing() _HB = _Hyperband( stats=stats, traj_logger=TrajLogger(output_dir=None, stats=stats), rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False, instances=[1, 2, 3, 4, 5], initial_budget=2, max_budget=5, eta=2, ) incumbent, inc_perf = self._exhaust_run_and_get_incumbent( _HB, rh, num_workers=1) # Just to make sure nothing has changed from the_HB instance side to make # this check invalid: # We add config values, so config 3 with 0 and 7 should be the lesser cost self.assertEqual(incumbent, self.config3) self.assertEqual(inc_perf, 7.0) # Do the same for HB, but have multiple_HB instance in there # This_HB instance will be created via num_workers==2 # in self._exhaust_run_and_get_incumbent HB = Hyperband( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), deterministic=True, run_obj_time=False, instances=[1, 2, 3, 4, 5], initial_budget=2, max_budget=5, eta=2, ) incumbent_phb, inc_perf_phb = self._exhaust_run_and_get_incumbent( HB, self.rh) self.assertEqual(incumbent, incumbent_phb) # This makes sure there is a single incumbent in HB self.assertEqual(inc_perf, inc_perf_phb) # We don't want to loose any configuration, and particularly # we want to make sure the values of_HB instance to HB match self.assertEqual(len(self.rh.data), len(rh.data)) # Because it is a deterministic run, the run histories must be the # same on exhaustion self.assertDictEqual(self.rh.data, rh.data)
class SMBO(BaseSolver): def __init__(self, scenario, tae_runner=None, acquisition_function=None, model=None, runhistory2epm=None, stats=None, rng=None): ''' Interface that contains the main Bayesian optimization loop Parameters ---------- scenario: smac.scenario.scenario.Scenario Scenario object tae_runner: object object that implements the following method to call the target algorithm (or any other arbitrary function): run(self, config) If not set, it will be initialized with the tae.ExecuteTARunOld() acquisition_function : AcquisitionFunction Object that implements the AbstractAcquisitionFunction. Will use EI if not set. model : object Model that implements train() and predict(). Will use a RandomForest if not set. runhistory2epm : RunHistory2EMP Object that implements the AbstractRunHistory2EPM. If None, will use RunHistory2EPM4Cost if objective is cost or RunHistory2EPM4LogCost if objective is runtime. stats: Stats optional stats object rng: numpy.random.RandomState Random number generator ''' if stats: self.stats = stats else: self.stats = Stats(scenario) self.runhistory = RunHistory() self.logger = logging.getLogger("smbo") if rng is None: self.num_run = np.random.randint(1234567980) self.rng = np.random.RandomState(seed=self.num_run) elif isinstance(rng, int): self.num_run = rng self.rng = np.random.RandomState(seed=rng) elif isinstance(rng, np.random.RandomState): self.num_run = rng.randint(1234567980) self.rng = rng else: raise TypeError('Unknown type %s for argument rng. Only accepts ' 'None, int or np.random.RandomState' % str(type(rng))) self.scenario = scenario self.config_space = scenario.cs self.traj_logger = TrajLogger(output_dir=self.scenario.output_dir, stats=self.stats) self.types = get_types(self.config_space, scenario.feature_array) if model is None: self.model = RandomForestWithInstances( self.types, scenario.feature_array, seed=self.rng.randint(1234567980)) else: self.model = model if acquisition_function is None: self.acquisition_func = EI(self.model) else: self.acquisition_func = acquisition_function self.local_search = LocalSearch(self.acquisition_func, self.config_space) self.incumbent = None if tae_runner is None: self.executor = ExecuteTARunOld(ta=scenario.ta, stats=self.stats, run_obj=scenario.run_obj, par_factor=scenario.par_factor) else: self.executor = tae_runner self.inten = Intensifier( executor=self.executor, stats=self.stats, traj_logger=self.traj_logger, instances=self.scenario.train_insts, cutoff=self.scenario.cutoff, deterministic=self.scenario.deterministic, run_obj_time=self.scenario.run_obj == "runtime", instance_specifics=self.scenario.instance_specific) num_params = len(self.config_space.get_hyperparameters()) self.objective = average_cost if self.scenario.run_obj == "runtime": if runhistory2epm is None: # if we log the performance data, # the RFRImputator will already get # log transform data from the runhistory cutoff = np.log10(self.scenario.cutoff) threshold = np.log10(self.scenario.cutoff * self.scenario.par_factor) imputor = RFRImputator(cs=self.config_space, rs=self.rng, cutoff=cutoff, threshold=threshold, model=self.model, change_threshold=0.01, max_iter=10) self.rh2EPM = RunHistory2EPM4LogCost(scenario=self.scenario, num_params=num_params, success_states=[ StatusType.SUCCESS, ], impute_censored_data=True, impute_state=[ StatusType.TIMEOUT, ], imputor=imputor) else: self.rh2EPM = runhistory2epm elif self.scenario.run_obj == 'quality': if runhistory2epm is None: self.rh2EPM = RunHistory2EPM4Cost\ (scenario=self.scenario, num_params=num_params, success_states=[StatusType.SUCCESS, ], impute_censored_data=False, impute_state=None) else: self.rh2EPM = runhistory2epm else: raise ValueError('Unknown run objective: %s. Should be either ' 'quality or runtime.' % self.scenario.run_obj) def run_initial_design(self): ''' runs algorithm runs for a initial design; default implementation: running the default configuration on a random instance-seed pair Side effect: adds runs to self.runhistory Returns ------- incumbent: Configuration() initial incumbent configuration ''' default_conf = self.config_space.get_default_configuration() self.incumbent = default_conf # add this incumbent right away to have an entry to time point 0 self.traj_logger.add_entry(train_perf=2**31, incumbent_id=1, incumbent=self.incumbent) rand_inst_id = self.rng.randint(0, len(self.scenario.train_insts)) # ignore instance specific values rand_inst = self.scenario.train_insts[rand_inst_id] if self.scenario.deterministic: initial_seed = 0 else: initial_seed = random.randint(0, MAXINT) status, cost, runtime, additional_info = self.executor.start( default_conf, instance=rand_inst, cutoff=self.scenario.cutoff, seed=initial_seed, instance_specific=self.scenario.instance_specific.get( rand_inst, "0")) if status in [StatusType.CRASHED or StatusType.ABORT]: self.logger.critical("First run crashed -- Abort") sys.exit(1) self.runhistory.add(config=default_conf, cost=cost, time=runtime, status=status, instance_id=rand_inst, seed=initial_seed, additional_info=additional_info) defaul_inst_seeds = set( self.runhistory.get_runs_for_config(default_conf)) default_perf = self.objective(default_conf, self.runhistory, defaul_inst_seeds) self.runhistory.update_cost(default_conf, default_perf) self.stats.inc_changed += 1 # first incumbent self.traj_logger.add_entry(train_perf=default_perf, incumbent_id=self.stats.inc_changed, incumbent=self.incumbent) return default_conf def run(self, max_iters=10): ''' Runs the Bayesian optimization loop for max_iters iterations Parameters ---------- max_iters: int The maximum number of iterations Returns ---------- incumbent: np.array(1, H) The best found configuration ''' self.stats.start_timing() #self.runhistory = RunHisory() self.incumbent = self.run_initial_design() # Main BO loop iteration = 1 while True: if self.scenario.shared_model: pSMAC.read(run_history=self.runhistory, output_directory=self.scenario.output_dir, configuration_space=self.config_space, logger=self.logger) start_time = time.time() X, Y = self.rh2EPM.transform(self.runhistory) self.logger.debug("Search for next configuration") # get all found configurations sorted according to acq challengers = self.choose_next(X, Y) time_spend = time.time() - start_time logging.debug( "Time spend to choose next configurations: %.2f sec" % (time_spend)) self.logger.debug("Intensify") self.incumbent, inc_perf = self.inten.intensify( challengers=challengers, incumbent=self.incumbent, run_history=self.runhistory, objective=self.objective, time_bound=max(0.01, time_spend)) # TODO: Write run history into database if self.scenario.shared_model: pSMAC.write(run_history=self.runhistory, output_directory=self.scenario.output_dir, num_run=self.num_run) if iteration == max_iters: break iteration += 1 logging.debug( "Remaining budget: %f (wallclock), %f (ta costs), %f (target runs)" % (self.stats.get_remaing_time_budget(), self.stats.get_remaining_ta_budget(), self.stats.get_remaining_ta_runs())) if self.stats.is_budget_exhausted(): break self.stats.print_stats(debug_out=True) return self.incumbent def choose_next(self, X, Y, num_interleaved_random=1010, num_configurations_by_random_search_sorted=1000, num_configurations_by_local_search=10): """Choose next candidate solution with Bayesian optimization. Parameters ---------- X : (N, D) numpy array Each row contains a configuration and one set of instance features. Y : (N, O) numpy array The function values for each configuration instance pair. Returns ------- list List of 2020 suggested configurations to evaluate. """ self.model.train(X, Y) if self.runhistory.empty(): incumbent_value = 0.0 elif self.incumbent is None: # TODO try to calculate an incumbent from the runhistory! incumbent_value = 0.0 else: incumbent_value = self.runhistory.get_cost(self.incumbent) self.acquisition_func.update(model=self.model, eta=incumbent_value) # Remove dummy acquisition function value next_configs_by_random_search = [ x[1] for x in self._get_next_by_random_search( num_points=num_interleaved_random) ] # Get configurations sorted by EI next_configs_by_random_search_sorted = \ self._get_next_by_random_search( num_configurations_by_random_search_sorted, _sorted=True) next_configs_by_local_search = \ self._get_next_by_local_search(num_configurations_by_local_search) next_configs_by_acq_value = next_configs_by_random_search_sorted + \ next_configs_by_local_search next_configs_by_acq_value.sort(reverse=True, key=lambda x: x[0]) self.logger.debug( "First 10 acq func values of selected configurations: %s" % (str([_[0] for _ in next_configs_by_acq_value[:10]]))) next_configs_by_acq_value = [_[1] for _ in next_configs_by_acq_value] challengers = list( itertools.chain(*zip(next_configs_by_acq_value, next_configs_by_random_search))) return challengers def _get_next_by_random_search(self, num_points=1000, _sorted=False): """Get candidate solutions via local search. Parameters ---------- num_points : int, optional (default=10) Number of local searches and returned values. _sorted : bool, optional (default=True) Whether to sort the candidate solutions by acquisition function value. Returns ------- list : (acquisition value, Candidate solutions) """ rand_configs = self.config_space.sample_configuration(size=num_points) if _sorted: imputed_rand_configs = map(ConfigSpace.util.impute_inactive_values, rand_configs) imputed_rand_configs = [ x.get_array() for x in imputed_rand_configs ] imputed_rand_configs = np.array(imputed_rand_configs, dtype=np.float64) acq_values = self.acquisition_func(imputed_rand_configs) # From here # http://stackoverflow.com/questions/20197990/how-to-make-argsort-result-to-be-random-between-equal-values random = self.rng.rand(len(acq_values)) # Last column is primary sort key! indices = np.lexsort((random.flatten(), acq_values.flatten())) for i in range(len(rand_configs)): rand_configs[i].origin = 'Random Search (sorted)' # Cannot use zip here because the indices array cannot index the # rand_configs list, because the second is a pure python list return [(acq_values[ind][0], rand_configs[ind]) for ind in indices[::-1]] else: for i in range(len(rand_configs)): rand_configs[i].origin = 'Random Search' return [(0, rand_configs[i]) for i in range(len(rand_configs))] def _get_next_by_local_search(self, num_points=10): """Get candidate solutions via local search. In case acquisition function values tie, these will be broken randomly. Parameters ---------- num_points : int, optional (default=10) Number of local searches and returned values. Returns ------- list : (acquisition value, Candidate solutions), ordered by their acquisition function value """ configs_acq = [] # Start N local search from different random start points for i in range(num_points): if i == 0 and self.incumbent is not None: start_point = self.incumbent else: start_point = self.config_space.sample_configuration() configuration, acq_val = self.local_search.maximize(start_point) configuration.origin = 'Local Search' configs_acq.append((acq_val[0][0], configuration)) # shuffle for random tie-break random.shuffle(configs_acq, self.rng.rand) # sort according to acq value # and return n best configurations configs_acq.sort(reverse=True, key=lambda x: x[0]) return configs_acq
class TestIntensify(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.rh = RunHistory() self.cs = get_config_space() self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100}) self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0}) self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100}) self.scen = Scenario({ "cutoff_time": 2, 'cs': self.cs, "run_obj": 'runtime', "output_dir": '' }) self.stats = Stats(scenario=self.scen) self.stats.start_timing() self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) def test_race_challenger(self): """ test _race_challenger without adaptive capping """ def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) intensifier.N = 1 inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh) self.assertEqual(inc, self.config2) def test_race_challenger_2(self): """ test _race_challenger with adaptive capping """ def target(x): time.sleep(1.5) return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1]) self.rh.add(config=self.config1, cost=.001, time=0.001, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None) intensifier.N = 1 # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc = intensifier._race_challenger( challenger=self.config2, incumbent=self.config1, run_history=self.rh, ) # self.assertTrue(False) self.assertEqual(inc, self.config1) def test_race_challenger_3(self): """ test _race_challenger with adaptive capping on a previously capped configuration """ def target(config: Configuration, seed: int, instance: str): if instance == 1: time.sleep(2.1) else: time.sleep(0.6) return (config['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime", par_factor=1) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), cutoff=2, instances=[1]) self.rh.add(config=self.config1, cost=0.5, time=.5, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None) # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) self.assertEqual(inc, self.config1) # further run for incumbent self.rh.add(config=self.config1, cost=2, time=2, status=StatusType.TIMEOUT, instance_id=2, seed=12345, additional_info=None) # give config2 a second chance - now it should run on both instances # run on instance 1 config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) # run on instance 2 config, _ = intensifier.get_next_challenger(challengers=[self.config3], chooser=None) self.assertEqual(config, self.config2) self.assertTrue(intensifier.continue_challenger) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) # the incumbent should still be config1 because # config2 should get on inst 1 a full timeout # such that c(config1) = 1.25 and c(config2) close to 1.3 self.assertEqual(inc, self.config1) # the capped run should not be counted in runs_perf_config self.assertAlmostEqual(self.rh.num_runs_per_config[2], 2) self.assertFalse(intensifier.continue_challenger) def test_race_challenger_large(self): """ test _race_challenger using solution_quality """ def target(x): return 1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), run_obj_time=False, deterministic=True) for i in range(10): self.rh.add(config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=12345, additional_info=None) intensifier.stage = IntensifierStage.RUN_CHALLENGER # tie on first instances and then challenger should always win # and be returned as inc while True: config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) # stop when challenger evaluation is over if not intensifier.stage == IntensifierStage.RUN_CHALLENGER: break self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True) self.assertEqual(len(runs), 10) def test_race_challenger_large_blocked_seed(self): """ test _race_challenger whether seeds are blocked for challenger runs """ def target(x): return 1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), run_obj_time=False, deterministic=False) for i in range(10): self.rh.add(config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) intensifier.stage = IntensifierStage.RUN_CHALLENGER # tie on first instances and then challenger should always win # and be returned as inc while True: config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc = intensifier._race_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) # stop when challenger evaluation is over if not intensifier.stage == IntensifierStage.RUN_CHALLENGER: break self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True) self.assertEqual(len(runs), 10) seeds = sorted([r.seed for r in runs]) self.assertEqual(seeds, list(range(10)), seeds) def test_add_inc_run_det(self): """ test _add_inc_run() """ def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) # since we assume deterministic=1, # the second call should not add any more runs # given only one instance intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) def test_add_inc_run_nondet(self): """ test _add_inc_run() """ def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, runhistory=self.rh, run_obj="solution_quality") intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], deterministic=False) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 2, self.rh.data) runs = self.rh.get_runs_for_config(config=self.config1, only_max_observed_budget=True) # exactly one run on each instance self.assertIn(1, [runs[0].instance, runs[1].instance]) self.assertIn(2, [runs[0].instance, runs[1].instance]) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 3, self.rh.data) def test_get_next_challenger(self): """ test get_next_challenger() """ intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True) intensifier.stage = IntensifierStage.RUN_CHALLENGER # get a new challenger to evaluate config, new = intensifier.get_next_challenger( challengers=[self.config1, self.config2], chooser=None) self.assertEqual(config, self.config1, intensifier.current_challenger) self.assertEqual(intensifier._chall_indx, 1) self.assertEqual(intensifier.N, 1) self.assertTrue(new) # when already evaluating a challenger, return the same challenger intensifier.to_run = [(1, 1, 0)] config, new = intensifier.get_next_challenger( challengers=[self.config2], chooser=None) self.assertEqual(config, self.config1, intensifier.current_challenger) self.assertEqual(intensifier._chall_indx, 1) self.assertFalse(new) def test_generate_challenger(self): """ test generate_challenger() """ # test get generator from a list of challengers intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=None, rng=np.random.RandomState(12345), instances=[1], deterministic=True) gen = intensifier._generate_challengers( challengers=[self.config1, self.config2], chooser=None) self.assertEqual(next(gen), self.config1) self.assertEqual(next(gen), self.config2) self.assertRaises(StopIteration, next, gen) # test get generator from a chooser - would return only 1 configuration intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=None, rng=np.random.RandomState(12345), instances=[1], deterministic=True) chooser = SMAC4AC(self.scen, rng=1).solver.epm_chooser gen = intensifier._generate_challengers(challengers=None, chooser=chooser) self.assertEqual(next(gen).get_dictionary(), {'a': 24, 'b': 68}) self.assertRaises(StopIteration, next, gen) # when both are none, raise error with self.assertRaisesRegex(ValueError, "No configurations/chooser provided"): intensifier._generate_challengers(challengers=None, chooser=None) def test_eval_challenger(self): """ test eval_challenger() - a complete intensification run """ def target(x): return x['a'] taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=False, always_race_against=self.config3, run_limit=1) # run incumbent first if it was not run before config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config1, self.config3], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=None, run_history=self.rh, ) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) # run challenger now that the incumbent has been executed config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config1, self.config3], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # challenger should have a better performance, so incumbent should have changed self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_DEFAULT) self.assertFalse(intensifier.continue_challenger) # run `always_race_against` now since the incumbent has changed config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config1, self.config3], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(inc, self.config1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 1) self.assertEqual(intensifier.n_iters, 1) self.assertIsInstance(intensifier.configs_to_run, collections.Iterator) with self.assertRaises(StopIteration): next(intensifier.configs_to_run)
def validate( self, config_mode: Union[str, typing.List[Configuration]] = 'def', instance_mode: Union[str, typing.List[str]] = 'test', repetitions: int = 1, n_jobs: int = 1, backend: str = 'threading', runhistory: typing.Optional[RunHistory] = None, tae: BaseRunner = None, output_fn: typing.Optional[str] = None, ) -> RunHistory: """ Validate configs on instances and save result in runhistory. If a runhistory is provided as input it is important that you run it on the same/comparable hardware. side effect: if output is specified, saves runhistory to specified output directory. Parameters ---------- config_mode: str or list<Configuration> string or directly a list of Configuration. string from [def, inc, def+inc, wallclock_time, cpu_time, all]. time evaluates at cpu- or wallclock-timesteps of: [max_time/2^0, max_time/2^1, max_time/2^3, ..., default] with max_time being the highest recorded time instance_mode: str or list<str> what instances to use for validation, either from [train, test, train+test] or directly a list of instances repetitions: int number of repetitions in nondeterministic algorithms n_jobs: int number of parallel processes used by joblib backend: str what backend joblib should use for parallel runs runhistory: RunHistory optional, RunHistory-object to reuse runs tae: BaseRunner tae to be used. if None, will initialize ExecuteTARunOld output_fn: str path to runhistory to be saved. if the suffix is not '.json', will be interpreted as directory and filename will be 'validated_runhistory.json' Returns ------- runhistory: RunHistory runhistory with validated runs """ self.logger.debug( "Validating configs '%s' on instances '%s', repeating %d times" " with %d parallel runs on backend '%s'.", config_mode, instance_mode, repetitions, n_jobs, backend) # Get all runs to be evaluated as list runs, validated_rh = self._get_runs(config_mode, instance_mode, repetitions, runhistory) # Create new Stats without limits inf_scen = Scenario({ 'run_obj': self.scen.run_obj, 'cutoff_time': self.scen.cutoff, # type: ignore[attr-defined] # noqa F821 'output_dir': "" }) inf_stats = Stats(inf_scen) inf_stats.start_timing() # Create TAE if not tae: tae = ExecuteTARunOld( ta=self.scen.ta, # type: ignore[attr-defined] # noqa F821 stats=inf_stats, run_obj=self.scen.run_obj, par_factor=self.scen. par_factor, # type: ignore[attr-defined] # noqa F821 cost_for_crash=self.scen.cost_for_crash ) # type: ignore[attr-defined] # noqa F821 else: # Inject endless-stats tae.stats = inf_stats # Validate! run_results = self._validate_parallel(tae, runs, n_jobs, backend, runhistory) assert len(run_results) == len(runs), (run_results, runs) # tae returns (status, cost, runtime, additional_info) # Add runs to RunHistory for run, result in zip(runs, run_results): validated_rh.add(config=run.config, cost=result.cost, time=result.time, status=result.status, instance_id=run.inst, seed=run.seed, additional_info=result.additional_info) self._save_results(validated_rh, output_fn, backup_fn="validated_runhistory.json") return validated_rh
class TestIntensify(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.rh = RunHistory(aggregate_func=average_cost) self.cs = get_config_space() self.config1 = Configuration(self.cs, values={'a': 0, 'b': 100}) self.config2 = Configuration(self.cs, values={'a': 100, 'b': 0}) self.config3 = Configuration(self.cs, values={'a': 100, 'b': 100}) self.scen = Scenario({ "cutoff_time": 2, 'cs': self.cs, "run_obj": 'runtime', "output_dir": '' }) self.stats = Stats(scenario=self.scen) self.stats.start_timing() self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__) def test_compare_configs_no_joint_set(self): intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) for i in range(2): self.rh.add(config=self.config1, cost=2, time=2, status=StatusType.SUCCESS, instance_id=1, seed=i, additional_info=None) for i in range(2, 5): self.rh.add(config=self.config2, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=i, additional_info=None) # The sets for the incumbent are completely disjoint. conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) self.assertIsNone(conf) # The incumbent has still one instance-seed pair left on which the # challenger was not run yet. self.rh.add(config=self.config2, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=1, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) self.assertIsNone(conf) def test_compare_configs_chall(self): ''' challenger is better ''' intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=2, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config2, cost=0, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) # challenger has enough runs and is better self.assertEqual(conf, self.config2, "conf: %s" % (conf)) def test_compare_configs_inc(self): ''' incumbent is better ''' intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config2, cost=2, time=2, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) # challenger worse than inc self.assertEqual(conf, self.config1, "conf: %s" % (conf)) def test_compare_configs_unknow(self): ''' challenger is better but has less runs; -> no decision (None) ''' intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config1, cost=1, time=2, status=StatusType.SUCCESS, instance_id=2, seed=None, additional_info=None) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=2, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) # challenger worse than inc self.assertIsNone(conf, "conf: %s" % (conf)) def test_race_challenger(self): ''' test _race_challenger without adaptive capping ''' def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) self.assertEqual(inc, self.config2) def test_race_challenger_2(self): ''' test _race_challenger with adaptive capping ''' def target(x): time.sleep(1.5) return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1]) self.rh.add(config=self.config1, cost=.001, time=0.001, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None) # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # self.assertTrue(False) self.assertEqual(inc, self.config1) def test_race_challenger_3(self): ''' test _race_challenger with adaptive capping on a previously capped configuration ''' def target(config: Configuration, seed: int, instance: str): if instance == 1: time.sleep(2.1) else: time.sleep(0.6) return (config['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime", par_factor=1) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), cutoff=2, instances=[1]) self.rh.add(config=self.config1, cost=0.5, time=.5, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None) # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # self.assertTrue(False) self.assertEqual(inc, self.config1) # further run for incumbent self.rh.add(config=self.config1, cost=2, time=2, status=StatusType.TIMEOUT, instance_id=2, seed=12345, additional_info=None) # give config2 a second chance inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # the incumbent should still be config1 because # config2 should get on inst 1 a full timeout # such that c(config1) = 1.25 and c(config2) close to 1.3 self.assertEqual(inc, self.config1) # the capped run should not be counted in runs_perf_config self.assertAlmostEqual(self.rh.runs_per_config[2], 2) def test_race_challenger_large(self): ''' test _race_challenger using solution_quality ''' def target(x): return 1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), deterministic=True) for i in range(10): self.rh.add(config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=12345, additional_info=None) # tie on first instances and then challenger should always win # and be returned as inc inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # self.assertTrue(False) self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1, self.rh.get_cost(self.config2)) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2) self.assertEqual(len(runs), 10) def test_race_challenger_large_blocked_seed(self): ''' test _race_challenger whether seeds are blocked for challenger runs ''' def target(x): return 1 taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), deterministic=False) for i in range(10): self.rh.add(config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) # tie on first instances and then challenger should always win # and be returned as inc inc = intensifier._race_challenger(challenger=self.config2, incumbent=self.config1, run_history=self.rh, aggregate_func=average_cost) # self.assertTrue(False) self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1, self.rh.get_cost(self.config2)) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2) self.assertEqual(len(runs), 10) seeds = sorted([r.seed for r in runs]) self.assertEqual(seeds, list(range(10)), seeds) def test_add_inc_run_det(self): ''' test _add_inc_run() ''' def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) # since we assume deterministic=1, # the second call should not add any more runs # given only one instance intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) def test_add_inc_run_nondet(self): ''' test _add_inc_run() ''' def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], deterministic=False) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 1, self.rh.data) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 2, self.rh.data) runs = self.rh.get_runs_for_config(config=self.config1) # exactly one run on each instance self.assertIn(1, [runs[0].instance, runs[1].instance]) self.assertIn(2, [runs[0].instance, runs[1].instance]) intensifier._add_inc_run(incumbent=self.config1, run_history=self.rh) self.assertEqual(len(self.rh.data), 3, self.rh.data) def test_adaptive_capping(self): ''' test _adapt_cutoff() ''' intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(5)), deterministic=False) for i in range(5): self.rh.add(config=self.config1, cost=i + 1, time=i + 1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) for i in range(3): self.rh.add(config=self.config2, cost=i + 1, time=i + 1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None) inst_seed_pairs = self.rh.get_runs_for_config(self.config1) # cost used by incumbent for going over all runs in inst_seed_pairs inc_sum_cost = sum_cost(config=self.config1, instance_seed_pairs=inst_seed_pairs, run_history=self.rh) cutoff = intensifier._adapt_cutoff(challenger=self.config2, incumbent=self.config1, run_history=self.rh, inc_sum_cost=inc_sum_cost) # 15*1.2 - 6 self.assertEqual(cutoff, 12) intensifier.cutoff = 5 cutoff = intensifier._adapt_cutoff(challenger=self.config2, incumbent=self.config1, run_history=self.rh, inc_sum_cost=inc_sum_cost) # scenario cutoff self.assertEqual(cutoff, 5)
memory_limit_factor = 2 for entry in trajectory: incumbent_id = entry.incumbent_id train_performance = entry.train_perf if incumbent_id not in incumbent_id_to_model: config = entry.incumbent logger = logging.getLogger('Testing:)') stats = Stats( Scenario({ 'cutoff_time': per_run_time_limit * 2, 'run_obj': 'quality', }) ) stats.start_timing() # To avoid the output "first run crashed"... stats.ta_runs += 1 ta = ExecuteTaFuncWithQueue(backend=automl._automl._backend, autosklearn_seed=seed, resampling_strategy='test', memory_limit=memory_limit_factor * automl_arguments['ml_memory_limit'], disable_file_output=True, logger=logger, stats=stats, all_scoring_functions=True, metric=metric) status, cost, runtime, additional_run_info = ta.start( config=config, instance=None, cutoff=per_run_time_limit*3) if status == StatusType.SUCCESS: