def test_eval_challenger_3(self): """ test eval_challenger for a resumed SMAC run (first run with incumbent) """ def target(x): return x['a'] taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=False, always_race_against=None, run_limit=1) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # adding run for incumbent configuration self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) # intensification - incumbent will be run, but not as RUN_FIRST_CONFIG stage config, _ = intensifier.get_next_challenger(challengers=[self.config2], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=self.config1, run_history=self.rh, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual( len( self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 2)
def test_eval_challenger(self): """ test eval_challenger() - a complete intensification run """ def target(x): return x['a'] taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=False, always_race_against=self.config3, run_limit=1) # run incumbent first if it was not run before config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config1, self.config3], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=None, run_history=self.rh, ) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) # run challenger now that the incumbent has been executed config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config1, self.config3], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # challenger should have a better performance, so incumbent should have changed self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_DEFAULT) self.assertFalse(intensifier.continue_challenger) # run `always_race_against` now since the incumbent has changed config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config1, self.config3], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(inc, self.config1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 1) self.assertEqual(intensifier.n_iters, 1) self.assertIsInstance(intensifier.configs_to_run, collections.Iterator) with self.assertRaises(StopIteration): next(intensifier.configs_to_run)
def test_no_new_intensification_wo_challenger_run(self): """ This test ensures that no new iteration is started if no challenger run was conducted """ def target(x): return 2 * x['a'] + x['b'] taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="quality") taf.runhistory = self.rh intensifier = Intensifier( tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=True, always_race_against=None, run_limit=1, min_chall=1, ) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) config, _ = intensifier.get_next_challenger(challengers=[self.config3], chooser=None) self.assertEqual(config, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=None, run_history=self.rh, ) self.assertEqual(inc, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(intensifier.n_iters, 1) # 1 intensification run complete! # regular intensification begins - run incumbent config, _ = intensifier.get_next_challenger( challengers=None, # since incumbent is run, no configs required chooser=None) self.assertEqual(config, inc) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(intensifier.n_iters, 1) # Check that we don't walk into the next iteration if the challenger is passed again config, _ = intensifier.get_next_challenger(challengers=[self.config3], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(intensifier.n_iters, 1) intensifier._next_iteration() # Add a configuration, then try to execute it afterwards self.assertEqual(intensifier.n_iters, 2) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=0, additional_info=None) intensifier.stage = IntensifierStage.RUN_CHALLENGER config, _ = intensifier.get_next_challenger(challengers=[self.config1], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(intensifier.n_iters, 2) self.assertEqual(intensifier.num_chall_run, 0) # This returns the config evaluating the incumbent again config, _ = intensifier.get_next_challenger(challengers=None, chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # This doesn't return a config because the array of configs is exhausted config, _ = intensifier.get_next_challenger(challengers=None, chooser=None) self.assertIsNone(config) # This finally gives a runable configuration config, _ = intensifier.get_next_challenger(challengers=[self.config2], chooser=None) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(intensifier.n_iters, 3) self.assertEqual(intensifier.num_chall_run, 1)
def test_eval_challenger_2(self): """ test eval_challenger() - a complete intensification run without a `always_race_against` configuration """ def target(x): return 2 * x['a'] + x['b'] taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="quality") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=True, always_race_against=None, run_limit=1) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # intensification iteration #1 # run first config as incumbent if incumbent is None config, _ = intensifier.get_next_challenger(challengers=[self.config3], chooser=None) self.assertEqual(config, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # eval config 2 (=first run) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=None, run_history=self.rh, ) self.assertEqual(inc, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual(intensifier.n_iters, 1) # 1 intensification run complete! # regular intensification begins - run incumbent config, _ = intensifier.get_next_challenger( challengers=None, # since incumbent is run, no configs required chooser=None) self.assertEqual(config, inc) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # no new TA runs as there are no more instances to run self.assertEqual(inc, self.config3) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual(self.stats.ta_runs, 1) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) # run challenger now that the incumbent has been executed config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config1], chooser=None) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(config, self.config2) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # challenger has a better performance, so incumbent has changed self.assertEqual(inc, self.config2) self.assertEqual(self.stats.inc_changed, 2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT ) # since there is no `always_race_against` self.assertFalse(intensifier.continue_challenger) self.assertEqual( intensifier.n_iters, 1) # iteration continues as `min_chall` condition is not met # intensification continues running incumbent again in same iteration... # run incumbent config, _ = intensifier.get_next_challenger( challengers=None, # don't need a new list here as old one is cont'd chooser=None) self.assertEqual(config, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # run challenger config, _ = intensifier.get_next_challenger( challengers=None, # don't need a new list here as old one is cont'd chooser=None) self.assertEqual(config, self.config1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(intensifier.n_iters, 2) # 2 intensification run complete! # no configs should be left at the end with self.assertRaises(StopIteration): next(intensifier.configs_to_run) self.assertEqual( len( self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 1) self.assertEqual( len( self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True)), 1) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 1)
def test_eval_challenger_1(self): """ test eval_challenger() - a complete intensification run with a `always_race_against` configuration """ def target(x): if x['a'] == 100: time.sleep(1) return x['a'] taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier(tae_runner=taf, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], run_obj_time=True, cutoff=2, deterministic=False, always_race_against=self.config3, run_limit=1) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # intensification iteration #1 # run first config as incumbent if incumbent is None config, _ = intensifier.get_next_challenger(challengers=[self.config2], chooser=None) self.assertEqual(config, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # eval config 2 (=first run) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=None, run_history=self.rh, ) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual(intensifier.n_iters, 1) # 1 intensification run complete! # intensification iteration #2 # regular intensification begins - run incumbent first config, _ = intensifier.get_next_challenger( challengers=None, # don't need a new list here as old one is cont'd chooser=None) self.assertEqual(config, inc) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(self.stats.ta_runs, 2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(self.stats.inc_changed, 1) # run challenger now that the incumbent has been executed config, _ = intensifier.get_next_challenger(challengers=[self.config1], chooser=None) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(config, self.config1) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # challenger has a better performance, but not run on all instances yet. so incumbent stays the same self.assertEqual(inc, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertTrue(intensifier.continue_challenger) # run challenger again on the other instance config, _ = intensifier.get_next_challenger( challengers=None, # don't need a new list here as old one is cont'd chooser=None) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(config, self.config1) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # challenger better than incumbent in both instances. so incumbent changed self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_BASIS) self.assertFalse(intensifier.continue_challenger) # run basis configuration (`always_race_against`) config, _ = intensifier.get_next_challenger( challengers=None, # don't need a new list here as old one is cont'd chooser=None) self.assertEqual(config, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_BASIS) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) # the basis configuration (config3) not better than incumbent, so can move on self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 2) self.assertEqual(self.stats.ta_runs, 5) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(list(self.rh.data.values())[4][2], StatusType.CAPPED) self.assertEqual( intensifier.n_iters, 1) # iteration continues as `min_chall` condition is not met self.assertIsInstance(intensifier.configs_to_run, collections.abc.Iterator) # no configs should be left at the end with self.assertRaises(StopIteration): next(intensifier.configs_to_run) # intensification continues running incumbent again in same iteration... config, _ = intensifier.get_next_challenger( challengers=None, # don't need a new list here as old one is cont'd chooser=None) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) inc, _ = intensifier.eval_challenger( challenger=config, incumbent=inc, run_history=self.rh, ) self.assertEqual(inc, self.config1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual( len( self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 3) self.assertEqual( len( self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True)), 2) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 0) # capped