def test_race_challenger_2(self): """ Makes sure that a racing configuration with better performance, that is capped, doesn't substitute the incumbent. """ def target(x): time.sleep(1.5) return (x["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], ) self.rh.add( config=self.config1, cost=0.001, time=0.001, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None, ) intensifier.N = 1 # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=self.config2, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=self.config2, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=True, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config1) self.assertEqual(intensifier.num_run, 1) self.assertEqual(intensifier.num_chall_run, 1)
def test_race_challenger_1(self): """ Makes sure that a racing configuration with better performance, is selected as incumbent No adaptive capping """ def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) intensifier.N = 1 inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=self.config2, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=self.config2, instance=instance, instance_specific="0", cutoff=cutoff, seed=seed, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.num_run, 1) self.assertEqual(intensifier.num_chall_run, 1)
def test_eval_challenger_3(self): """ test eval_challenger for a resumed SMAC run (first run with incumbent) """ def target(x): return x['a'] taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="quality") taf.runhistory = self.rh intensifier = Intensifier(stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=False, always_race_against=None, run_limit=1) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # adding run for incumbent configuration self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) # intensification - incumbent will be run, but not as RUN_FIRST_CONFIG stage intent_, run_info = intensifier.get_next_run( challengers=[self.config2], incumbent=self.config1, run_history=self.rh, chooser=None) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual( len( self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 2)
def test_eval_challenger_2(self): """ test eval_challenger() - a complete intensification run without a `always_race_against` configuration """ def target(x): return 2 * x["a"] + x["b"] taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="quality") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=True, always_race_against=None, run_limit=1, ) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # intensification iteration #1 # run first config as incumbent if incumbent is None intent, run_info = intensifier.get_next_run( challengers=[self.config3], incumbent=None, run_history=self.rh, chooser=None, ) self.assertEqual(run_info.config, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.PROCESS_FIRST_CONFIG_RUN) # eval config 2 (=first run) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=None, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual(intensifier.n_iters, 1) # 1 intensification run complete! # regular intensification begins - run incumbent # Normally a challenger will be given, which in this case is the incumbent # But no more instances are available. So to prevent cicles # where No iteration happens, provide the challengers intent, run_info = intensifier.get_next_run( challengers=[ self.config2, self.config1, ], # since incumbent is run, no configs required incumbent=inc, run_history=self.rh, chooser=None, ) # no new TA runs as there are no more instances to run self.assertEqual(inc, self.config3) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 1, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) # run challenger now that the incumbent has been executed # So this call happen above, to save one iteration self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(run_info.config, self.config2) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) # challenger has a better performance, so incumbent has changed self.assertEqual(inc, self.config2) self.assertEqual(self.stats.inc_changed, 2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT ) # since there is no `always_race_against` self.assertFalse(intensifier.continue_challenger) self.assertEqual( intensifier.n_iters, 1) # iteration continues as `min_chall` condition is not met # intensification continues running incumbent again in same iteration... # run incumbent # Same here, No further instance-seed pairs for incumbent available # so above call gets the new config to run self.assertEqual(run_info.config, self.config2) # There is a transition from: # IntensifierStage.RUN_FIRST_CONFIG-> IntensifierStage.RUN_INCUMBENT # Because after the first run, incumbent is run. # Nevertheless, there is now a transition: # IntensifierStage.RUN_INCUMBENT->IntensifierStage.RUN_CHALLENGER # because in add_inc_run, there are more available instance pairs # FROM: IntensifierStage.RUN_INCUMBENT TO: IntensifierStage.RUN_INCUMBENT WHY: no more to run # if all <instance, seed> have been run, compare challenger performance # self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) # run challenger intent, run_info = intensifier.get_next_run( challengers=None, # don't need a new list here as old one is cont'd incumbent=inc, run_history=self.rh, chooser=None, ) self.assertEqual(run_info.config, self.config1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(intensifier.n_iters, 2) # 2 intensification run complete! # no configs should be left at the end with self.assertRaises(StopIteration): next(intensifier.configs_to_run) self.assertEqual( len( self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 1, ) self.assertEqual( len( self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True)), 1, ) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 1, )
def test_eval_challenger_1(self): """ test eval_challenger() - a complete intensification run with a `always_race_against` configuration """ print(self.rh) def target(x): if x["a"] == 100: time.sleep(1) return x["a"] taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], run_obj_time=True, cutoff=2, deterministic=False, always_race_against=self.config3, run_limit=1, ) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) # intensification iteration #1 # run first config as incumbent if incumbent is None intent, run_info = intensifier.get_next_run( incumbent=None, run_history=self.rh, challengers=[self.config2], chooser=None, ) self.assertEqual(run_info.config, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.PROCESS_FIRST_CONFIG_RUN) # eval config 2 (=first run) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=None, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(self.stats.inc_changed, 1) self.assertEqual(intensifier.n_iters, 1) # 1 intensification run complete! # intensification iteration #2 # regular intensification begins - run incumbent first intent, run_info = intensifier.get_next_run( challengers=None, # don't need a new list here as old one is cont'd incumbent=inc, run_history=self.rh, chooser=None, ) self.assertEqual(run_info.config, inc) self.assertEqual(intensifier.stage, IntensifierStage.PROCESS_INCUMBENT_RUN) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(self.stats.inc_changed, 1) # run challenger now that the incumbent has been executed intent, run_info = intensifier.get_next_run(challengers=[self.config1], incumbent=inc, run_history=self.rh, chooser=None) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(run_info.config, self.config1) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) # challenger has a better performance, but not run on all instances yet. so incumbent stays the same self.assertEqual(inc, self.config2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertTrue(intensifier.continue_challenger) # run challenger again on the other instance intent, run_info = intensifier.get_next_run( challengers=None, # don't need a new list here as old one is cont'd incumbent=inc, run_history=self.rh, chooser=None, ) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual(run_info.config, self.config1) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) # challenger better than incumbent in both instances. so incumbent changed self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_BASIS) self.assertFalse(intensifier.continue_challenger) # run basis configuration (`always_race_against`) intent, run_info = intensifier.get_next_run( challengers=None, # don't need a new list here as old one is cont'd incumbent=inc, run_history=self.rh, chooser=None, ) self.assertEqual(run_info.config, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_BASIS) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) # the basis configuration (config3) not better than incumbent, so can move on self.assertEqual(inc, self.config1) self.assertEqual(self.stats.inc_changed, 2) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT, msg=self.rh.data.items()) self.assertEqual(list(self.rh.data.values())[4][2], StatusType.CAPPED) self.assertEqual( intensifier.n_iters, 1) # iteration continues as `min_chall` condition is not met self.assertIsInstance(intensifier.configs_to_run, collections.abc.Iterator) # no configs should be left at the end with self.assertRaises(StopIteration): next(intensifier.configs_to_run) # intensification continues running incumbent again in same iteration... intent, run_info = intensifier.get_next_run( challengers=None, # don't need a new list here as old one is cont'd incumbent=inc, run_history=self.rh, chooser=None, ) self.assertEqual(intensifier.stage, IntensifierStage.PROCESS_INCUMBENT_RUN) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config1) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) self.assertEqual( len( self.rh.get_runs_for_config(self.config1, only_max_observed_budget=True)), 3, ) self.assertEqual( len( self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True)), 2, ) self.assertEqual( len( self.rh.get_runs_for_config(self.config3, only_max_observed_budget=True)), 0, ) # capped
def test_add_inc_run_nondet(self): """ test _add_inc_run() """ def target(x): return (x["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="solution_quality") intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], deterministic=False, ) instance, seed, cutoff = intensifier._get_next_inc_run( available_insts=intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh)) run_info = RunInfo( config=self.config1, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(len(self.rh.data), 1, self.rh.data) instance, seed, cutoff = intensifier._get_next_inc_run( available_insts=intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh)) run_info = RunInfo( config=self.config1, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(len(self.rh.data), 2, self.rh.data) runs = self.rh.get_runs_for_config(config=self.config1, only_max_observed_budget=True) # exactly one run on each instance self.assertIn(1, [runs[0].instance, runs[1].instance]) self.assertIn(2, [runs[0].instance, runs[1].instance]) instance, seed, cutoff = intensifier._get_next_inc_run( available_insts=intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh)) run_info = RunInfo( config=self.config1, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(len(self.rh.data), 3, self.rh.data) # The number of runs performed should be 3 # No Next iteration call as an incumbent is provided self.assertEqual(intensifier.num_run, 2) self.assertEqual(intensifier.num_chall_run, 0)
def test_add_inc_run_det(self): """ test _add_inc_run() """ def target(x): return (x["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True, ) instance, seed, cutoff = intensifier._get_next_inc_run( available_insts=intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh)) run_info = RunInfo( config=self.config1, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) intensifier.stage = IntensifierStage.PROCESS_FIRST_CONFIG_RUN inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(len(self.rh.data), 1, self.rh.data) # since we assume deterministic=1, # the second call should not add any more runs # given only one instance # So the returned seed/instance is None so that a new # run to be triggered is not launched available_insts = intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh) # Make sure that the list is empty, and hence no new call # of incumbent will be triggered self.assertFalse(available_insts) # The following two tests evaluate to zero because _next_iteration is triggered by _add_inc_run # as it is the first evaluation of this intensifier # After the above incumbent run, the stage is # IntensifierStage.RUN_CHALLENGER. Change it to test next iteration intensifier.stage = IntensifierStage.PROCESS_FIRST_CONFIG_RUN inc, perf = intensifier.process_results( run_info=run_info, incumbent=None, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(intensifier.num_run, 0) self.assertEqual(intensifier.num_chall_run, 0)
def test_race_challenger_large_blocked_seed(self): """ test _race_challenger whether seeds are blocked for challenger runs """ def target(x): return 1 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), run_obj_time=False, deterministic=False, ) for i in range(10): self.rh.add( config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None, ) intensifier.stage = IntensifierStage.RUN_CHALLENGER # tie on first instances and then challenger should always win # and be returned as inc while True: if intensifier.continue_challenger: config = intensifier.current_challenger else: config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=config, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=config, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) # stop when challenger evaluation is over if not intensifier.stage == IntensifierStage.RUN_CHALLENGER: break self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True) self.assertEqual(len(runs), 10) seeds = sorted([r.seed for r in runs]) self.assertEqual(seeds, list(range(10)), seeds) self.assertEqual(intensifier.num_run, 10) self.assertEqual(intensifier.num_chall_run, 10)
def test_race_challenger_3(self): """ test _race_challenger with adaptive capping on a previously capped configuration """ def target(config: Configuration, seed: int, instance: str): if instance == 1: time.sleep(2.1) else: time.sleep(0.6) return (config["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict( use_pynisher=False, ta=target, stats=self.stats, run_obj="runtime", par_factor=1, ) taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), cutoff=2, instances=[1], ) self.rh.add( config=self.config1, cost=0.5, time=0.5, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None, ) # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=config, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=config, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=True, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config1) # further run for incumbent self.rh.add( config=self.config1, cost=2, time=2, status=StatusType.TIMEOUT, instance_id=2, seed=12345, additional_info=None, ) # give config2 a second chance - now it should run on both instances # run on instance 1 config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=config, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=config, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) # run on instance 2 config, _ = intensifier.get_next_challenger(challengers=[self.config3], chooser=None) self.assertEqual(config, self.config2) self.assertTrue(intensifier.continue_challenger) inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=config, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=config, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) # the incumbent should still be config1 because # config2 should get on inst 1 a full timeout # such that c(config1) = 1.25 and c(config2) close to 1.3 self.assertEqual(inc, self.config1) # the capped run should not be counted in runs_perf_config self.assertAlmostEqual(self.rh.num_runs_per_config[2], 2) self.assertFalse(intensifier.continue_challenger) self.assertEqual(intensifier.num_run, 3) self.assertEqual(intensifier.num_chall_run, 3)
def test_no_new_intensification_wo_challenger_run(self): """ This test ensures that no new iteration is started if no challenger run was conducted """ def target(x): return 2 * x["a"] + x["b"] taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="quality") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False, deterministic=True, always_race_against=None, run_limit=1, min_chall=1, ) self.assertEqual(intensifier.n_iters, 0) self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG) intent, run_info = intensifier.get_next_run( challengers=[self.config3], incumbent=None, run_history=self.rh, chooser=None, ) self.assertEqual(run_info.config, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.PROCESS_FIRST_CONFIG_RUN) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=None, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config3) self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT) self.assertEqual(intensifier.n_iters, 1) # 1 intensification run complete! # regular intensification begins - run incumbent # No further instance-seed pairs for incumbent available # Here None challenger is suggested. Code jumps to next iteration # This causes a transition from RUN_INCUMBENT->RUN_CHALLENGER # But then, the next configuration to run is the incumbent # We don't rerun the incumbent (see message): # Challenger was the same as the current incumbent; Skipping challenger # Then, we try to get more challengers, but below all challengers # Provided are config3, the incumbent which means nothing more to run intent, run_info = intensifier.get_next_run( challengers=[self.config3 ], # since incumbent is run, no configs required incumbent=inc, run_history=self.rh, chooser=None, ) self.assertEqual(run_info.config, None) self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER) intensifier._next_iteration() # Add a configuration, then try to execute it afterwards self.assertEqual(intensifier.n_iters, 2) self.rh.add( config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=0, additional_info=None, ) intensifier.stage = IntensifierStage.RUN_CHALLENGER # In the upcoming get next run, the stage is RUN_CHALLENGER # so the intensifier tries to run config1. Nevertheless, # there are no further instances for this configuration available. # In this scenario, the intensifier produces a SKIP intent as an indication # that a new iteration must be initiated, and for code simplicity, # relies on a new call to get_next_run to yield more configurations intent, run_info = intensifier.get_next_run(challengers=[self.config1], incumbent=inc, run_history=self.rh, chooser=None) self.assertEqual(intent, RunInfoIntent.SKIP) # This doesn't return a config because the array of configs is exhausted intensifier.stage = IntensifierStage.RUN_CHALLENGER config, _ = intensifier.get_next_challenger(challengers=None, chooser=None) self.assertIsNone(config) # This finally gives a runable configuration intent, run_info = intensifier.get_next_run(challengers=[self.config2], incumbent=inc, run_history=self.rh, chooser=None) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=inc, run_history=self.rh, time_bound=np.inf, result=result, ) # 4 Iterations due to the proactive runs # of get next challenger self.assertEqual(intensifier.n_iters, 3) self.assertEqual(intensifier.num_chall_run, 1)