def test_race_challenger_2(self): """ Makes sure that a racing configuration with better performance, that is capped, doesn't substitute the incumbent. """ def target(x): time.sleep(1.5) return (x["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="runtime") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], ) self.rh.add( config=self.config1, cost=0.001, time=0.001, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None, ) intensifier.N = 1 # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=self.config2, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=self.config2, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=True, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config1) self.assertEqual(intensifier.num_run, 1) self.assertEqual(intensifier.num_chall_run, 1)
def test_race_challenger_1(self): """ Makes sure that a racing configuration with better performance, is selected as incumbent No adaptive capping """ def target(x): return (x['a'] + 1) / 1000. taf = ExecuteTAFuncDict(ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier(stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], run_obj_time=False) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) intensifier.N = 1 inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=self.config2, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=self.config2, instance=instance, instance_specific="0", cutoff=cutoff, seed=seed, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config2) self.assertEqual(intensifier.num_run, 1) self.assertEqual(intensifier.num_chall_run, 1)
def test_race_challenger_large_blocked_seed(self): """ test _race_challenger whether seeds are blocked for challenger runs """ def target(x): return 1 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats) taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=list(range(10)), run_obj_time=False, deterministic=False, ) for i in range(10): self.rh.add( config=self.config1, cost=i + 1, time=1, status=StatusType.SUCCESS, instance_id=i, seed=i, additional_info=None, ) intensifier.stage = IntensifierStage.RUN_CHALLENGER # tie on first instances and then challenger should always win # and be returned as inc while True: if intensifier.continue_challenger: config = intensifier.current_challenger else: config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=config, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=config, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) # stop when challenger evaluation is over if not intensifier.stage == IntensifierStage.RUN_CHALLENGER: break self.assertEqual(inc, self.config2) self.assertEqual(self.rh.get_cost(self.config2), 1) # get data for config2 to check that the correct run was performed runs = self.rh.get_runs_for_config(self.config2, only_max_observed_budget=True) self.assertEqual(len(runs), 10) seeds = sorted([r.seed for r in runs]) self.assertEqual(seeds, list(range(10)), seeds) self.assertEqual(intensifier.num_run, 10) self.assertEqual(intensifier.num_chall_run, 10)
def test_race_challenger_3(self): """ test _race_challenger with adaptive capping on a previously capped configuration """ def target(config: Configuration, seed: int, instance: str): if instance == 1: time.sleep(2.1) else: time.sleep(0.6) return (config["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict( use_pynisher=False, ta=target, stats=self.stats, run_obj="runtime", par_factor=1, ) taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), cutoff=2, instances=[1], ) self.rh.add( config=self.config1, cost=0.5, time=0.5, status=StatusType.SUCCESS, instance_id=1, seed=12345, additional_info=None, ) # config2 should have a timeout (due to adaptive capping) # and config1 should still be the incumbent config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=config, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=config, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=True, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(inc, self.config1) # further run for incumbent self.rh.add( config=self.config1, cost=2, time=2, status=StatusType.TIMEOUT, instance_id=2, seed=12345, additional_info=None, ) # give config2 a second chance - now it should run on both instances # run on instance 1 config, _ = intensifier.get_next_challenger( challengers=[self.config2, self.config3], chooser=None) inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=config, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=config, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) # run on instance 2 config, _ = intensifier.get_next_challenger(challengers=[self.config3], chooser=None) self.assertEqual(config, self.config2) self.assertTrue(intensifier.continue_challenger) inc, instance, seed, cutoff = intensifier._get_next_racer( challenger=config, incumbent=self.config1, run_history=self.rh) run_info = RunInfo( config=config, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) # the incumbent should still be config1 because # config2 should get on inst 1 a full timeout # such that c(config1) = 1.25 and c(config2) close to 1.3 self.assertEqual(inc, self.config1) # the capped run should not be counted in runs_perf_config self.assertAlmostEqual(self.rh.num_runs_per_config[2], 2) self.assertFalse(intensifier.continue_challenger) self.assertEqual(intensifier.num_run, 3) self.assertEqual(intensifier.num_chall_run, 3)