def test_add_inc_run_nondet(self): """ test _add_inc_run() """ def target(x): return (x["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="solution_quality") intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1, 2], deterministic=False, ) instance, seed, cutoff = intensifier._get_next_inc_run( available_insts=intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh)) run_info = RunInfo( config=self.config1, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(len(self.rh.data), 1, self.rh.data) instance, seed, cutoff = intensifier._get_next_inc_run( available_insts=intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh)) run_info = RunInfo( config=self.config1, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(len(self.rh.data), 2, self.rh.data) runs = self.rh.get_runs_for_config(config=self.config1, only_max_observed_budget=True) # exactly one run on each instance self.assertIn(1, [runs[0].instance, runs[1].instance]) self.assertIn(2, [runs[0].instance, runs[1].instance]) instance, seed, cutoff = intensifier._get_next_inc_run( available_insts=intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh)) run_info = RunInfo( config=self.config1, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(len(self.rh.data), 3, self.rh.data) # The number of runs performed should be 3 # No Next iteration call as an incumbent is provided self.assertEqual(intensifier.num_run, 2) self.assertEqual(intensifier.num_chall_run, 0)
def test_add_inc_run_det(self): """ test _add_inc_run() """ def target(x): return (x["a"] + 1) / 1000.0 taf = ExecuteTAFuncDict(use_pynisher=False, ta=target, stats=self.stats, run_obj="solution_quality") taf.runhistory = self.rh intensifier = Intensifier( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=np.random.RandomState(12345), instances=[1], deterministic=True, ) instance, seed, cutoff = intensifier._get_next_inc_run( available_insts=intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh)) run_info = RunInfo( config=self.config1, instance=instance, instance_specific="0", seed=seed, cutoff=cutoff, capped=False, budget=0.0, ) result = eval_challenger(run_info, taf, self.stats, self.rh) intensifier.stage = IntensifierStage.PROCESS_FIRST_CONFIG_RUN inc, perf = intensifier.process_results( run_info=run_info, incumbent=self.config1, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(len(self.rh.data), 1, self.rh.data) # since we assume deterministic=1, # the second call should not add any more runs # given only one instance # So the returned seed/instance is None so that a new # run to be triggered is not launched available_insts = intensifier._get_inc_available_inst( incumbent=self.config1, run_history=self.rh) # Make sure that the list is empty, and hence no new call # of incumbent will be triggered self.assertFalse(available_insts) # The following two tests evaluate to zero because _next_iteration is triggered by _add_inc_run # as it is the first evaluation of this intensifier # After the above incumbent run, the stage is # IntensifierStage.RUN_CHALLENGER. Change it to test next iteration intensifier.stage = IntensifierStage.PROCESS_FIRST_CONFIG_RUN inc, perf = intensifier.process_results( run_info=run_info, incumbent=None, run_history=self.rh, time_bound=np.inf, result=result, ) self.assertEqual(intensifier.num_run, 0) self.assertEqual(intensifier.num_chall_run, 0)