def test_add_inc_run_nondet(self):
"""
test _add_inc_run()
"""
def target(x):
return (x["a"] + 1) / 1000.0
taf = ExecuteTAFuncDict(use_pynisher=False,
ta=target,
stats=self.stats,
run_obj="solution_quality")
intensifier = Intensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1, 2],
deterministic=False,
)
instance, seed, cutoff = intensifier._get_next_inc_run(
available_insts=intensifier._get_inc_available_inst(
incumbent=self.config1, run_history=self.rh))
run_info = RunInfo(
config=self.config1,
instance=instance,
instance_specific="0",
seed=seed,
cutoff=cutoff,
capped=False,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
instance, seed, cutoff = intensifier._get_next_inc_run(
available_insts=intensifier._get_inc_available_inst(
incumbent=self.config1, run_history=self.rh))
run_info = RunInfo(
config=self.config1,
instance=instance,
instance_specific="0",
seed=seed,
cutoff=cutoff,
capped=False,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(len(self.rh.data), 2, self.rh.data)
runs = self.rh.get_runs_for_config(config=self.config1,
only_max_observed_budget=True)
# exactly one run on each instance
self.assertIn(1, [runs[0].instance, runs[1].instance])
self.assertIn(2, [runs[0].instance, runs[1].instance])
instance, seed, cutoff = intensifier._get_next_inc_run(
available_insts=intensifier._get_inc_available_inst(
incumbent=self.config1, run_history=self.rh))
run_info = RunInfo(
config=self.config1,
instance=instance,
instance_specific="0",
seed=seed,
cutoff=cutoff,
capped=False,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(len(self.rh.data), 3, self.rh.data)
# The number of runs performed should be 3
# No Next iteration call as an incumbent is provided
self.assertEqual(intensifier.num_run, 2)
self.assertEqual(intensifier.num_chall_run, 0)
def test_add_inc_run_det(self):
"""
test _add_inc_run()
"""
def target(x):
return (x["a"] + 1) / 1000.0
taf = ExecuteTAFuncDict(use_pynisher=False,
ta=target,
stats=self.stats,
run_obj="solution_quality")
taf.runhistory = self.rh
intensifier = Intensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
deterministic=True,
)
instance, seed, cutoff = intensifier._get_next_inc_run(
available_insts=intensifier._get_inc_available_inst(
incumbent=self.config1, run_history=self.rh))
run_info = RunInfo(
config=self.config1,
instance=instance,
instance_specific="0",
seed=seed,
cutoff=cutoff,
capped=False,
budget=0.0,
)
result = eval_challenger(run_info, taf, self.stats, self.rh)
intensifier.stage = IntensifierStage.PROCESS_FIRST_CONFIG_RUN
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=self.config1,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(len(self.rh.data), 1, self.rh.data)
# since we assume deterministic=1,
# the second call should not add any more runs
# given only one instance
# So the returned seed/instance is None so that a new
# run to be triggered is not launched
available_insts = intensifier._get_inc_available_inst(
incumbent=self.config1, run_history=self.rh)
# Make sure that the list is empty, and hence no new call
# of incumbent will be triggered
self.assertFalse(available_insts)
# The following two tests evaluate to zero because _next_iteration is triggered by _add_inc_run
# as it is the first evaluation of this intensifier
# After the above incumbent run, the stage is
# IntensifierStage.RUN_CHALLENGER. Change it to test next iteration
intensifier.stage = IntensifierStage.PROCESS_FIRST_CONFIG_RUN
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=None,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(intensifier.num_run, 0)
self.assertEqual(intensifier.num_chall_run, 0)
