def test_no_new_intensification_wo_challenger_run(self):
"""
This test ensures that no new iteration is started if no challenger run was conducted
"""
def target(x):
return 2 * x["a"] + x["b"]
taf = ExecuteTAFuncDict(use_pynisher=False,
ta=target,
stats=self.stats,
run_obj="quality")
taf.runhistory = self.rh
intensifier = Intensifier(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
run_obj_time=False,
deterministic=True,
always_race_against=None,
run_limit=1,
min_chall=1,
)
self.assertEqual(intensifier.n_iters, 0)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG)
intent, run_info = intensifier.get_next_run(
challengers=[self.config3],
incumbent=None,
run_history=self.rh,
chooser=None,
)
self.assertEqual(run_info.config, self.config3)
self.assertEqual(intensifier.stage,
IntensifierStage.PROCESS_FIRST_CONFIG_RUN)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=None,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
self.assertEqual(inc, self.config3)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
self.assertEqual(intensifier.n_iters,
1) # 1 intensification run complete!
# regular intensification begins - run incumbent
# No further instance-seed pairs for incumbent available
# Here None challenger is suggested. Code jumps to next iteration
# This causes a transition from RUN_INCUMBENT->RUN_CHALLENGER
# But then, the next configuration to run is the incumbent
# We don't rerun the incumbent (see message):
# Challenger was the same as the current incumbent; Skipping challenger
# Then, we try to get more challengers, but below all challengers
# Provided are config3, the incumbent which means nothing more to run
intent, run_info = intensifier.get_next_run(
challengers=[self.config3
], # since incumbent is run, no configs required
incumbent=inc,
run_history=self.rh,
chooser=None,
)
self.assertEqual(run_info.config, None)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
intensifier._next_iteration()
# Add a configuration, then try to execute it afterwards
self.assertEqual(intensifier.n_iters, 2)
self.rh.add(
config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=0,
additional_info=None,
)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
# In the upcoming get next run, the stage is RUN_CHALLENGER
# so the intensifier tries to run config1. Nevertheless,
# there are no further instances for this configuration available.
# In this scenario, the intensifier produces a SKIP intent as an indication
# that a new iteration must be initiated, and for code simplicity,
# relies on a new call to get_next_run to yield more configurations
intent, run_info = intensifier.get_next_run(challengers=[self.config1],
incumbent=inc,
run_history=self.rh,
chooser=None)
self.assertEqual(intent, RunInfoIntent.SKIP)
# This doesn't return a config because the array of configs is exhausted
intensifier.stage = IntensifierStage.RUN_CHALLENGER
config, _ = intensifier.get_next_challenger(challengers=None,
chooser=None)
self.assertIsNone(config)
# This finally gives a runable configuration
intent, run_info = intensifier.get_next_run(challengers=[self.config2],
incumbent=inc,
run_history=self.rh,
chooser=None)
result = eval_challenger(run_info, taf, self.stats, self.rh)
inc, perf = intensifier.process_results(
run_info=run_info,
incumbent=inc,
run_history=self.rh,
time_bound=np.inf,
result=result,
)
# 4 Iterations due to the proactive runs
# of get next challenger
self.assertEqual(intensifier.n_iters, 3)
self.assertEqual(intensifier.num_chall_run, 1)
def test_no_new_intensification_wo_challenger_run(self):
"""
This test ensures that no new iteration is started if no challenger run was conducted
"""
def target(x):
return 2 * x['a'] + x['b']
taf = ExecuteTAFuncDict(ta=target, stats=self.stats, run_obj="quality")
taf.runhistory = self.rh
intensifier = Intensifier(
tae_runner=taf,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=np.random.RandomState(12345),
instances=[1],
run_obj_time=False,
deterministic=True,
always_race_against=None,
run_limit=1,
min_chall=1,
)
self.assertEqual(intensifier.n_iters, 0)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG)
config, _ = intensifier.get_next_challenger(challengers=[self.config3],
chooser=None)
self.assertEqual(config, self.config3)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_FIRST_CONFIG)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=None,
run_history=self.rh,
)
self.assertEqual(inc, self.config3)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
self.assertEqual(intensifier.n_iters,
1) # 1 intensification run complete!
# regular intensification begins - run incumbent
config, _ = intensifier.get_next_challenger(
challengers=None, # since incumbent is run, no configs required
chooser=None)
self.assertEqual(config, inc)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_INCUMBENT)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertEqual(intensifier.n_iters, 1)
# Check that we don't walk into the next iteration if the challenger is passed again
config, _ = intensifier.get_next_challenger(challengers=[self.config3],
chooser=None)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
self.assertEqual(intensifier.stage, IntensifierStage.RUN_CHALLENGER)
self.assertEqual(intensifier.n_iters, 1)
intensifier._next_iteration()
# Add a configuration, then try to execute it afterwards
self.assertEqual(intensifier.n_iters, 2)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=0,
additional_info=None)
intensifier.stage = IntensifierStage.RUN_CHALLENGER
config, _ = intensifier.get_next_challenger(challengers=[self.config1],
chooser=None)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
self.assertEqual(intensifier.n_iters, 2)
self.assertEqual(intensifier.num_chall_run, 0)
# This returns the config evaluating the incumbent again
config, _ = intensifier.get_next_challenger(challengers=None,
chooser=None)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
# This doesn't return a config because the array of configs is exhausted
config, _ = intensifier.get_next_challenger(challengers=None,
chooser=None)
self.assertIsNone(config)
# This finally gives a runable configuration
config, _ = intensifier.get_next_challenger(challengers=[self.config2],
chooser=None)
inc, _ = intensifier.eval_challenger(
challenger=config,
incumbent=inc,
run_history=self.rh,
)
self.assertEqual(intensifier.n_iters, 3)
self.assertEqual(intensifier.num_chall_run, 1)
