def test_compare_configs_no_joint_set(self):
intensifier = AbstractRacer(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=None, instances=[1])
for i in range(2):
self.rh.add(config=self.config1, cost=2, time=2,
status=StatusType.SUCCESS, instance_id=1,
seed=i, additional_info=None)
for i in range(2, 5):
self.rh.add(config=self.config2, cost=1, time=1,
status=StatusType.SUCCESS, instance_id=1,
seed=i, additional_info=None)
# The sets for the incumbent are completely disjoint.
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
self.assertIsNone(conf)
# The incumbent has still one instance-seed pair left on which the
# challenger was not run yet.
self.rh.add(config=self.config2, cost=1, time=1,
status=StatusType.SUCCESS, instance_id=1,
seed=1, additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
self.assertIsNone(conf)
def test_compare_configs_inc(self):
"""
incumbent is better
"""
intensifier = AbstractRacer(tae_runner=None,
stats=self.stats,
traj_logger=TrajLogger(output_dir=None,
stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config2,
cost=2,
time=2,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger worse than inc
self.assertEqual(conf, self.config1, "conf: %s" % (conf))
def test_compare_configs_unknow(self):
"""
challenger is better but has less runs;
-> no decision (None)
"""
intensifier = AbstractRacer(
stats=self.stats,
traj_logger=TrajLogger(output_dir=None, stats=self.stats),
rng=None,
instances=[1])
self.rh.add(config=self.config1, cost=1, time=1,
status=StatusType.SUCCESS, instance_id=1,
seed=None,
additional_info=None)
self.rh.add(config=self.config1, cost=1, time=2,
status=StatusType.SUCCESS, instance_id=2,
seed=None,
additional_info=None)
self.rh.add(config=self.config1, cost=1, time=1,
status=StatusType.SUCCESS, instance_id=2,
seed=None,
additional_info=None)
conf = intensifier._compare_configs(incumbent=self.config1,
challenger=self.config2,
run_history=self.rh)
# challenger worse than inc
self.assertIsNone(conf, "conf: %s" % (conf))