def test_compare_configs_no_joint_set(self): intensifier = AbstractRacer( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) for i in range(2): self.rh.add(config=self.config1, cost=2, time=2, status=StatusType.SUCCESS, instance_id=1, seed=i, additional_info=None) for i in range(2, 5): self.rh.add(config=self.config2, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=i, additional_info=None) # The sets for the incumbent are completely disjoint. conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh) self.assertIsNone(conf) # The incumbent has still one instance-seed pair left on which the # challenger was not run yet. self.rh.add(config=self.config2, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=1, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh) self.assertIsNone(conf)
def test_compare_configs_inc(self): """ incumbent is better """ intensifier = AbstractRacer(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config2, cost=2, time=2, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh) # challenger worse than inc self.assertEqual(conf, self.config1, "conf: %s" % (conf))
def test_compare_configs_unknow(self): """ challenger is better but has less runs; -> no decision (None) """ intensifier = AbstractRacer( stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config1, cost=1, time=2, status=StatusType.SUCCESS, instance_id=2, seed=None, additional_info=None) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=2, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh) # challenger worse than inc self.assertIsNone(conf, "conf: %s" % (conf))