def test_compare_configs_no_joint_set(self): intensifier = Intensifier( tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) for i in range(2): self.rh.add(config=self.config1, cost=2, time=2, status=StatusType.SUCCESS, instance_id=1, seed=i, additional_info=None) for i in range(2, 5): self.rh.add(config=self.config2, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=i, additional_info=None) # The sets for the incumbent are completely disjoint. conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) self.assertIsNone(conf) # The incumbent has still one instance-seed pair left on which the # challenger was not run yet. self.rh.add(config=self.config2, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=1, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) self.assertIsNone(conf)
def test_compare_configs_inc(self): ''' incumbent is better ''' intensifier = Intensifier(tae_runner=None, stats=self.stats, traj_logger=TrajLogger(output_dir=None, stats=self.stats), rng=None, instances=[1]) self.rh.add(config=self.config1, cost=1, time=1, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) self.rh.add(config=self.config2, cost=2, time=2, status=StatusType.SUCCESS, instance_id=1, seed=None, additional_info=None) conf = intensifier._compare_configs(incumbent=self.config1, challenger=self.config2, run_history=self.rh, aggregate_func=average_cost) # challenger worse than inc self.assertEqual(conf, self.config1, "conf: %s" % (conf))