def ensemble_run_history(request):
run_history = RunHistory()
run_history._add(
RunKey(config_id=3,
instance_id='{"task_id": "breast_cancer"}',
seed=1,
budget=3.0),
RunValue(cost=0.11347517730496459,
time=0.21858787536621094,
status=None,
starttime=time.time(),
endtime=time.time(),
additional_info={
'duration': 0.20323538780212402,
'num_run': 3,
'configuration_origin': 'Random Search'
}),
status=None,
origin=None,
)
run_history._add(
RunKey(config_id=6,
instance_id='{"task_id": "breast_cancer"}',
seed=1,
budget=6.0),
RunValue(cost=2 * 0.11347517730496459,
time=2 * 0.21858787536621094,
status=None,
starttime=time.time(),
endtime=time.time(),
additional_info={
'duration': 0.20323538780212402,
'num_run': 6,
'configuration_origin': 'Random Search'
}),
status=None,
origin=None,
)
return run_history
class SingleBestTest(unittest.TestCase):
def setUp(self):
self.run_history = RunHistory()
self.run_history._add(
RunKey(config_id=3,
instance_id='{"task_id": "breast_cancer"}',
seed=1,
budget=3.0),
RunValue(cost=0.11347517730496459,
time=0.21858787536621094,
status=None,
starttime=time.time(),
endtime=time.time(),
additional_info={
'duration': 0.20323538780212402,
'num_run': 3,
'configuration_origin': 'Random Search'
}),
status=None,
origin=None,
)
self.run_history._add(
RunKey(config_id=6,
instance_id='{"task_id": "breast_cancer"}',
seed=1,
budget=6.0),
RunValue(cost=2 * 0.11347517730496459,
time=2 * 0.21858787536621094,
status=None,
starttime=time.time(),
endtime=time.time(),
additional_info={
'duration': 0.20323538780212402,
'num_run': 6,
'configuration_origin': 'Random Search'
}),
status=None,
origin=None,
)
def test_get_identifiers_from_run_history_accuracy(self):
ensemble = SingleBest(
metric=accuracy,
random_state=1,
run_history=self.run_history,
)
# Just one model
self.assertEqual(len(ensemble.identifiers_), 1)
# That model must be the best
seed, num_run, budget = ensemble.identifiers_[0]
self.assertEqual(num_run, 3)
self.assertEqual(seed, 1)
self.assertEqual(budget, 3.0)
def test_get_identifiers_from_run_history_log_loss(self):
ensemble = SingleBest(
metric=log_loss,
random_state=1,
run_history=self.run_history,
)
# Just one model
self.assertEqual(len(ensemble.identifiers_), 1)
# That model must be the best
seed, num_run, budget = ensemble.identifiers_[0]
self.assertEqual(num_run, 3)
self.assertEqual(seed, 1)
self.assertEqual(budget, 3.0)
