def test_full_update(self):
rh = RunHistory()
cs = get_config_space()
config1 = Configuration(cs,
values={'a': 1, 'b': 2})
config2 = Configuration(cs,
values={'a': 1, 'b': 3})
rh.add(config=config1, cost=10, time=20,
status=StatusType.SUCCESS, instance_id=1,
seed=1)
rh.add(config=config2, cost=10, time=20,
status=StatusType.SUCCESS, instance_id=1,
seed=1)
rh.add(config=config2, cost=20, time=20,
status=StatusType.SUCCESS, instance_id=2,
seed=2)
cost_config2 = rh.get_cost(config2)
rh.compute_all_costs()
updated_cost_config2 = rh.get_cost(config2)
self.assertTrue(cost_config2 == updated_cost_config2)
rh.compute_all_costs(instances=[2])
updated_cost_config2 = rh.get_cost(config2)
self.assertTrue(cost_config2 != updated_cost_config2)
self.assertTrue(updated_cost_config2 == 20)
def merge_foreign_data(
scenario: Scenario, runhistory: RunHistory,
in_scenario_list: typing.List[Scenario],
in_runhistory_list: typing.List[RunHistory]
) -> typing.Tuple[Scenario, RunHistory]:
"""Extend <scenario> and <runhistory> with runhistory data from another
<in_scenario> assuming the same pcs, feature space, but different instances
Parameters
----------
scenario: Scenario
original scenario -- feature dictionary will be extended
runhistory: RunHistory
original runhistory -- will be extended by further data points
in_scenario_list: typing.List[Scenario]
input scenario
in_runhistory_list: typing.List[RunHistory]
list of runhistories wrt <in_scenario>
Returns
-------
scenario: Scenario
runhistory: Runhistory
"""
# add further instance features
for in_scenario in in_scenario_list:
if scenario.n_features != in_scenario.n_features:
raise ValueError(
"Feature Space has to be the same for both scenarios (%d vs %d)."
% (scenario.n_features, in_scenario.n_features))
if scenario.cs != in_scenario.cs: # type: ignore[attr-defined] # noqa F821
raise ValueError("PCS of both scenarios have to be identical.")
if scenario.cutoff != in_scenario.cutoff: # type: ignore[attr-defined] # noqa F821
raise ValueError("Cutoffs of both scenarios have to be identical.")
scenario.feature_dict.update(in_scenario.feature_dict)
# extend runhistory
for rh in in_runhistory_list:
runhistory.update(rh, origin=DataOrigin.EXTERNAL_DIFFERENT_INSTANCES)
for date in runhistory.data:
if scenario.feature_dict.get(date.instance_id) is None:
raise ValueError(
"Instance feature for \"%s\" was not found in scenario data." %
(date.instance_id))
runhistory.compute_all_costs(instances=scenario.train_insts)
return scenario, runhistory
def test_full_update(self):
rh = RunHistory(overwrite_existing_runs=True)
cs = get_config_space()
config1 = Configuration(cs, values={"a": 1, "b": 2})
config2 = Configuration(cs, values={"a": 1, "b": 3})
rh.add(
config=config1,
cost=[10, 40],
time=20,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
)
rh.add(
config=config1,
cost=[0, 100],
time=20,
status=StatusType.SUCCESS,
instance_id=2,
seed=2,
)
rh.add(
config=config2,
cost=[10, 40],
time=20,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
)
rh.add(
config=config2,
cost=[20, 80],
time=20,
status=StatusType.SUCCESS,
instance_id=2,
seed=2,
)
cost_config2 = rh.get_cost(config2)
rh.compute_all_costs()
updated_cost_config2 = rh.get_cost(config2)
self.assertEqual(cost_config2, updated_cost_config2)
rh.compute_all_costs(instances=[2])
updated_cost_config2 = rh.get_cost(config2)
self.assertAlmostEqual(updated_cost_config2, 0.833, places=3)
def merge_foreign_data(scenario: Scenario, runhistory: RunHistory,
in_scenario_list: typing.List[Scenario],
in_runhistory_list: typing.List[RunHistory]):
'''
extend <scenario> and <runhistory> with runhistory data from another <in_scenario>
assuming the same pcs, feature space, but different instances
Arguments
---------
scenario: Scenario
original scenario -- feature dictionary will be extended
runhistory: RunHistory
original runhistory -- will be extended by further data points
in_scenario_list: typing.List[Scenario]
input scenario
in_runhistory_list: typing.List[RunHistory]
list of runhistories wrt <in_scenario>
Returns
-------
scenario, runhistory
'''
# add further instance features
for in_scenario in in_scenario_list:
if scenario.n_features != in_scenario.n_features:
raise ValueError(
"Feature Space has to be the same for both scenarios (%d vs %d)."
% (scenario.n_features, in_scenario.n_features))
if scenario.cs != in_scenario.cs:
raise ValueError("PCS of both scenarios have to be identical.")
if scenario.cutoff != in_scenario.cutoff:
raise ValueError("Cutoffs of both scenarios have to be identical.")
scenario.feature_dict.update(in_scenario.feature_dict)
# extend runhistory
for rh in in_runhistory_list:
runhistory.update(rh, external_data=True)
for date in runhistory.data:
if scenario.feature_dict.get(date.instance_id) is None:
raise ValueError(
"Instance feature for \"%s\" was not found in scenario data." %
(date.instance_id))
runhistory.compute_all_costs(instances=scenario.train_insts)
return scenario, runhistory
def test_full_update(self):
rh = RunHistory()
cs = get_config_space()
config1 = Configuration(cs, values={"a": 1, "b": 2})
config2 = Configuration(cs, values={"a": 1, "b": 3})
rh.add(
config=config1,
cost=10,
time=20,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
)
rh.add(
config=config2,
cost=10,
time=20,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
)
rh.add(
config=config2,
cost=20,
time=20,
status=StatusType.SUCCESS,
instance_id=2,
seed=2,
)
cost_config2 = rh.get_cost(config2)
rh.compute_all_costs()
updated_cost_config2 = rh.get_cost(config2)
self.assertEqual(cost_config2, updated_cost_config2)
rh.compute_all_costs(instances=[2])
updated_cost_config2 = rh.get_cost(config2)
self.assertNotEqual(cost_config2, updated_cost_config2)
self.assertEqual(updated_cost_config2, 20)
rh = RunHistory()
cs = get_config_space()
config1 = Configuration(cs, values={"a": 1, "b": 2})
config2 = Configuration(cs, values={"a": 1, "b": 3})
rh.add(
config=config1,
cost=[10],
time=20,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
)
rh.add(
config=config2,
cost=[10],
time=20,
status=StatusType.SUCCESS,
instance_id=1,
seed=1,
)
rh.add(
config=config2,
cost=[20],
time=20,
status=StatusType.SUCCESS,
instance_id=2,
seed=2,
)
cost_config2 = rh.get_cost(config2)
rh.compute_all_costs()
updated_cost_config2 = rh.get_cost(config2)
self.assertEqual(cost_config2, updated_cost_config2)
rh.compute_all_costs(instances=[2])
updated_cost_config2 = rh.get_cost(config2)
self.assertNotEqual(cost_config2, updated_cost_config2)
self.assertEqual(updated_cost_config2, 20)
