def test_full_update(self): rh = RunHistory() cs = get_config_space() config1 = Configuration(cs, values={'a': 1, 'b': 2}) config2 = Configuration(cs, values={'a': 1, 'b': 3}) rh.add(config=config1, cost=10, time=20, status=StatusType.SUCCESS, instance_id=1, seed=1) rh.add(config=config2, cost=10, time=20, status=StatusType.SUCCESS, instance_id=1, seed=1) rh.add(config=config2, cost=20, time=20, status=StatusType.SUCCESS, instance_id=2, seed=2) cost_config2 = rh.get_cost(config2) rh.compute_all_costs() updated_cost_config2 = rh.get_cost(config2) self.assertTrue(cost_config2 == updated_cost_config2) rh.compute_all_costs(instances=[2]) updated_cost_config2 = rh.get_cost(config2) self.assertTrue(cost_config2 != updated_cost_config2) self.assertTrue(updated_cost_config2 == 20)
def merge_foreign_data( scenario: Scenario, runhistory: RunHistory, in_scenario_list: typing.List[Scenario], in_runhistory_list: typing.List[RunHistory] ) -> typing.Tuple[Scenario, RunHistory]: """Extend <scenario> and <runhistory> with runhistory data from another <in_scenario> assuming the same pcs, feature space, but different instances Parameters ---------- scenario: Scenario original scenario -- feature dictionary will be extended runhistory: RunHistory original runhistory -- will be extended by further data points in_scenario_list: typing.List[Scenario] input scenario in_runhistory_list: typing.List[RunHistory] list of runhistories wrt <in_scenario> Returns ------- scenario: Scenario runhistory: Runhistory """ # add further instance features for in_scenario in in_scenario_list: if scenario.n_features != in_scenario.n_features: raise ValueError( "Feature Space has to be the same for both scenarios (%d vs %d)." % (scenario.n_features, in_scenario.n_features)) if scenario.cs != in_scenario.cs: # type: ignore[attr-defined] # noqa F821 raise ValueError("PCS of both scenarios have to be identical.") if scenario.cutoff != in_scenario.cutoff: # type: ignore[attr-defined] # noqa F821 raise ValueError("Cutoffs of both scenarios have to be identical.") scenario.feature_dict.update(in_scenario.feature_dict) # extend runhistory for rh in in_runhistory_list: runhistory.update(rh, origin=DataOrigin.EXTERNAL_DIFFERENT_INSTANCES) for date in runhistory.data: if scenario.feature_dict.get(date.instance_id) is None: raise ValueError( "Instance feature for \"%s\" was not found in scenario data." % (date.instance_id)) runhistory.compute_all_costs(instances=scenario.train_insts) return scenario, runhistory
def test_full_update(self): rh = RunHistory(overwrite_existing_runs=True) cs = get_config_space() config1 = Configuration(cs, values={"a": 1, "b": 2}) config2 = Configuration(cs, values={"a": 1, "b": 3}) rh.add( config=config1, cost=[10, 40], time=20, status=StatusType.SUCCESS, instance_id=1, seed=1, ) rh.add( config=config1, cost=[0, 100], time=20, status=StatusType.SUCCESS, instance_id=2, seed=2, ) rh.add( config=config2, cost=[10, 40], time=20, status=StatusType.SUCCESS, instance_id=1, seed=1, ) rh.add( config=config2, cost=[20, 80], time=20, status=StatusType.SUCCESS, instance_id=2, seed=2, ) cost_config2 = rh.get_cost(config2) rh.compute_all_costs() updated_cost_config2 = rh.get_cost(config2) self.assertEqual(cost_config2, updated_cost_config2) rh.compute_all_costs(instances=[2]) updated_cost_config2 = rh.get_cost(config2) self.assertAlmostEqual(updated_cost_config2, 0.833, places=3)
def merge_foreign_data(scenario: Scenario, runhistory: RunHistory, in_scenario_list: typing.List[Scenario], in_runhistory_list: typing.List[RunHistory]): ''' extend <scenario> and <runhistory> with runhistory data from another <in_scenario> assuming the same pcs, feature space, but different instances Arguments --------- scenario: Scenario original scenario -- feature dictionary will be extended runhistory: RunHistory original runhistory -- will be extended by further data points in_scenario_list: typing.List[Scenario] input scenario in_runhistory_list: typing.List[RunHistory] list of runhistories wrt <in_scenario> Returns ------- scenario, runhistory ''' # add further instance features for in_scenario in in_scenario_list: if scenario.n_features != in_scenario.n_features: raise ValueError( "Feature Space has to be the same for both scenarios (%d vs %d)." % (scenario.n_features, in_scenario.n_features)) if scenario.cs != in_scenario.cs: raise ValueError("PCS of both scenarios have to be identical.") if scenario.cutoff != in_scenario.cutoff: raise ValueError("Cutoffs of both scenarios have to be identical.") scenario.feature_dict.update(in_scenario.feature_dict) # extend runhistory for rh in in_runhistory_list: runhistory.update(rh, external_data=True) for date in runhistory.data: if scenario.feature_dict.get(date.instance_id) is None: raise ValueError( "Instance feature for \"%s\" was not found in scenario data." % (date.instance_id)) runhistory.compute_all_costs(instances=scenario.train_insts) return scenario, runhistory
def test_full_update(self): rh = RunHistory() cs = get_config_space() config1 = Configuration(cs, values={"a": 1, "b": 2}) config2 = Configuration(cs, values={"a": 1, "b": 3}) rh.add( config=config1, cost=10, time=20, status=StatusType.SUCCESS, instance_id=1, seed=1, ) rh.add( config=config2, cost=10, time=20, status=StatusType.SUCCESS, instance_id=1, seed=1, ) rh.add( config=config2, cost=20, time=20, status=StatusType.SUCCESS, instance_id=2, seed=2, ) cost_config2 = rh.get_cost(config2) rh.compute_all_costs() updated_cost_config2 = rh.get_cost(config2) self.assertEqual(cost_config2, updated_cost_config2) rh.compute_all_costs(instances=[2]) updated_cost_config2 = rh.get_cost(config2) self.assertNotEqual(cost_config2, updated_cost_config2) self.assertEqual(updated_cost_config2, 20) rh = RunHistory() cs = get_config_space() config1 = Configuration(cs, values={"a": 1, "b": 2}) config2 = Configuration(cs, values={"a": 1, "b": 3}) rh.add( config=config1, cost=[10], time=20, status=StatusType.SUCCESS, instance_id=1, seed=1, ) rh.add( config=config2, cost=[10], time=20, status=StatusType.SUCCESS, instance_id=1, seed=1, ) rh.add( config=config2, cost=[20], time=20, status=StatusType.SUCCESS, instance_id=2, seed=2, ) cost_config2 = rh.get_cost(config2) rh.compute_all_costs() updated_cost_config2 = rh.get_cost(config2) self.assertEqual(cost_config2, updated_cost_config2) rh.compute_all_costs(instances=[2]) updated_cost_config2 = rh.get_cost(config2) self.assertNotEqual(cost_config2, updated_cost_config2) self.assertEqual(updated_cost_config2, 20)