def test_budget_selection(self):
'''
adding some rundata and check budget selection
'''
rh2epm = runhistory2epm.RunHistory2EPM4Cost(num_params=2,
success_states=[StatusType.SUCCESS,
StatusType.CRASHED,
StatusType.MEMOUT],
impute_censored_data=False,
scenario=self.scen)
self.rh.add(config=self.config1, cost=1, time=1,
status=StatusType.SUCCESS, instance_id=1,
seed=None, budget=1,
additional_info=None)
self.rh.add(config=self.config1, cost=2, time=2,
status=StatusType.SUCCESS, instance_id=1,
seed=None, budget=2,
additional_info=None)
X, y = rh2epm.transform(self.rh, budget_subset=[1])
self.assertTrue(np.allclose(X, np.array([[0.005, 0.995]]), atol=0.001))
self.assertTrue(np.allclose(y, np.array([[1]])))
X, y = rh2epm.transform(self.rh, budget_subset=[2])
self.assertTrue(np.allclose(X, np.array([[0.005, 0.995]]), atol=0.001))
self.assertTrue(np.allclose(y, np.array([[2]])))
def test_cost_quality(self):
'''
adding some rundata to RunHistory2EPM4LogCost
'''
self.scen = Scenario({"cutoff_time": 20, 'cs': self.cs, 'run_obj': 'quality',
'output_dir': ''})
rh2epm = runhistory2epm.RunHistory2EPM4Cost(num_params=2,
success_states=[StatusType.SUCCESS,
StatusType.CRASHED,
StatusType.MEMOUT],
impute_censored_data=False,
scenario=self.scen)
self.rh.add(config=self.config1, cost=1, time=10,
status=StatusType.SUCCESS, instance_id=23,
seed=None,
additional_info=None)
X, y = rh2epm.transform(self.rh)
self.assertTrue(np.allclose(X, np.array([[0.005, 0.995]]), atol=0.001))
# should use the cost field and not runtime
self.assertTrue(np.allclose(y, np.array([[1.]])))
# rh2epm should use cost and not time field later
self.rh.add(config=self.config3, cost=200, time=20,
status=StatusType.TIMEOUT, instance_id=1,
seed=45,
additional_info={"start_time": 20})
X, y = rh2epm.transform(self.rh)
self.assertTrue(
np.allclose(X, np.array([[0.005, 0.995], [0.995, 0.995]]), atol=0.001))
# log_10(20 * 10)
self.assertTrue(np.allclose(y, np.array([[1.], [200.]]), atol=0.001))
def test_cost_with_imputation(self):
'''
adding some rundata to RunHistory2EPM4Cost and impute censored data
'''
self.imputor = RFRImputator(
rng=np.random.RandomState(seed=12345),
cutoff=self.scen.cutoff,
threshold=self.scen.cutoff * self.scen.par_factor,
model=RandomForestWithInstances(
configspace=self.cs,
types=self.types,
bounds=self.bounds,
instance_features=None,
seed=12345,
n_points_per_tree=90,
ratio_features=1.0,
)
)
rh2epm = runhistory2epm.RunHistory2EPM4Cost(num_params=2,
scenario=self.scen,
impute_censored_data=True,
success_states=[StatusType.SUCCESS, ],
impute_state=[StatusType.TIMEOUT],
imputor=self.imputor)
self.rh.add(config=self.config1, cost=1, time=1,
status=StatusType.SUCCESS, instance_id=23,
seed=None,
additional_info=None)
X, y = rh2epm.transform(self.rh)
self.assertTrue(np.allclose(X, np.array([[0.005, 0.995]]), atol=0.001))
self.assertTrue(np.allclose(y, np.array([[1.]])))
# rh2epm should use time and not cost field later
self.rh.add(config=self.config3, cost=200, time=20,
status=StatusType.TIMEOUT, instance_id=1,
seed=45,
additional_info={"start_time": 20})
X, y = rh2epm.transform(self.rh)
self.assertTrue(
np.allclose(X, np.array([[0.005, 0.995], [0.995, 0.995]]), atol=0.001))
self.assertTrue(np.allclose(y, np.array([[1.], [200.]]), atol=0.001))
self.rh.add(config=self.config2, cost=100, time=10,
status=StatusType.TIMEOUT, instance_id=1,
seed=12354,
additional_info={"start_time": 10})
X, y = rh2epm.transform(self.rh)
np.testing.assert_array_almost_equal(X, np.array([[0.005, 0.995],
[0.995, 0.005],
[0.995, 0.995]]),
decimal=3)
np.testing.assert_array_almost_equal(y, np.array([[1.], [11.], [200.]]),
decimal=1)
def test_cost_without_imputation(self):
'''
adding some rundata to RunHistory2EPM4Cost without imputation
'''
rh2epm = runhistory2epm.RunHistory2EPM4Cost(num_params=2,
success_states=[
StatusType.SUCCESS,
StatusType.CRASHED,
StatusType.MEMOUT
],
impute_censored_data=False,
scenario=self.scen)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=23,
seed=None,
additional_info=None)
X, y = rh2epm.transform(self.rh)
self.assertTrue(np.allclose(X, np.array([[0.005, 0.995]]), atol=0.001))
self.assertTrue(np.allclose(y, np.array([[1.]])))
# rh2epm should use cost and not time
self.rh.add(config=self.config3,
cost=200,
time=20,
status=StatusType.TIMEOUT,
instance_id=1,
seed=45,
additional_info={"start_time": 20})
X, y = rh2epm.transform(self.rh)
np.testing.assert_allclose(X,
np.array([[0.005, 0.995], [0.995, 0.995]]),
atol=0.001)
# log_10(20 * 10)
np.testing.assert_allclose(y, np.array([[1.], [200.]]), atol=0.001)
self.rh.add(config=self.config2,
cost=100,
time=10,
status=StatusType.TIMEOUT,
instance_id=1,
seed=12354,
additional_info={"start_time": 10})
X, y = rh2epm.transform(self.rh)
# last entry gets skipped since imputation is disabled
self.assertTrue(
np.allclose(X,
np.array([[0.005, 0.995], [0.995, 0.995]]),
atol=0.001))
self.assertTrue(np.allclose(y, np.array([[1.], [200.]]), atol=0.001))
def test_get_X_y(self):
'''
add some data to RH and check returned values in X,y format
'''
self.scen = Scenario({'cutoff_time': 20, 'cs': self.cs,
'run_obj': 'runtime',
'instances': [['1'], ['2']],
'features': {
'1': [1, 1],
'2': [2, 2]
},
'output_dir': ''})
rh2epm = runhistory2epm.RunHistory2EPM4Cost(num_params=2,
success_states=[StatusType.SUCCESS, ],
impute_state=[StatusType.CAPPED, ],
impute_censored_data=False,
scenario=self.scen)
self.rh.add(config=self.config1, cost=1, time=10,
status=StatusType.SUCCESS, instance_id='1',
seed=None,
additional_info=None)
self.rh.add(config=self.config1, cost=2, time=10,
status=StatusType.SUCCESS, instance_id='2',
seed=None,
additional_info=None)
self.rh.add(config=self.config2, cost=1, time=10,
status=StatusType.TIMEOUT, instance_id='1',
seed=None,
additional_info=None)
self.rh.add(config=self.config2, cost=0.1, time=10,
status=StatusType.CAPPED, instance_id='2',
seed=None,
additional_info=None)
X, y, c = rh2epm.get_X_y(self.rh)
X_sol = np.array([[0, 100, 1, 1],
[0, 100, 2, 2],
[100, 0, 1, 1],
[100, 0, 2, 2]])
self.assertTrue(np.all(X == X_sol))
y_sol = np.array([1, 2, 1, 0.1])
self.assertTrue(np.all(y == y_sol))
c_sol = np.array([False, False, True, True])
self.assertTrue(np.all(c == c_sol))
def test_cost_with_imputation(self):
'''
adding some rundata to RunHistory2EPM4Cost and impute censored data
'''
self.imputor = RFRImputator(
rs=np.random.RandomState(seed=12345),
cutoff=self.scen.cutoff,
threshold=self.scen.cutoff * self.scen.par_factor,
model=RandomForestWithInstances(types=np.array([0, 0],
dtype=np.uint),
instance_features=None,
seed=12345))
rh2epm = runhistory2epm.RunHistory2EPM4Cost(
num_params=2,
scenario=self.scen,
impute_censored_data=True,
impute_state=[StatusType.TIMEOUT],
imputor=self.imputor)
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=23,
seed=None,
additional_info=None)
X, y = rh2epm.transform(self.rh)
self.assertTrue(np.allclose(X, np.array([[0.005, 0.995]]), atol=0.001))
self.assertTrue(np.allclose(y, np.array([[1.]])))
# rh2epm should use time and not cost field later
self.rh.add(config=self.config3,
cost=200,
time=20,
status=StatusType.TIMEOUT,
instance_id=1,
seed=45,
additional_info={"start_time": 20})
X, y = rh2epm.transform(self.rh)
self.assertTrue(
np.allclose(X,
np.array([[0.005, 0.995], [0.995, 0.995]]),
atol=0.001))
# log_10(20 * 10)
self.assertTrue(np.allclose(y, np.array([[1.], [200.]]), atol=0.001))
self.rh.add(config=self.config2,
cost=100,
time=10,
status=StatusType.TIMEOUT,
instance_id=1,
seed=12354,
additional_info={"start_time": 10})
X, y = rh2epm.transform(self.rh)
self.assertTrue(
np.allclose(X,
np.array([[0.005, 0.995], [0.995, 0.005],
[0.995, 0.995]]),
atol=0.001))
self.assertTrue(
np.allclose(y, np.array([[1.], [18.970], [200.]]), atol=0.001))
def test_run_selection(self):
'''
adding some rundata and check budget selection
'''
self.rh.add(config=self.config1,
cost=1,
time=1,
status=StatusType.SUCCESS,
instance_id=1,
seed=None,
budget=1,
additional_info=None)
self.rh.add(config=self.config2,
cost=2,
time=2,
status=StatusType.CRASHED,
instance_id=1,
seed=None,
budget=2,
additional_info=None)
self.rh.add(config=self.config3,
cost=3,
time=3,
status=StatusType.MEMOUT,
instance_id=1,
seed=None,
budget=2,
additional_info=None)
self.rh.add(config=self.config4,
cost=4,
time=4,
status=StatusType.DONOTADVANCE,
instance_id=1,
seed=None,
budget=3,
additional_info=None)
self.rh.add(config=self.config5,
cost=20,
time=20,
status=StatusType.TIMEOUT,
instance_id=1,
seed=None,
budget=4,
additional_info=None)
for s, v in [
(StatusType.SUCCESS, 1),
(StatusType.CRASHED, 2),
(StatusType.MEMOUT, 3),
(StatusType.DONOTADVANCE, 4),
]:
rh2epm = runhistory2epm.RunHistory2EPM4Cost(
num_params=2,
success_states=[
s,
],
impute_censored_data=False,
scenario=self.scen)
X, y = rh2epm.transform(self.rh, budget_subset=None)
self.assertSetEqual(set(y.flatten()), {v, 20})
for s, v in [
(StatusType.SUCCESS, [
1,
]),
(StatusType.CRASHED, []),
(StatusType.MEMOUT, []),
(StatusType.DONOTADVANCE, []),
(StatusType.TIMEOUT, []),
]:
rh2epm = runhistory2epm.RunHistory2EPM4Cost(
num_params=2,
success_states=[
s,
],
impute_censored_data=False,
scenario=self.scen)
X, y = rh2epm.transform(self.rh, budget_subset=[1])
self.assertSetEqual(set(y.flatten()), set(v))
# Test defaults set in SMAC facade
rh2epm = runhistory2epm.RunHistory2EPM4Cost(
num_params=2,
success_states=[
StatusType.SUCCESS,
StatusType.CRASHED,
StatusType.MEMOUT,
StatusType.DONOTADVANCE,
],
consider_for_higher_budgets_state=[
StatusType.TIMEOUT,
StatusType.CRASHED,
StatusType.MEMOUT,
StatusType.DONOTADVANCE,
],
impute_censored_data=False,
scenario=self.scen)
X, y = rh2epm.transform(self.rh, budget_subset=[1])
self.assertSetEqual(set(y.flatten()), {
1,
})
self.assertTrue(len(y) == 1)
X, y = rh2epm.transform(self.rh, budget_subset=[2])
self.assertSetEqual(set(y.flatten()), {2, 3})
self.assertTrue(len(y) == 2)
X, y = rh2epm.transform(self.rh, budget_subset=[3])
self.assertSetEqual(set(y.flatten()), {2, 3, 4})
self.assertTrue(len(y) == 3)
X, y = rh2epm.transform(self.rh, budget_subset=[4])
self.assertSetEqual(set(y.flatten()), {2, 3, 4, 20})
self.assertTrue(len(y) == 4)
X, y = rh2epm.transform(self.rh, budget_subset=[5])
self.assertSetEqual(set(y.flatten()), {2, 3, 4, 20})
self.assertTrue(len(y) == 4)
self.assertRaises(ValueError,
rh2epm.transform,
self.rh,
budget_subset=[4, 5])
