def testRandomImputation(self): rs = numpy.random.RandomState(1) for i in range(0, 150, 15): # First random imputation sanity check num_samples = max(1, i * 10) num_feat = max(1, i) num_censored = int(num_samples * 0.1) X = rs.rand(num_samples, num_feat) y = numpy.sin(X[:, 0:1]) cutoff = max(y) * 0.9 y[y > cutoff] = cutoff # We have some cen data cen_X = X[:num_censored, :] cen_y = y[:num_censored] uncen_X = X[num_censored:, :] uncen_y = y[num_censored:] cen_y /= 2 cs = ConfigurationSpace() for i in range(num_feat): cs.add_hyperparameter( UniformFloatHyperparameter(name="a_%d" % i, lower=0, upper=1, default_value=0.5)) types, bounds = get_types(cs, None) print(types) print(bounds) print('#' * 120) print(cen_X) print(uncen_X) print('~' * 120) self.model = RandomForestWithInstances(types=types, bounds=bounds, instance_features=None, seed=1234567980) imputor = rfr_imputator.RFRImputator(rng=rs, cutoff=cutoff, threshold=cutoff * 10, change_threshold=0.01, max_iter=5, model=self.model) imp_y = imputor.impute(censored_X=cen_X, censored_y=cen_y, uncensored_X=uncen_X, uncensored_y=uncen_y) if imp_y is None: continue for idx in range(cen_y.shape[0]): self.assertGreater(imp_y[idx], cen_y[idx]) self.assertTrue(numpy.isfinite(imp_y).all())
def testRealImputation(self): rs = numpy.random.RandomState(1) imputor = rfr_imputator.RFRImputator(rng=rs, cutoff=self.scen.cutoff, threshold=self.scen.cutoff*10, change_threshold=0.01, max_iter=10, model=self.model) r2e = runhistory2epm.RunHistory2EPM4LogCost( scenario=self.scen, num_params=3, success_states=[StatusType.SUCCESS, ], impute_censored_data=True, impute_state=[StatusType.TIMEOUT], imputor=imputor, rng=rs) print("%s" % str(r2e.transform(self.rh)[0]))
def testRealImputation(self): # Without instance features rs = numpy.random.RandomState(1) cs, rh = self.get_runhistory(num_success=5, num_timeout=1, num_capped=2) scen = self.get_scenario() model = self.get_model(cs) imputor = rfr_imputator.RFRImputator(rng=rs, cutoff=scen.cutoff, threshold=scen.cutoff * 10, change_threshold=0.01, max_iter=10, model=model) r2e = runhistory2epm.RunHistory2EPM4LogCost( scenario=scen, num_params=3, success_states=[ StatusType.SUCCESS, ], impute_censored_data=True, impute_state=[StatusType.TIMEOUT], imputor=imputor, rng=rs, ) self.assertEqual(r2e.transform(rh)[1].shape, (8, 1)) self.assertEqual(r2e.transform(rh)[1].shape, (8, 1)) # Now with instance features instance_features = { run_key.instance_id: numpy.random.rand(10) for run_key in rh.data } scen = self.get_scenario(instance_features) model = self.get_model(cs, instance_features) with unittest.mock.patch.object(model, attribute='train', wraps=model.train) as train_wrapper: imputor = rfr_imputator.RFRImputator(rng=rs, cutoff=scen.cutoff, threshold=scen.cutoff * 10, change_threshold=0.01, max_iter=10, model=model) r2e = runhistory2epm.RunHistory2EPM4LogCost( scenario=scen, num_params=3, success_states=[ StatusType.SUCCESS, ], impute_censored_data=True, impute_state=[StatusType.TIMEOUT], imputor=imputor, rng=rs, ) X, y = r2e.transform(rh) self.assertEqual(X.shape, (8, 13)) self.assertEqual(y.shape, (8, 1)) num_calls = len(train_wrapper.call_args_list) self.assertGreater(num_calls, 1) self.assertEqual(train_wrapper.call_args_list[0][0][0].shape, (5, 13)) self.assertEqual(train_wrapper.call_args_list[1][0][0].shape, (8, 13)) X, y = r2e.transform(rh) self.assertEqual(X.shape, (8, 13)) self.assertEqual(y.shape, (8, 1)) self.assertGreater(len(train_wrapper.call_args_list), num_calls + 1) self.assertEqual( train_wrapper.call_args_list[num_calls][0][0].shape, (5, 13)) self.assertEqual( train_wrapper.call_args_list[num_calls + 1][0][0].shape, (8, 13))