def test_var(self,):
# test that the estimator calcualtes var correctly
config = self._get_base_config()
config['honest'] = True
config['max_depth'] = 0
config['inference'] = True
config['n_estimators'] = 1000
config['subforest_size'] = 2
config['max_samples'] = .5
config['n_jobs'] = 1
n_features = 2
# test api
n = 100
random_state = 123
X, y, truth = self._get_regression_data(n, n_features, random_state)
forest = RegressionForest(**config).fit(X, y)
alpha = .1
mean, var = forest.predict_and_var(X)
lb = scipy.stats.norm.ppf(alpha / 2, loc=mean[:, 0], scale=np.sqrt(var[:, 0, 0])).reshape(-1, 1)
ub = scipy.stats.norm.ppf(1 - alpha / 2, loc=mean[:, 0], scale=np.sqrt(var[:, 0, 0])).reshape(-1, 1)
np.testing.assert_allclose(var, forest.predict_var(X))
lbtest, ubtest = forest.predict_interval(X, alpha=alpha)
np.testing.assert_allclose(lb, lbtest)
np.testing.assert_allclose(ub, ubtest)
meantest, lbtest, ubtest = forest.predict(X, interval=True, alpha=alpha)
np.testing.assert_allclose(mean, meantest)
np.testing.assert_allclose(lb, lbtest)
np.testing.assert_allclose(ub, ubtest)
np.testing.assert_allclose(np.sqrt(var[:, 0, 0]), forest.prediction_stderr(X)[:, 0])
# test accuracy
for n in [10, 100, 1000, 10000]:
random_state = 123
X, y, truth = self._get_regression_data(n, n_features, random_state)
forest = RegressionForest(**config).fit(X, y)
our_mean, our_var = forest.predict_and_var(X[:1])
true_mean, true_var = np.mean(y), np.var(y) / y.shape[0]
np.testing.assert_allclose(our_mean, true_mean, atol=0.05)
np.testing.assert_allclose(our_var, true_var, atol=0.05, rtol=.1)
for n, our_thr, true_thr in [(1000, .5, .25), (10000, .05, .05)]:
random_state = 123
config['max_depth'] = 1
X, y, truth = self._get_step_regression_data(n, n_features, random_state)
forest = RegressionForest(**config).fit(X, y)
posX = X[X[:, 0] > our_thr]
negX = X[X[:, 0] < -our_thr]
our_pos_mean, our_pos_var = forest.predict_and_var(posX)
our_neg_mean, our_neg_var = forest.predict_and_var(negX)
pos = X[:, 0] > true_thr
true_pos_mean, true_pos_var = np.mean(y[pos]), np.var(y[pos]) / y[pos].shape[0]
neg = X[:, 0] < -true_thr
true_neg_mean, true_neg_var = np.mean(y[neg]), np.var(y[neg]) / y[neg].shape[0]
np.testing.assert_allclose(our_pos_mean, true_pos_mean, atol=0.07)
np.testing.assert_allclose(our_pos_var, true_pos_var, atol=0.0, rtol=.25)
np.testing.assert_allclose(our_neg_mean, true_neg_mean, atol=0.07)
np.testing.assert_allclose(our_neg_var, true_neg_var, atol=0.0, rtol=.25)
return
def monte_carlo():
n = 5000
d = 5
x_grid = np.linspace(-1, 1, 1000)
X_test = np.hstack(
[x_grid.reshape(-1, 1),
np.random.normal(size=(1000, d - 1))])
coverage = []
exp_dict = {'point': [], 'low': [], 'up': []}
for it in range(100):
print(it)
X = np.random.normal(0, 1, size=(n, d))
y = X[:, 0] + np.random.normal(size=(n, ))
est = RegressionForest(n_estimators=1000, verbose=1)
est.fit(X, y)
point = est.predict(X_test)
low, up = est.predict_interval(X_test, alpha=0.05)
coverage.append((low <= x_grid) & (x_grid <= up))
exp_dict['point'].append(point)
exp_dict['low'].append(low)
exp_dict['up'].append(up)
if not os.path.exists('figures'):
os.makedirs('figures')
if not os.path.exists(os.path.join("figures", 'honestforest')):
os.makedirs(os.path.join("figures", 'honestforest'))
plt.figure()
plt.plot(x_grid, np.mean(coverage, axis=0))
plt.savefig('figures/honestforest/coverage.png')
plt.figure()
plt.plot(x_grid,
np.sqrt(np.mean((np.array(exp_dict['point']) - x_grid)**2,
axis=0)),
label='RMSE')
plt.savefig('figures/honestforest/rmse.png')
plt.figure()
plt.plot(x_grid,
np.mean(np.array(exp_dict['up']) - np.array(exp_dict['low']),
axis=0),
label='length')
plt.savefig('figures/honestforest/length.png')