#xx = sm.add_constant(x, prepend=True)
xx = sm.add_constant(x, prepend=False) #for Stata compatibility
#remove nan observation
mask = (xx != -999.0).all(1) #nan code in dta file
mask.shape
y = y[mask]
xx = xx[mask]
group = group[mask]
res_srs = sm.OLS(y, xx).fit()
print res_srs.params
print res_srs.bse
bse_cr = sw.cov_cluster(res_srs, group.astype(int))[1]
print bse_cr
res_stata = np.rec.array(
[('growth', '|', -0.1027121, 0.22917029999999999, -0.45000000000000001,
0.65500000000000003, -0.55483519999999997, 0.34941109999999997),
('emer', '|', -5.4449319999999997, 0.72939690000000001, -7.46, 0.0,
-6.8839379999999997, -4.0059269999999998),
('yr_rnd', '|', -51.075690000000002, 22.83615, -2.2400000000000002, 0.027,
-96.128439999999998, -6.0229350000000004),
('_cons', '|', 740.3981, 13.460760000000001, 55.0, 0.0,
713.84180000000003, 766.95439999999996)],
dtype=[('exogname', '|S6'), ('del', '|S1'), ('params', '<f8'),
('bse', '<f8'), ('tvalues', '<f8'), ('pvalues', '<f8'),
('cilow', '<f8'), ('ciupp', '<f8')])
group = srs['dnum']
#xx = sm.add_constant(x, prepend=True)
xx = sm.add_constant(x, prepend=False) #for Stata compatibility
#remove nan observation
mask = (xx!=-999.0).all(1) #nan code in dta file
mask.shape
y = y[mask]
xx = xx[mask]
group = group[mask]
res_srs = sm.OLS(y, xx).fit()
print res_srs.params
print res_srs.bse
bse_cr = sw.cov_cluster(res_srs, group.astype(int))[1]
print bse_cr
res_stata = np.rec.array(
[ ('growth', '|', -0.1027121, 0.22917029999999999, -0.45000000000000001, 0.65500000000000003, -0.55483519999999997, 0.34941109999999997),
('emer', '|', -5.4449319999999997, 0.72939690000000001, -7.46, 0.0, -6.8839379999999997, -4.0059269999999998),
('yr_rnd', '|', -51.075690000000002, 22.83615, -2.2400000000000002, 0.027, -96.128439999999998, -6.0229350000000004),
('_cons', '|', 740.3981, 13.460760000000001, 55.0, 0.0, 713.84180000000003, 766.95439999999996)],
dtype=[('exogname', '|S6'), ('del', '|S1'), ('params', '<f8'),
('bse', '<f8'), ('tvalues', '<f8'), ('pvalues', '<f8'),
('cilow', '<f8'), ('ciupp', '<f8')])
print bse_cr - res_stata.bse
assert_almost_equal(bse_cr, res_stata.bse, decimal=6)
#res.resid is of transformed model
#np.corrcoef(res.resid.reshape(-1,n_groups, order='F'))
y_pred = np.dot(mod.exog, res.params)
resid = y - y_pred
print np.corrcoef(resid.reshape(-1, n_groups, order='F'))
print resid.std()
err = y_pred - dgp.y_true
print err.std()
#OLS standard errors are too small
mod.res_pooled.params
mod.res_pooled.bse
#heteroscedasticity robust doesn't help
mod.res_pooled.HC1_se
#compare with cluster robust se
import gwstatsmodels.sandbox.panel.sandwich_covariance as sw
print sw.cov_cluster(mod.res_pooled, dgp.groups.astype(int))[1]
#not bad, pretty close to panel estimator
#and with Newey-West Hac
print sw.se_cov(sw.cov_nw_panel(mod.res_pooled, 5, mod.group.groupidx))
#too small, assuming no bugs,
#see Peterson assuming it refers to same kind of model
print dgp.cov
mod2 = ShortPanelGLS(y, dgp.exog, dgp.groups)
res2 = mod2.fit_iterative(2)
print res2.params
print res2.bse
#both implementations produce the same results:
from numpy.testing import assert_almost_equal
assert_almost_equal(res.params, res2.params, decimal=14)
assert_almost_equal(res.bse, res2.bse, decimal=14)
# test White
assert_almost_equal(bse_w, self.HC0_se, 15)
bse_wc = sw.cov_white_simple(self, use_correction=True)[1]
print bse_wc
# test White
assert_almost_equal(bse_wc, self.HC1_se, 15)
groups = np.repeat(np.arange(5), 20)
idx = np.nonzero(np.diff(groups))[0].tolist()
groupidx = zip([0] + idx, idx + [len(groups)])
ngroups = len(groupidx)
print sw.cov_cluster(self, groups)[1]
# two strange looking corner cases BUG?
print sw.cov_cluster(self, np.ones(len(endog), int), use_correction=False)[1]
print sw.cov_crosssection_0(self, np.arange(len(endog)))[1]
# these results are close to simple (no group) white, 50 groups 2 obs each
groups = np.repeat(np.arange(50), 100 // 50)
print sw.cov_cluster(self, groups)[1]
# 2 groups with 50 obs each, what was the interpretation again?
groups = np.repeat(np.arange(2), 100 // 2)
print sw.cov_cluster(self, groups)[1]
"http://www.kellogg.northwestern.edu/faculty/petersen/htm/papers/se/test_data.txt"
"""
test <- read.table(
url(paste("http://www.kellogg.northwestern.edu/",
"faculty/petersen/htm/papers/se/",
#res.resid is of transformed model
#np.corrcoef(res.resid.reshape(-1,n_groups, order='F'))
y_pred = np.dot(mod.exog, res.params)
resid = y - y_pred
print np.corrcoef(resid.reshape(-1,n_groups, order='F'))
print resid.std()
err = y_pred - dgp.y_true
print err.std()
#OLS standard errors are too small
mod.res_pooled.params
mod.res_pooled.bse
#heteroscedasticity robust doesn't help
mod.res_pooled.HC1_se
#compare with cluster robust se
import gwstatsmodels.sandbox.panel.sandwich_covariance as sw
print sw.cov_cluster(mod.res_pooled, dgp.groups.astype(int))[1]
#not bad, pretty close to panel estimator
#and with Newey-West Hac
print sw.se_cov(sw.cov_nw_panel(mod.res_pooled, 5, mod.group.groupidx))
#too small, assuming no bugs,
#see Peterson assuming it refers to same kind of model
print dgp.cov
mod2 = ShortPanelGLS(y, dgp.exog, dgp.groups)
res2 = mod2.fit_iterative(2)
print res2.params
print res2.bse
#both implementations produce the same results:
from numpy.testing import assert_almost_equal
assert_almost_equal(res.params, res2.params, decimal=14)
assert_almost_equal(res.bse, res2.bse, decimal=14)
#test White
assert_almost_equal(bse_w, self.HC0_se, 15)
bse_wc = sw.cov_white_simple(self, use_correction=True)[1]
print bse_wc
#test White
assert_almost_equal(bse_wc, self.HC1_se, 15)
groups = np.repeat(np.arange(5), 20)
idx = np.nonzero(np.diff(groups))[0].tolist()
groupidx = zip([0]+idx, idx+[len(groups)])
ngroups = len(groupidx)
print sw.cov_cluster(self, groups)[1]
#two strange looking corner cases BUG?
print sw.cov_cluster(self, np.ones(len(endog), int), use_correction=False)[1]
print sw.cov_crosssection_0(self, np.arange(len(endog)))[1]
#these results are close to simple (no group) white, 50 groups 2 obs each
groups = np.repeat(np.arange(50), 100//50)
print sw.cov_cluster(self, groups)[1]
#2 groups with 50 obs each, what was the interpretation again?
groups = np.repeat(np.arange(2), 100//2)
print sw.cov_cluster(self, groups)[1]
"http://www.kellogg.northwestern.edu/faculty/petersen/htm/papers/se/test_data.txt"
'''
test <- read.table(
url(paste("http://www.kellogg.northwestern.edu/",
"faculty/petersen/htm/papers/se/",
