def gradient(self, par):
beta = np.reshape(np.array(par), (self.k, 1))
q = 2 * self.y - 1
qxb = q * spdot(self.x, beta)
lamb = q * norm.pdf(qxb) / norm.cdf(qxb)
gradient = spdot(lamb.T, self.x)[0]
return gradient
def par_est(self):
start = np.dot(la.inv(spdot(self.x.T, self.x)),
spdot(self.x.T, self.y))
flogl = lambda par: -self.ll(par)
if self.optim == 'newton':
fgrad = lambda par: self.gradient(par)
fhess = lambda par: self.hessian(par)
par_hat = newton(flogl, start, fgrad, fhess, self.maxiter)
warn = par_hat[2]
else:
fgrad = lambda par: -self.gradient(par)
if self.optim == 'bfgs':
par_hat = op.fmin_bfgs(flogl,
start,
fgrad,
full_output=1,
disp=0)
warn = par_hat[6]
if self.optim == 'ncg':
fhess = lambda par: -self.hessian(par)
par_hat = op.fmin_ncg(flogl,
start,
fgrad,
fhess=fhess,
full_output=1,
disp=0)
warn = par_hat[5]
if warn > 0:
warn = True
else:
warn = False
return par_hat, warn
def par_est(self):
start = np.dot(la.inv(spdot(self.x.T, self.x)),
spdot(self.x.T, self.y))
flogl = lambda par: -self.ll(par)
if self.optim == 'newton':
fgrad = lambda par: self.gradient(par)
fhess = lambda par: self.hessian(par)
par_hat = newton(flogl, start, fgrad, fhess, self.maxiter)
warn = par_hat[2]
else:
fgrad = lambda par: -self.gradient(par)
if self.optim == 'bfgs':
par_hat = op.fmin_bfgs(
flogl, start, fgrad, full_output=1, disp=0)
warn = par_hat[6]
if self.optim == 'ncg':
fhess = lambda par: -self.hessian(par)
par_hat = op.fmin_ncg(
flogl, start, fgrad, fhess=fhess, full_output=1, disp=0)
warn = par_hat[5]
if warn > 0:
warn = True
else:
warn = False
return par_hat, warn
def gradient(self, par):
beta = np.reshape(np.array(par), (self.k, 1))
q = 2 * self.y - 1
qxb = q * spdot(self.x, beta)
lamb = q * norm.pdf(qxb) / norm.cdf(qxb)
gradient = spdot(lamb.T, self.x)[0]
return gradient
def __init__(self, y, x, yend, q=None, h=None,
robust=None, gwk=None, sig2n_k=False):
if issubclass(type(q), np.ndarray) and issubclass(type(h), np.ndarray):
raise Exception, "Please do not provide 'q' and 'h' together"
if q is None and h is None:
raise Exception, "Please provide either 'q' or 'h'"
self.y = y
self.n = y.shape[0]
self.x = x
self.kstar = yend.shape[1]
# including exogenous and endogenous variables
z = sphstack(self.x, yend)
if type(h).__name__ not in ['ndarray', 'csr_matrix']:
# including exogenous variables and instrument
h = sphstack(self.x, q)
self.z = z
self.h = h
self.q = q
self.yend = yend
# k = number of exogenous variables and endogenous variables
self.k = z.shape[1]
hth = spdot(h.T, h)
hthi = la.inv(hth)
zth = spdot(z.T, h)
hty = spdot(h.T, y)
factor_1 = np.dot(zth, hthi)
factor_2 = np.dot(factor_1, zth.T)
# this one needs to be in cache to be used in AK
varb = la.inv(factor_2)
factor_3 = np.dot(varb, factor_1)
betas = np.dot(factor_3, hty)
self.betas = betas
self.varb = varb
self.zthhthi = factor_1
# predicted values
self.predy = spdot(z, betas)
# residuals
u = y - self.predy
self.u = u
# attributes used in property
self.hth = hth # Required for condition index
self.hthi = hthi # Used in error models
self.htz = zth.T
if robust:
self.vm = ROBUST.robust_vm(reg=self, gwk=gwk, sig2n_k=sig2n_k)
self._cache = {}
if sig2n_k:
self.sig2 = self.sig2n_k
else:
self.sig2 = self.sig2n
def __init__(self, y, x, yend, q=None, h=None,
robust=None, gwk=None, sig2n_k=False):
if issubclass(type(q), np.ndarray) and issubclass(type(h), np.ndarray):
raise Exception, "Please do not provide 'q' and 'h' together"
if q == None and h == None:
raise Exception, "Please provide either 'q' or 'h'"
self.y = y
self.n = y.shape[0]
self.x = x
self.kstar = yend.shape[1]
# including exogenous and endogenous variables
z = sphstack(self.x, yend)
if type(h).__name__ not in ['ndarray', 'csr_matrix']:
# including exogenous variables and instrument
h = sphstack(self.x, q)
self.z = z
self.h = h
self.q = q
self.yend = yend
# k = number of exogenous variables and endogenous variables
self.k = z.shape[1]
hth = spdot(h.T, h)
hthi = la.inv(hth)
zth = spdot(z.T, h)
hty = spdot(h.T, y)
factor_1 = np.dot(zth, hthi)
factor_2 = np.dot(factor_1, zth.T)
# this one needs to be in cache to be used in AK
varb = la.inv(factor_2)
factor_3 = np.dot(varb, factor_1)
betas = np.dot(factor_3, hty)
self.betas = betas
self.varb = varb
self.zthhthi = factor_1
# predicted values
self.predy = spdot(z, betas)
# residuals
u = y - self.predy
self.u = u
# attributes used in property
self.hth = hth # Required for condition index
self.hthi = hthi # Used in error models
self.htz = zth.T
if robust:
self.vm = ROBUST.robust_vm(reg=self, gwk=gwk, sig2n_k=sig2n_k)
self._cache = {}
if sig2n_k:
self.sig2 = self.sig2n_k
else:
self.sig2 = self.sig2n
def hessian(self, par):
beta = np.reshape(np.array(par), (self.k, 1))
q = 2 * self.y - 1
xb = spdot(self.x, beta)
qxb = q * xb
lamb = q * norm.pdf(qxb) / norm.cdf(qxb)
hessian = spdot(self.x.T, spbroadcast(self.x, -lamb * (lamb + xb)))
return hessian
def hessian(self, par):
beta = np.reshape(np.array(par), (self.k, 1))
q = 2 * self.y - 1
xb = spdot(self.x, beta)
qxb = q * xb
lamb = q * norm.pdf(qxb) / norm.cdf(qxb)
hessian = spdot(self.x.T, spbroadcast(self.x,-lamb * (lamb + xb)))
return hessian
def slopes_vm(self):
if 'slopes_vm' not in self._cache:
x = self.xmean
b = self.betas
dfdb = np.eye(self.k) - spdot(b.T, x) * spdot(b, x.T)
slopes_vm = (self.scale ** 2) * \
np.dot(np.dot(dfdb, self.vm), dfdb.T)
self._cache['slopes_vm'] = slopes_vm[1:, 1:]
return self._cache['slopes_vm']
def xb(self):
try:
return self._cache['xb']
except AttributeError:
self._cache = {}
self._cache['xb'] = spdot(self.x, self.betas)
except KeyError:
self._cache['xb'] = spdot(self.x, self.betas)
return self._cache['xb']
def slopes_vm(self):
if 'slopes_vm' not in self._cache:
x = self.xmean
b = self.betas
dfdb = np.eye(self.k) - spdot(b.T, x) * spdot(b, x.T)
slopes_vm = (self.scale ** 2) * \
np.dot(np.dot(dfdb, self.vm), dfdb.T)
self._cache['slopes_vm'] = slopes_vm[1:, 1:]
return self._cache['slopes_vm']
def _get_spat_diag_props(self,y, x, w, yend, q, w_lags, lag_q):
self._cache = {}
yend, q = set_endog(y, x, w, yend, q, w_lags, lag_q)
x = USER.check_constant(x)
x = REGI.regimeX_setup(x, self.regimes, [True] * x.shape[1], self.regimes_set)
self.z = sphstack(x,REGI.regimeX_setup(yend, self.regimes, [True] * (yend.shape[1]-1)+[False], self.regimes_set))
self.h = sphstack(x,REGI.regimeX_setup(q, self.regimes, [True] * q.shape[1], self.regimes_set))
hthi = np.linalg.inv(spdot(self.h.T,self.h))
zth = spdot(self.z.T,self.h)
self.varb = np.linalg.inv(spdot(spdot(zth,hthi),zth.T))
def _get_spat_diag_props(self, results, regi_ids, x, yend, q):
self._cache = {}
x = USER.check_constant(x)
x = REGI.regimeX_setup(
x, self.regimes, [True] * x.shape[1], self.regimes_set)
self.z = sphstack(x, REGI.regimeX_setup(
yend, self.regimes, [True] * yend.shape[1], self.regimes_set))
self.h = sphstack(
x, REGI.regimeX_setup(q, self.regimes, [True] * q.shape[1], self.regimes_set))
hthi = np.linalg.inv(spdot(self.h.T, self.h))
zth = spdot(self.z.T, self.h)
self.varb = np.linalg.inv(spdot(spdot(zth, hthi), zth.T))
def _get_spat_diag_props(self, y, x, w, yend, q, w_lags, lag_q):
self._cache = {}
yend, q = set_endog(y, x, w, yend, q, w_lags, lag_q)
x = USER.check_constant(x)
x = REGI.regimeX_setup(
x, self.regimes, [True] * x.shape[1], self.regimes_set)
self.z = sphstack(x, REGI.regimeX_setup(
yend, self.regimes, [True] * (yend.shape[1] - 1) + [False], self.regimes_set))
self.h = sphstack(
x, REGI.regimeX_setup(q, self.regimes, [True] * q.shape[1], self.regimes_set))
hthi = np.linalg.inv(spdot(self.h.T, self.h))
zth = spdot(self.z.T, self.h)
self.varb = np.linalg.inv(spdot(spdot(zth, hthi), zth.T))
def AB(self):
"""
Computes A and B matrices as in Cliff-Ord 1981, p. 203
"""
if 'AB' not in self._cache:
U = (self.w.sparse + self.w.sparse.T) / 2.
z = spdot(U, self.reg.x, array_out=False)
c1 = spdot(self.reg.x.T, z, array_out=False)
c2 = spdot(z.T, z, array_out=False)
G = self.reg.xtxi
A = spdot(G, c1)
B = spdot(G, c2)
self._cache['AB'] = [A, B]
return self._cache['AB']
def AB(self):
"""
Computes A and B matrices as in Cliff-Ord 1981, p. 203
"""
if 'AB' not in self._cache:
U = (self.w.sparse + self.w.sparse.T) / 2.
z = spdot(U, self.reg.x, array_out=False)
c1 = spdot(self.reg.x.T, z, array_out=False)
c2 = spdot(z.T, z, array_out=False)
G = self.reg.xtxi
A = spdot(G, c1)
B = spdot(G, c2)
self._cache['AB'] = [A, B]
return self._cache['AB']
def _get_spat_diag_props(self, results, regi_ids, x, yend, q):
self._cache = {}
x = USER.check_constant(x)
x = REGI.regimeX_setup(x, self.regimes, [True] * x.shape[1],
self.regimes_set)
self.z = sphstack(
x,
REGI.regimeX_setup(yend, self.regimes, [True] * yend.shape[1],
self.regimes_set))
self.h = sphstack(
x,
REGI.regimeX_setup(q, self.regimes, [True] * q.shape[1],
self.regimes_set))
hthi = np.linalg.inv(spdot(self.h.T, self.h))
zth = spdot(self.z.T, self.h)
self.varb = np.linalg.inv(spdot(spdot(zth, hthi), zth.T))
def __init__(self, y, x, w):
#1a. OLS --> \tilde{betas}
ols = OLS.BaseOLS(y=y, x=x)
self.n, self.k = ols.x.shape
self.x = ols.x
self.y = ols.y
#1b. GMM --> \tilde{\lambda1}
moments = _momentsGM_Error(w, ols.u)
lambda1 = optim_moments(moments)
#2a. OLS -->\hat{betas}
xs = get_spFilter(w, lambda1, self.x)
ys = get_spFilter(w, lambda1, self.y)
ols2 = OLS.BaseOLS(y=ys, x=xs)
#Output
self.predy = spdot(self.x, ols2.betas)
self.u = y - self.predy
self.betas = np.vstack((ols2.betas, np.array([[lambda1]])))
self.sig2 = ols2.sig2n
self.e_filtered = self.u - lambda1*w*self.u
self.vm = self.sig2 * ols2.xtxi
se_betas = np.sqrt(self.vm.diagonal())
self._cache = {}
def _get_spat_diag_props(self, x, sig2n_k):
self.k = self.kr
self._cache = {}
x = np.hstack((np.ones((x.shape[0], 1)), x))
self.x = REGI.regimeX_setup(x, self.regimes, [True] * x.shape[1], self.regimes_set)
self.xtx = spdot(self.x.T, self.x)
self.xtxi = np.linalg.inv(self.xtx)
def __init__(self, y, x, yend, q, w):
# 1a. TSLS --> \tilde{betas}
tsls = TSLS.BaseTSLS(y=y, x=x, yend=yend, q=q)
self.n, self.k = tsls.z.shape
self.x = tsls.x
self.y = tsls.y
self.yend, self.z = tsls.yend, tsls.z
# 1b. GMM --> \tilde{\lambda1}
moments = _momentsGM_Error(w, tsls.u)
lambda1 = optim_moments(moments)
# 2a. 2SLS -->\hat{betas}
xs = get_spFilter(w, lambda1, self.x)
ys = get_spFilter(w, lambda1, self.y)
yend_s = get_spFilter(w, lambda1, self.yend)
tsls2 = TSLS.BaseTSLS(ys, xs, yend_s, h=tsls.h)
# Output
self.betas = np.vstack((tsls2.betas, np.array([[lambda1]])))
self.predy = spdot(tsls.z, tsls2.betas)
self.u = y - self.predy
self.sig2 = float(np.dot(tsls2.u.T, tsls2.u)) / self.n
self.e_filtered = self.u - lambda1 * w * self.u
self.vm = self.sig2 * tsls2.varb
self._cache = {}
def __init__(self, y, x, w):
# 1a. OLS --> \tilde{betas}
ols = OLS.BaseOLS(y=y, x=x)
self.n, self.k = ols.x.shape
self.x = ols.x
self.y = ols.y
# 1b. GMM --> \tilde{\lambda1}
moments = _momentsGM_Error(w, ols.u)
lambda1 = optim_moments(moments)
# 2a. OLS -->\hat{betas}
xs = get_spFilter(w, lambda1, self.x)
ys = get_spFilter(w, lambda1, self.y)
ols2 = OLS.BaseOLS(y=ys, x=xs)
# Output
self.predy = spdot(self.x, ols2.betas)
self.u = y - self.predy
self.betas = np.vstack((ols2.betas, np.array([[lambda1]])))
self.sig2 = ols2.sig2n
self.e_filtered = self.u - lambda1 * w * self.u
self.vm = self.sig2 * ols2.xtxi
se_betas = np.sqrt(self.vm.diagonal())
self._cache = {}
def __init__(self, y, x, yend, q, w):
#1a. TSLS --> \tilde{betas}
tsls = TSLS.BaseTSLS(y=y, x=x, yend=yend, q=q)
self.n, self.k = tsls.z.shape
self.x = tsls.x
self.y = tsls.y
self.yend, self.z = tsls.yend, tsls.z
#1b. GMM --> \tilde{\lambda1}
moments = _momentsGM_Error(w, tsls.u)
lambda1 = optim_moments(moments)
#2a. 2SLS -->\hat{betas}
xs = get_spFilter(w, lambda1, self.x)
ys = get_spFilter(w, lambda1, self.y)
yend_s = get_spFilter(w, lambda1, self.yend)
tsls2 = TSLS.BaseTSLS(ys, xs, yend_s, h=tsls.h)
#Output
self.betas = np.vstack((tsls2.betas, np.array([[lambda1]])))
self.predy = spdot(tsls.z, tsls2.betas)
self.u = y - self.predy
self.sig2 = float(np.dot(tsls2.u.T,tsls2.u)) / self.n
self.e_filtered = self.u - lambda1*w*self.u
self.vm = self.sig2 * tsls2.varb
self._cache = {}
def _get_spat_diag_props(self, x, sig2n_k):
self.k = self.kr
self._cache = {}
x = np.hstack((np.ones((x.shape[0], 1)), x))
self.x = REGI.regimeX_setup(
x, self.regimes, [True] * x.shape[1], self.regimes_set)
self.xtx = spdot(self.x.T, self.x)
self.xtxi = np.linalg.inv(self.xtx)
def _optimal_weight(reg, sig2n_k, warn=True):
try:
Hu = reg.h.toarray() * reg.u ** 2
except:
Hu = reg.h * reg.u ** 2
if sig2n_k:
S = spdot(reg.h.T, Hu, array_out=True) / (reg.n - reg.k)
else:
S = spdot(reg.h.T, Hu, array_out=True) / reg.n
Si = np.linalg.inv(S)
ZtH = spdot(reg.z.T, reg.h)
ZtHSi = spdot(ZtH, Si)
fac2 = np.linalg.inv(spdot(ZtHSi, ZtH.T, array_out=True))
fac3 = spdot(ZtHSi, spdot(reg.h.T, reg.y), array_out=True)
betas = np.dot(fac2, fac3)
if sig2n_k:
vm = fac2 * (reg.n - reg.k)
else:
vm = fac2 * reg.n
RegressionProps_basic(reg, betas=betas, vm=vm, sig2=False)
reg.title += " (Optimal-Weighted GMM)"
if warn:
set_warn(
reg, "Residuals treated as homoskedastic for the purpose of diagnostics.")
return
def _optimal_weight(reg, sig2n_k, warn=True):
try:
Hu = reg.h.toarray() * reg.u**2
except:
Hu = reg.h * reg.u**2
if sig2n_k:
S = spdot(reg.h.T, Hu, array_out=True) / (reg.n - reg.k)
else:
S = spdot(reg.h.T, Hu, array_out=True) / reg.n
Si = np.linalg.inv(S)
ZtH = spdot(reg.z.T, reg.h)
ZtHSi = spdot(ZtH, Si)
fac2 = np.linalg.inv(spdot(ZtHSi, ZtH.T, array_out=True))
fac3 = spdot(ZtHSi, spdot(reg.h.T, reg.y), array_out=True)
betas = np.dot(fac2, fac3)
if sig2n_k:
vm = fac2 * (reg.n - reg.k)
else:
vm = fac2 * reg.n
RegressionProps_basic(reg, betas=betas, vm=vm, sig2=False)
reg.title += " (Optimal-Weighted GMM)"
if warn:
set_warn(
reg,
"Residuals treated as homoskedastic for the purpose of diagnostics."
)
return
def _optimal_weight(self,sig2n_k):
''' Computes optimal weights GMM. '''
H = spdot(spdot(self.h,self.hthi),self.htz)
fac2, ZtHSi = self._get_fac2_het(H,self.u,sig2n_k)
fac3 = spdot(ZtHSi,spdot(H.T,self.y),array_out=True)
betas = np.dot(fac2,fac3)
"""
# Updating S_hat to S_tilde
predy = spdot(self.z, betas)
u = self.y - predy
fac2, ZtHSi = self._get_fac2_het(u,sig2n_k)
"""
if sig2n_k:
vm = fac2*(self.n-self.k)
else:
vm = fac2*self.n
#return betas, predy, u, vm #Use this line instead of next if updating S_hat to S_tilde.
return betas, vm
def j(self):
if 'j' not in self._cache:
wxb = self.w.sparse * self.reg.predy
wxb2 = np.dot(wxb.T, wxb)
xwxb = spdot(self.reg.x.T, wxb)
num1 = wxb2 - np.dot(xwxb.T, np.dot(self.reg.xtxi, xwxb))
num = num1 + (self.t * self.reg.sig2n)
den = self.reg.n * self.reg.sig2n
self._cache['j'] = num / den
return self._cache['j']
def j(self):
if 'j' not in self._cache:
wxb = self.w.sparse * self.reg.predy
wxb2 = np.dot(wxb.T, wxb)
xwxb = spdot(self.reg.x.T, wxb)
num1 = wxb2 - np.dot(xwxb.T, np.dot(self.reg.xtxi, xwxb))
num = num1 + (self.t * self.reg.sig2n)
den = self.reg.n * self.reg.sig2n
self._cache['j'] = num / den
return self._cache['j']
def slopes_vm(self):
try:
return self._cache['slopes_vm']
except AttributeError:
self._cache = {}
x = self.xmean
b = self.betas
dfdb = np.eye(self.k) - spdot(b.T, x) * spdot(b, x.T)
slopes_vm = (self.scale ** 2) * \
np.dot(np.dot(dfdb, self.vm), dfdb.T)
self._cache['slopes_vm'] = slopes_vm[1:, 1:]
except KeyError:
x = self.xmean
b = self.betas
dfdb = np.eye(self.k) - spdot(b.T, x) * spdot(b, x.T)
slopes_vm = (self.scale ** 2) * \
np.dot(np.dot(dfdb, self.vm), dfdb.T)
self._cache['slopes_vm'] = slopes_vm[1:, 1:]
return self._cache['slopes_vm']
def akTest(iv, w, spDcache):
"""
Computes AK-test for the general case (end. reg. + sp. lag)
...
Parameters
----------
iv : STSLS_dev
Instance from spatial 2SLS regression
w : W
Spatial weights instance
spDcache : spDcache
Instance of spDcache class
Attributes
----------
mi : float
Moran's I statistic for IV residuals
ak : float
Square of corrected Moran's I for residuals::
.. math::
ak = \dfrac{N \times I^*}{\phi^2}
p : float
P-value of the test
ToDo:
* Code in as Nancy
* Compare both
"""
mi = get_mI(iv, w, spDcache)
# Phi2
etwz = spdot(iv.u.T, spdot(w.sparse, iv.z))
a = np.dot(etwz,np.dot(iv.varb,etwz.T))
s12 = (w.s0 / w.n)**2
phi2 = ( spDcache.t + (4.0 / iv.sig2n) * a ) / (s12 * w.n)
ak = w.n * mi**2 / phi2
pval = chisqprob(ak, 1)
return (mi, ak[0][0], pval[0][0])
def __init__(self, y, x, robust=None, gwk=None, sig2n_k=True):
self.x = x
self.xtx = spdot(self.x.T, self.x)
xty = spdot(self.x.T, y)
self.xtxi = la.inv(self.xtx)
self.betas = np.dot(self.xtxi, xty)
predy = spdot(self.x, self.betas)
u = y - predy
self.u = u
self.predy = predy
self.y = y
self.n, self.k = self.x.shape
if sig2n_k:
self.sig2 = self.sig2n_k
else:
self.sig2 = self.sig2n
if robust is not None:
self.vm = ROBUST.robust_vm(reg=self, gwk=gwk, sig2n_k=sig2n_k)
def __init__(self, y, x, robust=None, gwk=None, sig2n_k=True):
self.x = x
self.xtx = spdot(self.x.T, self.x)
xty = spdot(self.x.T, y)
self.xtxi = la.inv(self.xtx)
self.betas = np.dot(self.xtxi, xty)
predy = spdot(self.x, self.betas)
u = y - predy
self.u = u
self.predy = predy
self.y = y
self.n, self.k = self.x.shape
if sig2n_k:
self.sig2 = self.sig2n_k
else:
self.sig2 = self.sig2n
if robust is not None:
self.vm = ROBUST.robust_vm(reg=self, gwk=gwk, sig2n_k=sig2n_k)
def hac_multi(reg, gwk, constant=False):
"""
HAC robust estimation of the variance-covariance matrix for multi-regression object
Parameters
----------
reg : Regression object (OLS or TSLS)
output instance from a regression model
gwk : PySAL weights object
Spatial weights based on kernel functions
Returns
--------
psi : kxk array
Robust estimation of the variance-covariance
"""
if not constant:
reg.hac_var = check_constant(reg.hac_var)
xu = spbroadcast(reg.hac_var, reg.u)
gwkxu = lag_spatial(gwk, xu)
psi0 = spdot(xu.T, gwkxu)
counter = 0
for m in reg.multi:
reg.multi[m].robust = 'hac'
reg.multi[m].name_gwk = reg.name_gwk
try:
psi1 = spdot(reg.multi[m].varb, reg.multi[m].zthhthi)
reg.multi[m].vm = spdot(psi1, np.dot(psi0, psi1.T))
except:
reg.multi[m].vm = spdot(
reg.multi[m].xtxi, np.dot(psi0, reg.multi[m].xtxi))
reg.vm[(counter * reg.kr):((counter + 1) * reg.kr),
(counter * reg.kr):((counter + 1) * reg.kr)] = reg.multi[m].vm
counter += 1
def hac_multi(reg, gwk, constant=False):
"""
HAC robust estimation of the variance-covariance matrix for multi-regression object
Parameters
----------
reg : Regression object (OLS or TSLS)
output instance from a regression model
gwk : PySAL weights object
Spatial weights based on kernel functions
Returns
--------
psi : kxk array
Robust estimation of the variance-covariance
"""
if not constant:
reg.hac_var = check_constant(reg.hac_var)
xu = spbroadcast(reg.hac_var, reg.u)
gwkxu = lag_spatial(gwk, xu)
psi0 = spdot(xu.T, gwkxu)
counter = 0
for m in reg.multi:
reg.multi[m].robust = 'hac'
reg.multi[m].name_gwk = reg.name_gwk
try:
psi1 = spdot(reg.multi[m].varb, reg.multi[m].zthhthi)
reg.multi[m].vm = spdot(psi1, np.dot(psi0, psi1.T))
except:
reg.multi[m].vm = spdot(reg.multi[m].xtxi,
np.dot(psi0, reg.multi[m].xtxi))
reg.vm[(counter * reg.kr):((counter + 1) * reg.kr),
(counter * reg.kr):((counter + 1) * reg.kr)] = reg.multi[m].vm
counter += 1
def _get_fac2_het(self,H,u,sig2n_k):
D = SP.lil_matrix((self.n, self.n))
D.setdiag(u**2)
if sig2n_k:
S = spdot(spdot(self.z.T,D),self.z,array_out=True)/(self.n-self.k)
else:
S = spdot(spdot(self.z.T,D),self.z,array_out=True)/self.n
Si = np.linalg.inv(S)
ZtH = spdot(self.z.T,H)
ZtHSi = spdot(ZtH,Si)
fac2 = np.linalg.inv(spdot(ZtHSi,ZtH.T,array_out=True))
return fac2, ZtHSi
def ll(self, par):
beta = np.reshape(np.array(par), (self.k, 1))
q = 2 * self.y - 1
qxb = q * spdot(self.x, beta)
ll = sum(np.log(norm.cdf(qxb)))
return ll
def xb(self):
if 'xb' not in self._cache:
self._cache['xb'] = spdot(self.x, self.betas)
return self._cache['xb']
def ll(self, par):
beta = np.reshape(np.array(par), (self.k, 1))
q = 2 * self.y - 1
qxb = q * spdot(self.x, beta)
ll = sum(np.log(norm.cdf(qxb)))
return ll
def robust_vm(reg, gwk=None):
"""
Robust estimation of the variance-covariance matrix. Estimated by White (default) or HAC (if wk is provided).
Parameters
----------
reg : Regression object (OLS or TSLS)
output instance from a regression model
gwk : PySAL weights object
Optional. Spatial weights based on kernel functions
If provided, returns the HAC variance estimation
Returns
--------
psi : kxk array
Robust estimation of the variance-covariance
Examples
--------
>>> import numpy as np
>>> import pysal
>>> from ols import OLS
>>> from twosls import TSLS
>>> db=pysal.open(pysal.examples.get_path("NAT.dbf"),"r")
>>> y = np.array(db.by_col("HR90"))
>>> y = np.reshape(y, (y.shape[0],1))
>>> X = []
>>> X.append(db.by_col("RD90"))
>>> X.append(db.by_col("DV90"))
>>> X = np.array(X).T
Example with OLS with unadjusted standard errors
>>> ols = OLS(y,X)
>>> ols.vm
array([[ 0.17004545, 0.00226532, -0.02243898],
[ 0.00226532, 0.00941319, -0.00031638],
[-0.02243898, -0.00031638, 0.00313386]])
Example with OLS and White
>>> ols = OLS(y,X, robust='white')
>>> ols.vm
array([[ 0.24491641, 0.01092258, -0.03438619],
[ 0.01092258, 0.01796867, -0.00071345],
[-0.03438619, -0.00071345, 0.00501042]])
Example with OLS and HAC
>>> wk = pysal.kernelW_from_shapefile(pysal.examples.get_path('NAT.shp'),k=15,function='triangular', fixed=False)
>>> wk.transform = 'o'
>>> ols = OLS(y,X, robust='hac', gwk=wk)
>>> ols.vm
array([[ 0.29213532, 0.01670361, -0.03948199],
[ 0.01655557, 0.02295829, -0.00116874],
[-0.03941483, -0.00119077, 0.00568314]])
Example with 2SLS and White
>>> yd = []
>>> yd.append(db.by_col("UE90"))
>>> yd = np.array(yd).T
>>> q = []
>>> q.append(db.by_col("UE80"))
>>> q = np.array(q).T
>>> tsls = TSLS(y, X, yd, q=q, robust='white')
>>> tsls.vm
array([[ 0.29569954, 0.04119843, -0.02496858, -0.01640185],
[ 0.04119843, 0.03647762, 0.004702 , -0.00987345],
[-0.02496858, 0.004702 , 0.00648262, -0.00292891],
[-0.01640185, -0.00987345, -0.00292891, 0.0053322 ]])
Example with 2SLS and HAC
>>> tsls = TSLS(y, X, yd, q=q, robust='hac', gwk=wk)
>>> tsls.vm
array([[ 0.41985329, 0.06823119, -0.02883889, -0.02788116],
[ 0.06867042, 0.04887508, 0.00497443, -0.01367746],
[-0.02856454, 0.00501402, 0.0072195 , -0.00321604],
[-0.02810131, -0.01364908, -0.00318197, 0.00713251]])
"""
if hasattr(reg, 'h'): #If reg has H, do 2SLS estimator. OLS otherwise.
tsls = True
xu = spbroadcast(reg.h, reg.u)
else:
tsls = False
xu = spbroadcast(reg.x, reg.u)
if gwk: #If gwk do HAC. White otherwise.
gwkxu = lag_spatial(gwk, xu)
psi0 = spdot(xu.T, gwkxu)
else:
psi0 = spdot(xu.T, xu)
if tsls:
psi1 = spdot(reg.varb, reg.zthhthi)
psi = spdot(psi1, np.dot(psi0, psi1.T))
else:
psi = spdot(reg.xtxi, np.dot(psi0, reg.xtxi))
return psi
def __init__(self, y, x, equationID, yend=None, q=None, w=None,\
cores=None, sig2n_k=False, name_y=None, name_x=None, name_yend=None,\
name_q=None, name_equationID=None, name_ds=None, name_w=None, vm=False,\
w_lags=None,lag_q=None):
self.equationID = equationID
eq_set = list(set(self.equationID))
self.eq_set = eq_set
self.n_eq = len(eq_set)
if w:
self.n = w.n
assert self.n_eq * w.n == y.shape[
0], "Number of equations, weights dimension and lenght of vector Y are not aligned."
ws = w.sparse
else:
if w_lags:
raise Exception, "W matrix required to run spatial lag model."
ws = None
eq_ids = dict(
(r, list(np.where(np.array(equationID) == r)[0])) for r in eq_set)
#Running 2SLS for each equation separately
stp2 = {}
if system() == 'Windows':
for r in eq_set:
stp2[r] = _run_stp1(y, x, yend, q, eq_ids, r, sig2n_k, ws,
w_lags, lag_q)
results_stp2 = stp2
else:
pool = mp.Pool(cores)
for r in eq_set:
stp2[r] = pool.apply_async(_run_stp1,
args=(
y,
x,
yend,
q,
eq_ids,
r,
sig2n_k,
ws,
w_lags,
lag_q,
))
pool.close()
pool.join()
results_stp2 = dict((r, stp2[r].get()) for r in eq_set)
if not w:
self.n = results_stp2[eq_set[0]].n
assert self.n_eq * self.n == y.shape[
0], "Number of equations and lenght of vector Y are not aligned."
#Building sigma matrix
if sig2n_k:
dof = list(results_stp2[r].n - results_stp2[r].k for r in eq_set)
dof = np.array(dof).reshape(self.n_eq, 1)
den = np.dot(dof, dof.T)**0.5
else:
den = np.array([float(self.n)] * self.n_eq**2).reshape(
self.n_eq, self.n_eq)
BigU1 = np.hstack((results_stp2[r].u for r in eq_set))
sig = la.inv(np.dot(BigU1.T, BigU1) / den)
#Building stacked matrices:
BigZ = np.hstack((results_stp2[r].z.flatten() for r in eq_set))
BigZhat = np.hstack((results_stp2[r].zhat.flatten() for r in eq_set))
k_z = 0
indices_z, indptr_z = [], []
for r in eq_set:
indices_z += range(k_z, results_stp2[r].z.shape[1] + k_z) * self.n
indptr_z += list(
np.arange(0, self.n) * results_stp2[r].z.shape[1] +
k_z * self.n)
k_z += results_stp2[r].z.shape[1]
BigZ = SP.csr_matrix((BigZ, indices_z, indptr_z + [BigZ.shape[0]]))
BigZhat = SP.csr_matrix(
(BigZhat, indices_z, indptr_z + [BigZhat.shape[0]]))
BigY = np.vstack((results_stp2[r].y for r in eq_set))
#Estimating parameters
BigZhattsig = np.vstack((np.hstack(
(results_stp2[eq_set[i]].zhat.T * sig[i, j]
for j in range(self.n_eq))) for i in range(self.n_eq)))
#BigZhattsig = BigZhat.T*SP.kron(sig,SP.identity(self.n)) #slower than line above
self.vm = la.inv(spdot(BigZhattsig, BigZhat))
fact2 = spdot(BigZhattsig, BigY)
self.betas = spdot(self.vm, fact2)
self.std_err = np.sqrt(self.vm.diagonal())
#Prepare output
self.multi = results_stp2
k_b = 0
self.predy = np.zeros(y.shape, float)
self.u = np.zeros(y.shape, float)
sig1 = np.zeros((self.n_eq, self.n_eq), float)
for r_i in range(self.n_eq):
r_j = r_i
r = eq_set[r_i]
k_b1 = self.multi[r].betas.shape[0]
self.multi[r].betas[:, ] = self.betas[k_b:k_b + k_b1, ]
self.multi[r].vm = self.vm[k_b:k_b + k_b1, k_b:k_b + k_b1]
k_b += k_b1
self.multi[r].predy = spdot(self.multi[r].z, self.multi[r].betas)
self.multi[r].u = self.multi[r].y - self.multi[r].predy
self.predy[eq_ids[r], ] = self.multi[r].predy
self.u[eq_ids[r], ] = self.multi[r].u
while r_j >= 0:
sig1[r_i, r_j] = np.dot(self.multi[eq_set[r_i]].u.T,
self.multi[eq_set[r_j]].u)
sig1[r_j, r_i] = sig1[r_i, r_j]
r_j += -1
sig_var = sig1.diagonal().reshape(1, self.n_eq)
self.u_cov = sig1 / np.sqrt(np.dot(sig_var.T, sig_var))
if not w_lags:
title = "THREE-STAGE LEAST SQUARES - EQUATION "
if yend != None:
self.multi = USER.set_name_multi(self.multi,
eq_set,
name_equationID,
y,
x,
name_y,
name_x,
name_ds,
title,
name_w,
robust=None,
endog=(yend, q, name_yend,
name_q),
sp_lag=False)
SUMMARY.TSLS_multi(reg=self,
multireg=self.multi,
vm=vm,
spat_diag=False,
sur=True)
else:
self.multi = USER.set_name_multi(self.multi,
eq_set,
name_equationID,
y,
x,
name_y,
name_x,
name_ds,
title,
name_w,
robust=None,
endog=False,
sp_lag=False)
SUMMARY.OLS_multi(reg=self,
multireg=self.multi,
vm=vm,
nonspat_diag=False,
moran=False,
white_test=False,
spat_diag=False,
sur=True)
def trA(self):
if 'trA' not in self._cache:
xtwx = spdot(self.reg.x.T, spdot(self.w.sparse, self.reg.x))
mw = np.dot(self.reg.xtxi, xtwx)
self._cache['trA'] = np.sum(mw.diagonal())
return self._cache['trA']
def __init__(self, y, x, w, method='full', epsilon=0.0000001):
# set up main regression variables and spatial filters
self.y = y
self.x = x
self.n, self.k = self.x.shape
self.method = method
self.epsilon = epsilon
#W = w.full()[0]
#Wsp = w.sparse
ylag = ps.lag_spatial(w, y)
# b0, b1, e0 and e1
xtx = spdot(self.x.T, self.x)
xtxi = la.inv(xtx)
xty = spdot(self.x.T, self.y)
xtyl = spdot(self.x.T, ylag)
b0 = np.dot(xtxi, xty)
b1 = np.dot(xtxi, xtyl)
e0 = self.y - spdot(x, b0)
e1 = ylag - spdot(x, b1)
methodML = method.upper()
# call minimizer using concentrated log-likelihood to get rho
if methodML in ['FULL', 'LU', 'ORD']:
if methodML == 'FULL':
W = w.full()[0] # moved here
res = minimize_scalar(lag_c_loglik, 0.0, bounds=(-1.0, 1.0),
args=(
self.n, e0, e1, W), method='bounded',
tol=epsilon)
elif methodML == 'LU':
I = sp.identity(w.n)
Wsp = w.sparse # moved here
res = minimize_scalar(lag_c_loglik_sp, 0.0, bounds=(-1.0,1.0),
args=(self.n, e0, e1, I, Wsp),
method='bounded', tol=epsilon)
elif methodML == 'ORD':
# check on symmetry structure
if w.asymmetry(intrinsic=False) == []:
ww = symmetrize(w)
WW = ww.todense()
evals = la.eigvalsh(WW)
else:
W = w.full()[0] # moved here
evals = la.eigvals(W)
res = minimize_scalar(lag_c_loglik_ord, 0.0, bounds=(-1.0, 1.0),
args=(
self.n, e0, e1, evals), method='bounded',
tol=epsilon)
else:
# program will crash, need to catch
print("{0} is an unsupported method".format(methodML))
self = None
return
self.rho = res.x[0][0]
# compute full log-likelihood, including constants
ln2pi = np.log(2.0 * np.pi)
llik = -res.fun - self.n / 2.0 * ln2pi - self.n / 2.0
self.logll = llik[0][0]
# b, residuals and predicted values
b = b0 - self.rho * b1
self.betas = np.vstack((b, self.rho)) # rho added as last coefficient
self.u = e0 - self.rho * e1
self.predy = self.y - self.u
xb = spdot(x, b)
self.predy_e = inverse_prod(
w.sparse, xb, self.rho, inv_method="power_exp", threshold=epsilon)
self.e_pred = self.y - self.predy_e
# residual variance
self.sig2 = self.sig2n # no allowance for division by n-k
# information matrix
a = -self.rho * W
np.fill_diagonal(a, 1.0)
ai = la.inv(a)
wai = np.dot(W, ai)
tr1 = np.trace(wai)
wai2 = np.dot(wai, wai)
tr2 = np.trace(wai2)
waiTwai = np.dot(wai.T, wai)
tr3 = np.trace(waiTwai)
wpredy = ps.lag_spatial(w, self.predy_e)
wpyTwpy = np.dot(wpredy.T, wpredy)
xTwpy = spdot(x.T, wpredy)
# order of variables is beta, rho, sigma2
v1 = np.vstack(
(xtx / self.sig2, xTwpy.T / self.sig2, np.zeros((1, self.k))))
v2 = np.vstack(
(xTwpy / self.sig2, tr2 + tr3 + wpyTwpy / self.sig2, tr1 / self.sig2))
v3 = np.vstack(
(np.zeros((self.k, 1)), tr1 / self.sig2, self.n / (2.0 * self.sig2 ** 2)))
v = np.hstack((v1, v2, v3))
self.vm1 = la.inv(v) # vm1 includes variance for sigma2
self.vm = self.vm1[:-1, :-1] # vm is for coefficients only
def __init__(self, y, x, equationID, w,\
cores=None, name_y=None, name_x=None,\
name_equationID=None, name_ds=None, name_w=None, vm=False,\
maxiter=1000, epsilon=0.00001):
eq_ids, results_stp2 = _sur_frame.__init__(self, y, x, equationID, w, cores)
#Building stacked matrices:
BigU1 = np.hstack((results_stp2[r].u for r in self.eq_set))
BigY = np.hstack((results_stp2[r].y for r in self.eq_set))
#Running SUR:
xw = dict((r,lag_spatial(w,results_stp2[r].x)) for r in self.eq_set)
BigWY = lag_spatial(w,BigY)
#wroot = SP.linalg.eigs(w.sparse)
k_tot = self.n_eq*self.k
XX = np.zeros((k_tot,k_tot),float)
WXX = np.zeros((k_tot,k_tot),float)
XWX = np.zeros((k_tot,k_tot),float)
XWWX = np.zeros((k_tot,k_tot),float)
XY = np.zeros((k_tot,self.n_eq),float)
WXY = np.zeros((k_tot,self.n_eq),float)
XWY = np.zeros((k_tot,self.n_eq),float)
XWWY = np.zeros((k_tot,self.n_eq),float)
r_temp = 0
for r in range(self.n_eq):
xi = results_stp2[self.eq_set[r]].x
wxi = xw[self.eq_set[r]]
j_temp = 0
for j in range(self.n_eq):
xj = results_stp2[self.eq_set[j]].x
wxj = xw[self.eq_set[j]]
XX[r_temp:r_temp+self.k,j_temp:j_temp+self.k] = spdot(xi.T,xj)
WXX[r_temp:r_temp+self.k,j_temp:j_temp+self.k] = spdot(wxi.T,xj)
XWX[r_temp:r_temp+self.k,j_temp:j_temp+self.k] = spdot(xi.T,wxj)
XWWX[r_temp:r_temp+self.k,j_temp:j_temp+self.k] = spdot(wxi.T,wxj)
XY[r_temp:r_temp+self.k,j] = spdot(xi.T,BigY[:,j])
WXY[r_temp:r_temp+self.k,j] = spdot(wxi.T,BigY[:,j])
XWY[r_temp:r_temp+self.k,j] = spdot(xi.T,BigWY[:,j])
XWWY[r_temp:r_temp+self.k,j] = spdot(wxi.T,BigWY[:,j])
j_temp += self.k
r_temp += self.k
#Start of main loop
n_iter = 0
L1 = 0
lambd1 = 0.5*np.ones((self.n_eq,1),float)
while np.abs(lambd1-lambd)>epsilon and n_iter<=maxiter:
n_iter += 1
lambd = lambd1
L0 = L1
ulam = BigU1 - lag_spatial(w,BigU1)*lambd1.T
sig = np.dot(ulam.T,ulam) / self.n
try:
sigi = la.inv(sig)
det1 = determinant(sigi)
except:
raise Exception, "ERROR: singular variance matrix"
# FGLS for betas
xomxi = np.zeros((k_tot,k_tot),float)
xomyi = np.zeros((k_tot,1),float)
r_temp = 0
for r in range(self.n_eq):
r_temp2 = r_temp+self.k
lami=lambd[r][0]
j_temp = 0
for j in range(self.n_eq):
j_temp2 = j_temp+self.k
lamj=lambd[j][0]
xomxi[r_temp:r_temp2,j_temp:j_temp2] = sigi[r,j]*(XX[r_temp:r_temp2,j_temp:j_temp2] - lami*WXX[r_temp:r_temp2,j_temp:j_temp2] - lamj*XWX[r_temp:r_temp2,j_temp:j_temp2] + (lami*lamj)*XWWX[r_temp:r_temp2,j_temp:j_temp2])
xomyi[r_temp:r_temp2,0] = xomyi[r_temp:r_temp2,0]+sigi[r,j]*(XY[r_temp:r_temp2,j] - lami*WXY[r_temp:r_temp2,j] - lamj*XWY[r_temp:r_temp2,j] + (lami*lamj)*XWWY[r_temp:r_temp2,j])
j_temp += self.k
r_temp += self.k
try:
xomix = la.inv(xomxi)
except:
raise Exception, "ERROR: singular variance matrix"
bml1 = np.dot(xomix,xomyi)
def xb(self):
if 'xb' not in self._cache:
self._cache['xb'] = spdot(self.x, self.betas)
return self._cache['xb']
#print('initial_lamb_sig:',lambda1,sig_v,sig_1)
#print('theta:', 1 - np.sqrt(sig_v)/ np.sqrt(sig_1))
Xi_a = SP.diags([(sig_v*sig_v)/(T-1),sig_1*sig_1])
if full_weights:
Tau = _get_Tau(w.sparse,trace_w2)
else:
Tau = SP.identity(3)
Xi = SP.kron(Xi_a,Tau)
moments_b,_ = _moments_kkp(w.sparse, ols.u, 1,trace_w2)
G = np.vstack((np.hstack((moments[0],np.zeros((3,1)))),moments_b[0]))
moments6 = [G,np.vstack((moments[1],moments_b[1]))]
lambda2,sig_vb,sig_1b = optim_moments(moments6, vcX=Xi.toarray(), all_par=True, start=[lambda1,sig_v,sig_1])
# 2a. reg -->\hat{betas}
theta = 1 - np.sqrt(sig_vb)/np.sqrt(sig_1b)
#print('theta:', theta)
gls_w = SP.identity(N*T) - theta*Q1
#With omega
xs = gls_w.dot(get_spFilter(w, lambda2, x))
ys = gls_w.dot(get_spFilter(w, lambda2, y))
ols_s = OLS.BaseOLS(y=ys, x=xs)
self.predy = spdot(self.x, ols_s.betas)
self.u = self.y - self.predy
self.vm = ols_s.vm #Check
self.betas = np.vstack((ols_s.betas, lambda2, sig_vb, sig_1b))
self.e_filtered = self.u - lambda2 * SP.kron(SP.identity(T),
w.sparse).dot(self.u)
self.t, self.n = T, N
self._cache = {}
def robust_vm(reg, gwk=None, sig2n_k=False):
"""
Robust estimation of the variance-covariance matrix. Estimated by White (default) or HAC (if wk is provided).
Parameters
----------
reg : Regression object (OLS or TSLS)
output instance from a regression model
gwk : PySAL weights object
Optional. Spatial weights based on kernel functions
If provided, returns the HAC variance estimation
sig2n_k : boolean
If True, then use n-k to rescale the vc matrix.
If False, use n. (White only)
Returns
--------
psi : kxk array
Robust estimation of the variance-covariance
Examples
--------
>>> import numpy as np
>>> import pysal
>>> from ols import OLS
>>> from twosls import TSLS
>>> db=pysal.open(pysal.examples.get_path("NAT.dbf"),"r")
>>> y = np.array(db.by_col("HR90"))
>>> y = np.reshape(y, (y.shape[0],1))
>>> X = []
>>> X.append(db.by_col("RD90"))
>>> X.append(db.by_col("DV90"))
>>> X = np.array(X).T
Example with OLS with unadjusted standard errors
>>> ols = OLS(y,X)
>>> ols.vm
array([[ 0.17004545, 0.00226532, -0.02243898],
[ 0.00226532, 0.00941319, -0.00031638],
[-0.02243898, -0.00031638, 0.00313386]])
Example with OLS and White
>>> ols = OLS(y,X, robust='white')
>>> ols.vm
array([[ 0.24515481, 0.01093322, -0.03441966],
[ 0.01093322, 0.01798616, -0.00071414],
[-0.03441966, -0.00071414, 0.0050153 ]])
Example with OLS and HAC
>>> wk = pysal.kernelW_from_shapefile(pysal.examples.get_path('NAT.shp'),k=15,function='triangular', fixed=False)
>>> wk.transform = 'o'
>>> ols = OLS(y,X, robust='hac', gwk=wk)
>>> ols.vm
array([[ 0.29213532, 0.01670361, -0.03948199],
[ 0.01655557, 0.02295829, -0.00116874],
[-0.03941483, -0.00119077, 0.00568314]])
Example with 2SLS and White
>>> yd = []
>>> yd.append(db.by_col("UE90"))
>>> yd = np.array(yd).T
>>> q = []
>>> q.append(db.by_col("UE80"))
>>> q = np.array(q).T
>>> tsls = TSLS(y, X, yd, q=q, robust='white')
>>> tsls.vm
array([[ 0.29569954, 0.04119843, -0.02496858, -0.01640185],
[ 0.04119843, 0.03647762, 0.004702 , -0.00987345],
[-0.02496858, 0.004702 , 0.00648262, -0.00292891],
[-0.01640185, -0.00987345, -0.00292891, 0.0053322 ]])
Example with 2SLS and HAC
>>> tsls = TSLS(y, X, yd, q=q, robust='hac', gwk=wk)
>>> tsls.vm
array([[ 0.41985329, 0.06823119, -0.02883889, -0.02788116],
[ 0.06867042, 0.04887508, 0.00497443, -0.01367746],
[-0.02856454, 0.00501402, 0.0072195 , -0.00321604],
[-0.02810131, -0.01364908, -0.00318197, 0.00713251]])
"""
if hasattr(reg, 'h'): # If reg has H, do 2SLS estimator. OLS otherwise.
tsls = True
xu = spbroadcast(reg.h, reg.u)
else:
tsls = False
xu = spbroadcast(reg.x, reg.u)
if gwk: # If gwk do HAC. White otherwise.
gwkxu = lag_spatial(gwk, xu)
psi0 = spdot(xu.T, gwkxu)
else:
psi0 = spdot(xu.T, xu)
if sig2n_k:
psi0 = psi0 * (1. * reg.n / (reg.n - reg.k))
if tsls:
psi1 = spdot(reg.varb, reg.zthhthi)
psi = spdot(psi1, np.dot(psi0, psi1.T))
else:
psi = spdot(reg.xtxi, np.dot(psi0, reg.xtxi))
return psi
def __init__(self, y, x, equationID, yend=None, q=None, w=None,\
cores=None, sig2n_k=False, name_y=None, name_x=None, name_yend=None,\
name_q=None, name_equationID=None, name_ds=None, name_w=None, vm=False,\
w_lags=None,lag_q=None):
self.equationID = equationID
eq_set = list(set(self.equationID))
self.eq_set = eq_set
self.n_eq = len(eq_set)
if w:
self.n = w.n
assert self.n_eq*w.n==y.shape[0], "Number of equations, weights dimension and lenght of vector Y are not aligned."
ws = w.sparse
else:
if w_lags:
raise Exception, "W matrix required to run spatial lag model."
ws = None
eq_ids = dict((r, list(np.where(np.array(equationID) == r)[0])) for r in eq_set)
#Running 2SLS for each equation separately
stp2 = {}
if system() == 'Windows':
for r in eq_set:
stp2[r] = _run_stp1(y,x,yend,q,eq_ids,r,sig2n_k,ws,w_lags,lag_q)
results_stp2 = stp2
else:
pool = mp.Pool(cores)
for r in eq_set:
stp2[r] = pool.apply_async(_run_stp1,args=(y,x,yend,q,eq_ids,r,sig2n_k,ws,w_lags,lag_q, ))
pool.close()
pool.join()
results_stp2 = dict((r, stp2[r].get()) for r in eq_set)
if not w:
self.n = results_stp2[eq_set[0]].n
assert self.n_eq*self.n==y.shape[0], "Number of equations and lenght of vector Y are not aligned."
#Building sigma matrix
if sig2n_k:
dof = list(results_stp2[r].n - results_stp2[r].k for r in eq_set)
dof = np.array(dof).reshape(self.n_eq,1)
den = np.dot(dof,dof.T)**0.5
else:
den = np.array([float(self.n)]*self.n_eq**2).reshape(self.n_eq,self.n_eq)
BigU1 = np.hstack((results_stp2[r].u for r in eq_set))
sig = la.inv(np.dot(BigU1.T,BigU1)/den)
#Building stacked matrices:
BigZ = np.hstack((results_stp2[r].z.flatten() for r in eq_set))
BigZhat = np.hstack((results_stp2[r].zhat.flatten() for r in eq_set))
k_z = 0
indices_z,indptr_z = [],[]
for r in eq_set:
indices_z += range(k_z,results_stp2[r].z.shape[1]+k_z)*self.n
indptr_z += list(np.arange(0,self.n)*results_stp2[r].z.shape[1] + k_z*self.n)
k_z += results_stp2[r].z.shape[1]
BigZ = SP.csr_matrix((BigZ,indices_z,indptr_z+[BigZ.shape[0]]))
BigZhat = SP.csr_matrix((BigZhat,indices_z,indptr_z+[BigZhat.shape[0]]))
BigY = np.vstack((results_stp2[r].y for r in eq_set))
#Estimating parameters
BigZhattsig = np.vstack((np.hstack((results_stp2[eq_set[i]].zhat.T*sig[i,j] for j in range(self.n_eq))) for i in range(self.n_eq)))
#BigZhattsig = BigZhat.T*SP.kron(sig,SP.identity(self.n)) #slower than line above
self.vm = la.inv(spdot(BigZhattsig,BigZhat))
fact2 = spdot(BigZhattsig,BigY)
self.betas = spdot(self.vm,fact2)
self.std_err = np.sqrt(self.vm.diagonal())
#Prepare output
self.multi = results_stp2
k_b = 0
self.predy = np.zeros(y.shape,float)
self.u = np.zeros(y.shape,float)
sig1 = np.zeros((self.n_eq,self.n_eq),float)
for r_i in range(self.n_eq):
r_j = r_i
r = eq_set[r_i]
k_b1 = self.multi[r].betas.shape[0]
self.multi[r].betas[:,] = self.betas[k_b:k_b+k_b1,]
self.multi[r].vm = self.vm[k_b:k_b+k_b1,k_b:k_b+k_b1]
k_b += k_b1
self.multi[r].predy = spdot(self.multi[r].z,self.multi[r].betas)
self.multi[r].u = self.multi[r].y - self.multi[r].predy
self.predy[eq_ids[r],] = self.multi[r].predy
self.u[eq_ids[r],] = self.multi[r].u
while r_j >= 0:
sig1[r_i,r_j] = np.dot(self.multi[eq_set[r_i]].u.T,self.multi[eq_set[r_j]].u)
sig1[r_j,r_i] = sig1[r_i,r_j]
r_j += -1
sig_var = sig1.diagonal().reshape(1,self.n_eq)
self.u_cov = sig1/np.sqrt(np.dot(sig_var.T,sig_var))
if not w_lags:
title = "THREE-STAGE LEAST SQUARES - EQUATION "
if yend != None:
self.multi = USER.set_name_multi(self.multi,eq_set,name_equationID,y,x,name_y,name_x,name_ds,title,name_w,robust=None,endog=(yend,q,name_yend,name_q),sp_lag=False)
SUMMARY.TSLS_multi(reg=self, multireg=self.multi, vm=vm, spat_diag=False, sur=True)
else:
self.multi = USER.set_name_multi(self.multi,eq_set,name_equationID,y,x,name_y,name_x,name_ds,title,name_w,robust=None,endog=False,sp_lag=False)
SUMMARY.OLS_multi(reg=self, multireg=self.multi, vm=vm, nonspat_diag=False, moran=False, white_test=False, spat_diag=False, sur=True)
def trA(self):
if 'trA' not in self._cache:
xtwx = spdot(self.reg.x.T, spdot(self.w.sparse, self.reg.x))
mw = np.dot(self.reg.xtxi, xtwx)
self._cache['trA'] = np.sum(mw.diagonal())
return self._cache['trA']
