def get_x_lag(self, w, regimes_att):
if regimes_att:
xlag = weights.lag_spatial(w, regimes_att['x'])
xlag = REGI.Regimes_Frame.__init__(self, xlag,
regimes_att['regimes'], constant_regi=None, cols2regi=regimes_att['cols2regi'])[0]
xlag = xlag.toarray()
else:
xlag = weights.lag_spatial(w, self.x)
return xlag
def _build_local_environment(
data, groups, w=None, bandwidth=1000, function="triangular"
):
"""Convert observations into spatially-weighted sums.
Parameters
----------
data : GeoDataFrame
dataframe with local observations
w : libpysal.weights object
weights matrix defining the local environment
Returns
-------
DataFrame
Spatialized data
"""
data = data.copy().reset_index()
if data.crs.is_geographic:
warnings.warn(
"GeoDataFrame appears to have a geographic coordinate system and likely needs to be reprojected"
)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
if not w:
w = Kernel.from_dataframe(data, bandwidth=bandwidth, function=function)
new_data = []
w = fill_diagonal(w)
for y in data[groups]:
new_data.append(lag_spatial(w, data[y]))
new_data = pd.DataFrame(dict(zip(groups, new_data))).round(0)
new_data = data.geometry.to_frame().join(new_data.reset_index())
return new_data
def _calc(self, Y, w, k):
wY = weights.lag_spatial(w, Y)
dx = Y[:, -1] - Y[:, 0]
dy = wY[:, -1] - wY[:, 0]
self.wY = wY
self.Y = Y
r = np.sqrt(dx * dx + dy * dy)
theta = np.arctan2(dy, dx)
neg = theta < 0.0
utheta = theta * (1 - neg) + neg * (2 * np.pi + theta)
counts, bins = np.histogram(utheta, self.cuts)
results = {}
results['counts'] = counts
results['theta'] = theta
results['bins'] = bins
results['r'] = r
results['lag'] = wY
results['dx'] = dx
results['dy'] = dy
return results
def __init__(self, y, x, w, method='full', epsilon=0.0000001):
# set up main regression variables and spatial filters
self.y = y
self.x = x
self.n, self.k = self.x.shape
self.method = method
self.epsilon = epsilon
#W = w.full()[0]
#Wsp = w.sparse
ylag = weights.lag_spatial(w, y)
# b0, b1, e0 and e1
xtx = spdot(self.x.T, self.x)
xtxi = la.inv(xtx)
xty = spdot(self.x.T, self.y)
xtyl = spdot(self.x.T, ylag)
b0 = spdot(xtxi, xty)
b1 = spdot(xtxi, xtyl)
e0 = self.y - spdot(x, b0)
e1 = ylag - spdot(x, b1)
methodML = method.upper()
# call minimizer using concentrated log-likelihood to get rho
if methodML in ['FULL', 'LU', 'ORD']:
if methodML == 'FULL':
W = w.full()[0] # moved here
res = minimize_scalar(lag_c_loglik,
0.0,
bounds=(-1.0, 1.0),
args=(self.n, e0, e1, W),
method='bounded',
tol=epsilon)
elif methodML == 'LU':
I = sp.identity(w.n)
Wsp = w.sparse # moved here
W = Wsp
res = minimize_scalar(lag_c_loglik_sp,
0.0,
bounds=(-1.0, 1.0),
args=(self.n, e0, e1, I, Wsp),
method='bounded',
tol=epsilon)
elif methodML == 'ORD':
# check on symmetry structure
if w.asymmetry(intrinsic=False) == []:
ww = symmetrize(w)
WW = np.array(ww.todense())
evals = la.eigvalsh(WW)
W = WW
else:
W = w.full()[0] # moved here
evals = la.eigvals(W)
res = minimize_scalar(lag_c_loglik_ord,
0.0,
bounds=(-1.0, 1.0),
args=(self.n, e0, e1, evals),
method='bounded',
tol=epsilon)
else:
# program will crash, need to catch
print(("{0} is an unsupported method".format(methodML)))
self = None
return
self.rho = res.x[0][0]
# compute full log-likelihood, including constants
ln2pi = np.log(2.0 * np.pi)
llik = -res.fun - self.n / 2.0 * ln2pi - self.n / 2.0
self.logll = llik[0][0]
# b, residuals and predicted values
b = b0 - self.rho * b1
self.betas = np.vstack((b, self.rho)) # rho added as last coefficient
self.u = e0 - self.rho * e1
self.predy = self.y - self.u
xb = spdot(x, b)
self.predy_e = inverse_prod(w.sparse,
xb,
self.rho,
inv_method="power_exp",
threshold=epsilon)
self.e_pred = self.y - self.predy_e
# residual variance
self._cache = {}
self.sig2 = self.sig2n # no allowance for division by n-k
# information matrix
# if w should be kept sparse, how can we do the following:
a = -self.rho * W
spfill_diagonal(a, 1.0)
ai = spinv(a)
wai = spdot(W, ai)
tr1 = wai.diagonal().sum() #same for sparse and dense
wai2 = spdot(wai, wai)
tr2 = wai2.diagonal().sum()
waiTwai = spdot(wai.T, wai)
tr3 = waiTwai.diagonal().sum()
### to here
wpredy = weights.lag_spatial(w, self.predy_e)
wpyTwpy = spdot(wpredy.T, wpredy)
xTwpy = spdot(x.T, wpredy)
# order of variables is beta, rho, sigma2
v1 = np.vstack(
(xtx / self.sig2, xTwpy.T / self.sig2, np.zeros((1, self.k))))
v2 = np.vstack((xTwpy / self.sig2, tr2 + tr3 + wpyTwpy / self.sig2,
tr1 / self.sig2))
v3 = np.vstack((np.zeros(
(self.k, 1)), tr1 / self.sig2, self.n / (2.0 * self.sig2**2)))
v = np.hstack((v1, v2, v3))
self.vm1 = la.inv(v) # vm1 includes variance for sigma2
self.vm = self.vm1[:-1, :-1] # vm is for coefficients only
"""
import pandas as pd
import geopandas as gpd
import matplotlib.pyplot as plt
import seaborn
from libpysal import weights
from esda import Moran, Moran_Local
from shapely.geometry import Polygon
from pysal.lib import cg as geometry
#import splot
#from splot.esda import moran_scatterplot, plot_moran, lisa_cluster
df = gpd.read_file('/Users/heatherkay/q_research/test/help.shp')
w = weights.KNN.from_dataframe(df)
df['w_q'] = weights.lag_spatial(w,df['q'])
#can I remove rows that have no neighbours?
#new = df[df['w_q'].notna()]
#weights relationship based on distance
w_kernel = weights.distance.Kernel.from_dataframe(df)
#the kernel varies depending on the distance between sentre of polygons
w_adaptive = weights.distance.Kernel.from_dataframe(df, fixed=False, k=15)
w_adaptive.bandwidth
#but need to take curvature of the eart into account
radius = geometry.sphere.RADIUS_EARTH_MILES
w_radius_knn = weights.distance.KNN.from_dataframe(df, k=4, radius=radius)
x_names = ["NROOM","NBATH","PATIO","FIREPL","AC","GAR","AGE","LOTSZ","SQFT"]
x = np.array([db.by_col(var) for var in x_names]).T
ww = libpysal.io.open(libpysal.examples.get_path("baltim_q.gal"))
w = ww.read()
ww.close()
w_name = "baltim_q.gal"
w.transform = 'r'
n = USER.check_arrays(y, x)
n, k = x.shape
method = "full"
epsilon = 0.0000001
regimes_att = None
ylag = weights.lag_spatial(w, y)
xlag = get_x_lag(w, regimes_att)
# b0, b1, e0 and e1
xtx = spdot(x.T, x)
xtxi = la.inv(xtx)
xty = spdot(x.T, y)
xtyl = spdot(x.T, ylag)
b0 = spdot(xtxi, xty)
b1 = spdot(xtxi, xtyl)
e0 = y - spdot(x, b0)
e1 = ylag - spdot(x, b1)
methodML = method.upper()
W = w.full()[0] # moved here
def __init__(self,
y,
x,
w,
method='full',
epsilon=0.0000001,
regimes_att=None):
# set up main regression variables and spatial filters
self.y = y
if regimes_att:
self.x = x.toarray()
else:
self.x = x
self.n, self.k = self.x.shape
self.method = method
self.epsilon = epsilon
#W = w.full()[0] #wait to build pending what is needed
#Wsp = w.sparse
ylag = weights.lag_spatial(w, self.y)
xlag = self.get_x_lag(w, regimes_att)
# call minimizer using concentrated log-likelihood to get lambda
methodML = method.upper()
if methodML in ['FULL', 'LU', 'ORD']:
if methodML == 'FULL':
W = w.full()[0] # need dense here
res = minimize_scalar(err_c_loglik,
0.0,
bounds=(-1.0, 1.0),
args=(self.n, self.y, ylag, self.x, xlag,
W),
method='bounded',
tol=epsilon)
elif methodML == 'LU':
I = sp.identity(w.n)
Wsp = w.sparse # need sparse here
res = minimize_scalar(err_c_loglik_sp,
0.0,
bounds=(-1.0, 1.0),
args=(self.n, self.y, ylag, self.x, xlag,
I, Wsp),
method='bounded',
tol=epsilon)
W = Wsp
elif methodML == 'ORD':
# check on symmetry structure
if w.asymmetry(intrinsic=False) == []:
ww = symmetrize(w)
WW = np.array(ww.todense())
evals = la.eigvalsh(WW)
W = WW
else:
W = w.full()[0] # need dense here
evals = la.eigvals(W)
res = minimize_scalar(err_c_loglik_ord,
0.0,
bounds=(-1.0, 1.0),
args=(self.n, self.y, ylag, self.x, xlag,
evals),
method='bounded',
tol=epsilon)
else:
raise Exception("{0} is an unsupported method".format(method))
self.lam = res.x
# compute full log-likelihood, including constants
ln2pi = np.log(2.0 * np.pi)
llik = -res.fun - self.n / 2.0 * ln2pi - self.n / 2.0
self.logll = llik
# b, residuals and predicted values
ys = self.y - self.lam * ylag
xs = self.x - self.lam * xlag
xsxs = np.dot(xs.T, xs)
xsxsi = np.linalg.inv(xsxs)
xsys = np.dot(xs.T, ys)
b = np.dot(xsxsi, xsys)
self.betas = np.vstack((b, self.lam))
self.u = y - np.dot(self.x, b)
self.predy = self.y - self.u
# residual variance
self.e_filtered = self.u - self.lam * weights.lag_spatial(w, self.u)
self.sig2 = np.dot(self.e_filtered.T, self.e_filtered) / self.n
# variance-covariance matrix betas
varb = self.sig2 * xsxsi
# variance-covariance matrix lambda, sigma
a = -self.lam * W
spfill_diagonal(a, 1.0)
ai = spinv(a)
wai = spdot(W, ai)
tr1 = wai.diagonal().sum()
wai2 = spdot(wai, wai)
tr2 = wai2.diagonal().sum()
waiTwai = spdot(wai.T, wai)
tr3 = waiTwai.diagonal().sum()
v1 = np.vstack((tr2 + tr3, tr1 / self.sig2))
v2 = np.vstack((tr1 / self.sig2, self.n / (2.0 * self.sig2**2)))
v = np.hstack((v1, v2))
self.vm1 = np.linalg.inv(v)
# create variance matrix for beta, lambda
vv = np.hstack((varb, np.zeros((self.k, 1))))
vv1 = np.hstack((np.zeros((1, self.k)), self.vm1[0, 0] * np.ones(
(1, 1))))
self.vm = np.vstack((vv, vv1))
