def get_x_lag(self, w, regimes_att): if regimes_att: xlag = weights.lag_spatial(w, regimes_att['x']) xlag = REGI.Regimes_Frame.__init__(self, xlag, regimes_att['regimes'], constant_regi=None, cols2regi=regimes_att['cols2regi'])[0] xlag = xlag.toarray() else: xlag = weights.lag_spatial(w, self.x) return xlag
def _build_local_environment( data, groups, w=None, bandwidth=1000, function="triangular" ): """Convert observations into spatially-weighted sums. Parameters ---------- data : GeoDataFrame dataframe with local observations w : libpysal.weights object weights matrix defining the local environment Returns ------- DataFrame Spatialized data """ data = data.copy().reset_index() if data.crs.is_geographic: warnings.warn( "GeoDataFrame appears to have a geographic coordinate system and likely needs to be reprojected" ) with warnings.catch_warnings(): warnings.simplefilter("ignore") if not w: w = Kernel.from_dataframe(data, bandwidth=bandwidth, function=function) new_data = [] w = fill_diagonal(w) for y in data[groups]: new_data.append(lag_spatial(w, data[y])) new_data = pd.DataFrame(dict(zip(groups, new_data))).round(0) new_data = data.geometry.to_frame().join(new_data.reset_index()) return new_data
def _calc(self, Y, w, k): wY = weights.lag_spatial(w, Y) dx = Y[:, -1] - Y[:, 0] dy = wY[:, -1] - wY[:, 0] self.wY = wY self.Y = Y r = np.sqrt(dx * dx + dy * dy) theta = np.arctan2(dy, dx) neg = theta < 0.0 utheta = theta * (1 - neg) + neg * (2 * np.pi + theta) counts, bins = np.histogram(utheta, self.cuts) results = {} results['counts'] = counts results['theta'] = theta results['bins'] = bins results['r'] = r results['lag'] = wY results['dx'] = dx results['dy'] = dy return results
def __init__(self, y, x, w, method='full', epsilon=0.0000001): # set up main regression variables and spatial filters self.y = y self.x = x self.n, self.k = self.x.shape self.method = method self.epsilon = epsilon #W = w.full()[0] #Wsp = w.sparse ylag = weights.lag_spatial(w, y) # b0, b1, e0 and e1 xtx = spdot(self.x.T, self.x) xtxi = la.inv(xtx) xty = spdot(self.x.T, self.y) xtyl = spdot(self.x.T, ylag) b0 = spdot(xtxi, xty) b1 = spdot(xtxi, xtyl) e0 = self.y - spdot(x, b0) e1 = ylag - spdot(x, b1) methodML = method.upper() # call minimizer using concentrated log-likelihood to get rho if methodML in ['FULL', 'LU', 'ORD']: if methodML == 'FULL': W = w.full()[0] # moved here res = minimize_scalar(lag_c_loglik, 0.0, bounds=(-1.0, 1.0), args=(self.n, e0, e1, W), method='bounded', tol=epsilon) elif methodML == 'LU': I = sp.identity(w.n) Wsp = w.sparse # moved here W = Wsp res = minimize_scalar(lag_c_loglik_sp, 0.0, bounds=(-1.0, 1.0), args=(self.n, e0, e1, I, Wsp), method='bounded', tol=epsilon) elif methodML == 'ORD': # check on symmetry structure if w.asymmetry(intrinsic=False) == []: ww = symmetrize(w) WW = np.array(ww.todense()) evals = la.eigvalsh(WW) W = WW else: W = w.full()[0] # moved here evals = la.eigvals(W) res = minimize_scalar(lag_c_loglik_ord, 0.0, bounds=(-1.0, 1.0), args=(self.n, e0, e1, evals), method='bounded', tol=epsilon) else: # program will crash, need to catch print(("{0} is an unsupported method".format(methodML))) self = None return self.rho = res.x[0][0] # compute full log-likelihood, including constants ln2pi = np.log(2.0 * np.pi) llik = -res.fun - self.n / 2.0 * ln2pi - self.n / 2.0 self.logll = llik[0][0] # b, residuals and predicted values b = b0 - self.rho * b1 self.betas = np.vstack((b, self.rho)) # rho added as last coefficient self.u = e0 - self.rho * e1 self.predy = self.y - self.u xb = spdot(x, b) self.predy_e = inverse_prod(w.sparse, xb, self.rho, inv_method="power_exp", threshold=epsilon) self.e_pred = self.y - self.predy_e # residual variance self._cache = {} self.sig2 = self.sig2n # no allowance for division by n-k # information matrix # if w should be kept sparse, how can we do the following: a = -self.rho * W spfill_diagonal(a, 1.0) ai = spinv(a) wai = spdot(W, ai) tr1 = wai.diagonal().sum() #same for sparse and dense wai2 = spdot(wai, wai) tr2 = wai2.diagonal().sum() waiTwai = spdot(wai.T, wai) tr3 = waiTwai.diagonal().sum() ### to here wpredy = weights.lag_spatial(w, self.predy_e) wpyTwpy = spdot(wpredy.T, wpredy) xTwpy = spdot(x.T, wpredy) # order of variables is beta, rho, sigma2 v1 = np.vstack( (xtx / self.sig2, xTwpy.T / self.sig2, np.zeros((1, self.k)))) v2 = np.vstack((xTwpy / self.sig2, tr2 + tr3 + wpyTwpy / self.sig2, tr1 / self.sig2)) v3 = np.vstack((np.zeros( (self.k, 1)), tr1 / self.sig2, self.n / (2.0 * self.sig2**2))) v = np.hstack((v1, v2, v3)) self.vm1 = la.inv(v) # vm1 includes variance for sigma2 self.vm = self.vm1[:-1, :-1] # vm is for coefficients only
""" import pandas as pd import geopandas as gpd import matplotlib.pyplot as plt import seaborn from libpysal import weights from esda import Moran, Moran_Local from shapely.geometry import Polygon from pysal.lib import cg as geometry #import splot #from splot.esda import moran_scatterplot, plot_moran, lisa_cluster df = gpd.read_file('/Users/heatherkay/q_research/test/help.shp') w = weights.KNN.from_dataframe(df) df['w_q'] = weights.lag_spatial(w,df['q']) #can I remove rows that have no neighbours? #new = df[df['w_q'].notna()] #weights relationship based on distance w_kernel = weights.distance.Kernel.from_dataframe(df) #the kernel varies depending on the distance between sentre of polygons w_adaptive = weights.distance.Kernel.from_dataframe(df, fixed=False, k=15) w_adaptive.bandwidth #but need to take curvature of the eart into account radius = geometry.sphere.RADIUS_EARTH_MILES w_radius_knn = weights.distance.KNN.from_dataframe(df, k=4, radius=radius)
x_names = ["NROOM","NBATH","PATIO","FIREPL","AC","GAR","AGE","LOTSZ","SQFT"] x = np.array([db.by_col(var) for var in x_names]).T ww = libpysal.io.open(libpysal.examples.get_path("baltim_q.gal")) w = ww.read() ww.close() w_name = "baltim_q.gal" w.transform = 'r' n = USER.check_arrays(y, x) n, k = x.shape method = "full" epsilon = 0.0000001 regimes_att = None ylag = weights.lag_spatial(w, y) xlag = get_x_lag(w, regimes_att) # b0, b1, e0 and e1 xtx = spdot(x.T, x) xtxi = la.inv(xtx) xty = spdot(x.T, y) xtyl = spdot(x.T, ylag) b0 = spdot(xtxi, xty) b1 = spdot(xtxi, xtyl) e0 = y - spdot(x, b0) e1 = ylag - spdot(x, b1) methodML = method.upper() W = w.full()[0] # moved here
def __init__(self, y, x, w, method='full', epsilon=0.0000001, regimes_att=None): # set up main regression variables and spatial filters self.y = y if regimes_att: self.x = x.toarray() else: self.x = x self.n, self.k = self.x.shape self.method = method self.epsilon = epsilon #W = w.full()[0] #wait to build pending what is needed #Wsp = w.sparse ylag = weights.lag_spatial(w, self.y) xlag = self.get_x_lag(w, regimes_att) # call minimizer using concentrated log-likelihood to get lambda methodML = method.upper() if methodML in ['FULL', 'LU', 'ORD']: if methodML == 'FULL': W = w.full()[0] # need dense here res = minimize_scalar(err_c_loglik, 0.0, bounds=(-1.0, 1.0), args=(self.n, self.y, ylag, self.x, xlag, W), method='bounded', tol=epsilon) elif methodML == 'LU': I = sp.identity(w.n) Wsp = w.sparse # need sparse here res = minimize_scalar(err_c_loglik_sp, 0.0, bounds=(-1.0, 1.0), args=(self.n, self.y, ylag, self.x, xlag, I, Wsp), method='bounded', tol=epsilon) W = Wsp elif methodML == 'ORD': # check on symmetry structure if w.asymmetry(intrinsic=False) == []: ww = symmetrize(w) WW = np.array(ww.todense()) evals = la.eigvalsh(WW) W = WW else: W = w.full()[0] # need dense here evals = la.eigvals(W) res = minimize_scalar(err_c_loglik_ord, 0.0, bounds=(-1.0, 1.0), args=(self.n, self.y, ylag, self.x, xlag, evals), method='bounded', tol=epsilon) else: raise Exception("{0} is an unsupported method".format(method)) self.lam = res.x # compute full log-likelihood, including constants ln2pi = np.log(2.0 * np.pi) llik = -res.fun - self.n / 2.0 * ln2pi - self.n / 2.0 self.logll = llik # b, residuals and predicted values ys = self.y - self.lam * ylag xs = self.x - self.lam * xlag xsxs = np.dot(xs.T, xs) xsxsi = np.linalg.inv(xsxs) xsys = np.dot(xs.T, ys) b = np.dot(xsxsi, xsys) self.betas = np.vstack((b, self.lam)) self.u = y - np.dot(self.x, b) self.predy = self.y - self.u # residual variance self.e_filtered = self.u - self.lam * weights.lag_spatial(w, self.u) self.sig2 = np.dot(self.e_filtered.T, self.e_filtered) / self.n # variance-covariance matrix betas varb = self.sig2 * xsxsi # variance-covariance matrix lambda, sigma a = -self.lam * W spfill_diagonal(a, 1.0) ai = spinv(a) wai = spdot(W, ai) tr1 = wai.diagonal().sum() wai2 = spdot(wai, wai) tr2 = wai2.diagonal().sum() waiTwai = spdot(wai.T, wai) tr3 = waiTwai.diagonal().sum() v1 = np.vstack((tr2 + tr3, tr1 / self.sig2)) v2 = np.vstack((tr1 / self.sig2, self.n / (2.0 * self.sig2**2))) v = np.hstack((v1, v2)) self.vm1 = np.linalg.inv(v) # create variance matrix for beta, lambda vv = np.hstack((varb, np.zeros((self.k, 1)))) vv1 = np.hstack((np.zeros((1, self.k)), self.vm1[0, 0] * np.ones( (1, 1)))) self.vm = np.vstack((vv, vv1))