def contiguity_from_shapefile(shapefile, criteria='rook'):
"""
Create a spatial weights object based on a contiguity criteria from a
shapefile.
Produces a "*.gal" file in the directory of the shapefile
Argument
--------
shapefile: string with full path to shapefile
criteria: string for type of contiguity ['rook'|'queen']
Returns
-------
cards: nx1 numpy array with the number of neighbors for each element based
on criterion
"""
if criteria == 'rook':
PS.rook_from_shapefile(shapefile)
abb = 'r'
else:
PS.queen_from_shapefile(shapefile)
abb = 'q'
cards = NP.array(w.cardinalities.values())
cards.shape = (len(cards),1)
galfile = shapefile.split(".")[0] + "_" + abb + ".gal"
gal = PS.open(galfile,'w')
gal.write(w)
gal.close()
return cards
def generateWeightsUsingShapefile(shapeFilePath,
idVariable=None,
weights=None,
kind="queen",
k=None,
binary=False):
# use weights from shapefile for purely geographic
w = None
if weights == None:
if kind == "queen":
w = pysal.queen_from_shapefile(shapeFilePath,
idVariable=idVariable)
if kind == "rook":
w = pysal.rook_from_shapefile(shapeFilePath, idVariable=idVariable)
if kind == "knn" and type(k) == int:
w = pysal.knnW_from_shapefile(shapefile=shapeFilePath,
k=k,
idVariable=idVariable)
if kind == "band":
threshold = pysal.min_threshold_dist_from_shapefile(
shapeFilePath, idVariable=idVariable)
if binary == True:
w = pysal.weights.DistanceBand.from_shapefile(
shapeFilePath,
threshold=threshold,
binary=True,
idVariable=idVariable)
else:
w = pysal.threshold_continuousW_from_shapefile(
shapefile=shapeFilePath,
threshold=threshold,
idVariable=idVariable)
if kind == "kernel":
w = pysal.adaptive_kernelW_from_shapefile(shapeFilePath,
diagonal=True,
k=5,
idVariable=idVariable)
# else use user defined weights to create "space" instead of "place"
else:
if kind == "rook":
w = pysal.rook_from_shapefile(shapeFilePath, idVariable=idVariable)
if kind == "knn":
w = pysal.knnW_from_shapefile(shapeFilePath,
k=k,
idVariable=idVariable)
else:
w = pysal.queen_from_shapefile(shapeFilePath,
idVariable=idVariable)
neighbors = w.neighbor_offsets
w = pysal.W(neighbors, weights=weights)
# row standardize the matrix. better to do it here and use it somewhere else.
w.transform = 'r'
if binary == True:
w.transform = 'b'
return w
def south(df=False):
"""
Sets up the data for the US southern counties example.
Returns
-------
dictionary or (dictionary, dataframe), where the dictionary is keyed on:
X : Data from southern counties, columns "GI89", "BLK90", "HR90"
Y : Outcome variate, "DNL90"
Z : upper-level variate, the state average "FH90"
W : queen weights matrix between counties
M : queen matrix between states
membership : membership vector relating counties to their states
and the dataframe contains the raw dataset
"""
import geopandas
data = geopandas.read_file(pysal.examples.get_path('south.shp'))
data = data[data.STATE_NAME != 'District of Columbia']
X = data[['GI89', 'BLK90', 'HR90']].values
N = X.shape[0]
Z = data.groupby('STATE_NAME')['FH90'].mean()
Z = Z.values.reshape(-1, 1)
J = Z.shape[0]
Y = data.DNL90.values.reshape(-1, 1)
W2 = pysal.queen_from_shapefile(pysal.examples.get_path('us48.shp'),
idVariable='STATE_NAME')
W2 = pysal.w_subset(
W2,
ids=data.STATE_NAME.unique().tolist()) #only keep what's in the data
W1 = pysal.queen_from_shapefile(pysal.examples.get_path('south.shp'),
idVariable='FIPS')
W1 = pysal.w_subset(
W1, ids=data.FIPS.tolist()) #again, only keep what's in the data
W1.transform = 'r'
W2.transform = 'r'
membership = data.STATE_NAME.apply(lambda x: W2.id_order.index(x)).values
d = {'X': X, 'Y': Y, 'Z': Z, 'W': W1, 'M': W2, 'membership': membership}
if df:
return d, data
else:
return d
def test_spatial(self):
w = pysal.queen_from_shapefile(pysal.examples.get_path('columbus.shp'))
reg = TSLS_Regimes(self.y,
self.x,
self.yd,
self.q,
self.regimes,
spat_diag=True,
w=w,
regime_err_sep=False)
betas = np.array([[80.23408166], [5.48218125], [82.98396737],
[0.49775429], [-3.72663211], [-1.27451485]])
np.testing.assert_allclose(reg.betas, betas, RTOL)
vm = np.array([[ 495.16048523, 78.89742341, 0. , 0. ,\
-47.12971066, 0. ],\
[ 78.89742341, 64.58341083, 0. , 0. ,\
-30.74878934, 0. ],\
[ 0. , 0. , 732.05899155, 11.21128921,\
0. , -20.96804956],\
[ 0. , 0. , 11.21128921, 7.48778398,\
0. , -2.56752553],\
[ -47.12971066, -30.74878934, 0. , 0. ,\
14.89456384, 0. ],\
[ 0. , 0. , -20.96804956, -2.56752553,\
0. , 1.3154267 ]])
np.testing.assert_allclose(reg.vm, vm, RTOL)
ak_test = np.array([0.69774552, 0.40354227])
np.testing.assert_allclose(reg.ak_test, ak_test, RTOL)
def processAlgorithm(self, progress):
field = self.getParameterValue(self.FIELD)
field = field[0:10] # try to handle Shapefile field length limit
filename = self.getParameterValue(self.INPUT)
layer = dataobjects.getObjectFromUri(filename)
filename = dataobjects.exportVectorLayer(layer)
contiguity = self.getParameterValue(self.CONTIGUITY)
if self.CONTIGUITY_OPTIONS[contiguity] == 'queen':
print 'INFO: Getis and Ord\'s G using queen contiguity'
w = pysal.queen_from_shapefile(filename)
elif self.CONTIGUITY_OPTIONS[contiguity] == 'rook':
print 'INFO: Getis and Ord\'s G using rook contiguity'
w = pysal.rook_from_shapefile(filename)
f = pysal.open(
pysal.examples.get_path(filename.replace('.shp', '.dbf')))
y = np.array(f.by_col[str(field)])
g = pysal.G(y, w, permutations=999)
self.setOutputValue(self.G, g.G)
print "Getis and Ord's G: %f" % (g.G)
print "expected value: %f" % (g.EG)
print "p_norm: %f" % (g.p_norm)
print "p_sim: %f" % (g.p_sim)
print "INFO: p values smaller than 0.05 indicate spatial autocorrelation that is significant at the 5% level."
print "z_norm: %f" % (g.z_norm)
print "z_sim: %f" % (g.z_sim)
print "INFO: z values greater than 1.96 or smaller than -1.96 indicate spatial autocorrelation that is significant at the 5% level."
def contiguity_from_shapefile(shapefile, criteria='rook'):
print shapefile
if criteria == 'rook':
PS.rook_from_shapefile(shapefile)
abb = 'r'
else:
PS.queen_from_shapefile(shapefile)
abb = 'q'
cards = NP.array(w.cardinalities.values())
cards.shape = (len(cards),1)
galfile = shapefile.split(".")[0] + "_" + abb + ".gal"
gal = PS.open(galfile,'w')
gal.write(w)
gal.close()
return cards
def test_names(self):
w = pysal.queen_from_shapefile(pysal.examples.get_path('columbus.shp'))
gwk = pysal.kernelW_from_shapefile(pysal.examples.get_path('columbus.shp'),k=5,function='triangular', fixed=False)
name_x = ['inc']
name_y = 'crime'
name_yend = ['hoval']
name_q = ['discbd']
name_w = 'queen'
name_gwk = 'k=5'
name_ds = 'columbus'
reg = TSLS(self.y, self.X, self.yd, self.q,
spat_diag=True, w=w, robust='hac', gwk=gwk,
name_x=name_x, name_y=name_y, name_q=name_q, name_w=name_w,
name_yend=name_yend, name_gwk=name_gwk, name_ds=name_ds)
betas = np.array([[ 88.46579584], [ 0.5200379 ], [ -1.58216593]])
np.testing.assert_array_almost_equal(reg.betas, betas, 7)
vm = np.array([[ 225.0795089 , 17.11660041, -12.22448566],
[ 17.67097154, 2.47483461, -1.4183641 ],
[ -12.45093722, -1.40495464, 0.8700441 ]])
np.testing.assert_array_almost_equal(reg.vm, vm, 7)
self.assertListEqual(reg.name_x, ['CONSTANT']+name_x)
self.assertListEqual(reg.name_yend, name_yend)
self.assertListEqual(reg.name_q, name_q)
self.assertEqual(reg.name_y, name_y)
self.assertEqual(reg.name_w, name_w)
self.assertEqual(reg.name_gwk, name_gwk)
self.assertEqual(reg.name_ds, name_ds)
def test_d3viz():
import pysal
shp = pysal.open(pysal.examples.get_path('NAT.shp'),'r')
dbf = pysal.open(pysal.examples.get_path('NAT.dbf'),'r')
#shp = pysal.open('/Users/xun/github/PySAL-Viz/test_data/man_road.shp','r')
#dbf = pysal.open('/Users/xun/github/PySAL-Viz/test_data/man_road.dbf','r')
import d3viz
d3viz.setup()
d3viz.init_map(shp)
d3viz.show_map(shp)
d3viz.scatter_plot(shp, field_x="HR90", field_y="HC90")
w = pysal.queen_from_shapefile(pysal.examples.get_path('NAT.shp'))
d3viz.moran_scatter_plot(shp, dbf, "HR90", w)
d3viz.quantile_map(shp, "HR90", 5)
d3viz.quantile_map(shp, "HR90", 5, basemap="leaflet")
import numpy as np
y = np.array(dbf.by_col["HR90"])
lm = pysal.Moran_Local(y, w)
d3viz.lisa_map(shp, "HR90", lm)
def __init__(self, filepath, outname, namelist, idlist, nb="queen", factor=2):
"""
Initiation of modules
"""
f=Dbf(filepath+".dbf")
#Create mapping of locations to row id
self.locations = dict()
i=0
for row in f:
uid=unicode("".join([row[k] for k in idlist]))
locnames = unicode(", ".join([row[k] for k in namelist]),"ascii","ignore")
self.locations[i] = {outname:locnames,"id":uid}
i+=1
self.__dict__[outname]= self.locations
self.outname = outname
#Get Neightbor weights by queen, rook, knn, distance
if nb=="queen":
self.wt = pysal.queen_from_shapefile(filepath+".shp")
elif nb=="rook":
self.wt = pysal.rook_from_shapefile(filepath+".shp")
elif nb=="knn":
self.wt = pysal.knnW_from_shapefile(filepath+".shp", k=factor)
elif nb=="distance":
self.wt = pysal.threshold_binaryW_from_shapefile(filepath+".shp",k)
#Create dictionary of neighbors for each region
self.neighbors ={}
for i,j in enumerate(self.wt):
self.neighbors[self.locations[i]["id"]] = {self.outname:self.locations[i][self.outname]
,"neighbors":dict([[self.locations[k]["id"],self.locations[k][self.outname]] for k in j.keys()])}
def compute(self, vlayer, tfield, idvar, matType):
vlayer = qgis.utils.iface.activeLayer()
idvar = self.idVariable.currentText()
# print type(idvar)
tfield = self.inField.currentText()
# print type(tfield)
provider = vlayer.dataProvider()
allAttrs = provider.attributeIndexes()
caps = vlayer.dataProvider().capabilities()
if caps & QgsVectorDataProvider.AddAttributes:
TestField = idvar[:5] + "_qrr"
res = vlayer.dataProvider().addAttributes(
[QgsField(TestField, QVariant.Double)])
wp = str(self.dic[str(self.inShape.currentText())])
if matType == "Rook":
w = py.rook_from_shapefile(wp, idVariable=unicode(idvar))
else:
w = py.queen_from_shapefile(wp, idVariable=unicode(idvar))
w1 = wp[:-3] + "dbf"
db = py.open(w1)
y = np.array(db.by_col[unicode(tfield)])
np.random.seed(12345)
gc = py.Geary(y, w)
#lm=py.Moran_Local(y,w)
#l=lm.p_sim
gg = gc.C
self.SAresult.setText("The Global Geary's C index is " + str(gg))
def setUp(self):
db=pysal.open(pysal.examples.get_path("columbus.dbf"),"r")
y = np.array(db.by_col("CRIME"))
self.y = np.reshape(y, (49,1))
X = []
X.append(db.by_col("HOVAL"))
X.append(db.by_col("INC"))
self.X = np.array(X).T
X2 = []
X2.append(db.by_col("INC"))
self.X2 = np.array(X2).T
yd = []
yd.append(db.by_col("HOVAL"))
self.yd = np.array(yd).T
q = []
q.append(db.by_col("DISCBD"))
self.q = np.array(q).T
self.w = pysal.queen_from_shapefile(pysal.examples.get_path("columbus.shp"))
self.w.transform = 'r'
self.r_var = 'NSA'
self.regimes = db.by_col(self.r_var)
#Artficial:
n = 256
self.n2 = n/2
self.x_a1 = np.random.uniform(-10,10,(n,1))
self.x_a2 = np.random.uniform(1,5,(n,1))
self.q_a = self.x_a2 + np.random.normal(0,1,(n,1))
self.x_a = np.hstack((self.x_a1,self.x_a2))
self.y_a = np.dot(np.hstack((np.ones((n,1)),self.x_a)),np.array([[1],[0.5],[2]])) + np.random.normal(0,1,(n,1))
latt = int(np.sqrt(n))
self.w_a = pysal.lat2W(latt,latt)
self.w_a.transform='r'
self.regi_a = [0]*(n/2) + [1]*(n/2)
self.w_a1 = pysal.lat2W(latt/2,latt)
self.w_a1.transform='r'
def processAlgorithm(self, progress):
field = self.getParameterValue(self.FIELD)
field = field[0:10] # try to handle Shapefile field length limit
filename = self.getParameterValue(self.INPUT)
layer = dataobjects.getObjectFromUri(filename)
filename = dataobjects.exportVectorLayer(layer)
contiguity = self.getParameterValue(self.CONTIGUITY)
if contiguity == 0: # queen
print 'INFO: Moran\'s using queen contiguity'
w=pysal.queen_from_shapefile(filename)
else: # 1 for rook
print 'INFO: Moran\'s using rook contiguity'
w=pysal.rook_from_shapefile(filename)
f = pysal.open(filename.replace('.shp','.dbf'))
y=np.array(f.by_col[str(field)])
m = pysal.Moran(y,w,transformation = "r", permutations = 999)
self.setOutputValue(self.I,m.I)
print "Moran's I: %f" % (m.I)
print "INFO: Moran's I values range from -1 (indicating perfect dispersion) to +1 (perfect correlation). Values close to -1/(n-1) indicate a random spatial pattern."
print "p_norm: %f" % (m.p_norm)
print "p_rand: %f" % (m.p_rand)
print "p_sim: %f" % (m.p_sim)
print "INFO: p values smaller than 0.05 indicate spatial autocorrelation that is significant at the 5% level."
print "z_norm: %f" % (m.z_norm)
print "z_rand: %f" % (m.z_rand)
print "z_sim: %f" % (m.z_sim)
print "INFO: z values greater than 1.96 or smaller than -1.96 indicate spatial autocorrelation that is significant at the 5% level."
def read_files(filepath, **kwargs):
"""
Reads a dbf/shapefile pair, squashing geometries into a "geometry" column.
"""
#keyword arguments wrapper will strip all around dbf2df's required arguments
geomcol = kwargs.pop('geomcol', 'geometry')
weights = kwargs.pop('weights', '')
dbf_path, shp_path = _pairpath(filepath)
df = dbf2df(dbf_path, **kwargs)
df[geomcol] = shp2series(shp_path)
if weights != '' and isinstance(weights, str):
if weights.lower() in ['rook', 'queen']:
if weights.lower() == 'rook':
df.W = ps.rook_from_shapefile(shp_path)
else:
df.W = ps.queen_from_shapefile(shp_path)
else:
try:
W_path = os.path.splitext(dbf_path)[0] + '.' + weights
df.W = ps.open(W_path).read()
except IOError:
print('Weights construction failed! Passing on weights')
return df
def compute(self, vlayer, tfield, idvar,matType):
vlayer=qgis.utils.iface.activeLayer()
idvar=self.idVariable.currentText()
# print type(idvar)
tfield=self.inField.currentText()
# print type(tfield)
provider=vlayer.dataProvider()
allAttrs=provider.attributeIndexes()
caps=vlayer.dataProvider().capabilities()
if caps & QgsVectorDataProvider.AddAttributes:
TestField = idvar[:5]+"_qrr"
res = vlayer.dataProvider().addAttributes([QgsField(TestField, QVariant.Double)])
wp=str(self.dic[str(self.inShape.currentText())])
if matType == "Rook":
w = py.rook_from_shapefile(wp, idVariable=unicode(idvar))
else:
w = py.queen_from_shapefile(wp, idVariable=unicode(idvar))
w1=wp[:-3]+"dbf"
db=py.open(w1)
y=np.array(db.by_col[unicode(tfield)])
np.random.seed(12345)
gc = py.Geary(y,w)
#lm=py.Moran_Local(y,w)
#l=lm.p_sim
gg = gc.C
self.SAresult.setText("The Global Geary's C index is " + str(gg))
def accept(self):
# look for open shapefile layers, if none
if len(self.comboBox.currentText()) == 0 and self.lineEdit.text() == "":
QMessageBox.information(self, self.tr("Weights from Shapefile"), self.tr("Please select input polygon vector layer"))
# elif self.outShape.text() == "":
# QMessageBox.information(self, self.tr("Sum Line Lengths In Polyons"), self.tr("Please specify output shapefile"))
else:
# run the PySAL logic
if str(self.comboBox.currentText()) == "":
shapefile = str(self.shapefile)
else:
shapefile = str(self.d[str(self.comboBox.currentText())])
if self.radioButton.isChecked():
w = PS.queen_from_shapefile(shapefile)
abb = 'q'
else:
w = PS.rook_from_shapefile(shapefile)
abb = 'r'
cards = NP.array(w.cardinalities.values())
cards.shape = (len(cards),1)
galfile = shapefile.split(".")[0] + "_" + abb + ".gal"
gal = PS.open(galfile,'w')
gal.write(w)
gal.close()
QDialog.accept(self)
def test2():
#d6cd52286e5d3e9e08a5a42489180df3.shp
import pysal
shp = pysal.open('../www/tmp/d6cd52286e5d3e9e08a5a42489180df3.shp')
dbf = pysal.open('../www/tmp/d6cd52286e5d3e9e08a5a42489180df3.dbf')
w = pysal.queen_from_shapefile('../www/tmp/d6cd52286e5d3e9e08a5a42489180df3.shp')
#d3viz.moran_scatter_plot(shp, dbf, "dog_cnt", w)
import numpy as np
y = np.array(dbf.by_col["dog_cnt"])
lm = pysal.Moran_Local(y, w)
for i,j in enumerate(lm.q):
if lm.p_sim[i] >= 0.05:
print 0
else:
print j
import d3viz
d3viz.setup()
d3viz.show_map(shp)
d3viz.scatter_plot(shp, ["dog_cnt","home_cnt"])
d3viz.lisa_map(shp, "dog_cnt", lm)
def read_files(filepath, **kwargs):
"""
Reads a dbf/shapefile pair, squashing geometries into a "geometry" column.
"""
#keyword arguments wrapper will strip all around dbf2df's required arguments
geomcol = kwargs.pop('geomcol', 'geometry')
weights = kwargs.pop('weights', '')
dbf_path, shp_path = _pairpath(filepath)
df = dbf2df(dbf_path, **kwargs)
df[geomcol] = shp2series(shp_path)
if weights != '' and isinstance(weights, str):
if weights.lower() in ['rook', 'queen']:
if weights.lower() == 'rook':
df.W = ps.rook_from_shapefile(shp_path)
else:
df.W = ps.queen_from_shapefile(shp_path)
else:
try:
W_path = os.path.splitext(dbf_path)[0] + '.' + weights
df.W = ps.open(W_path).read()
except IOError:
print('Weights construction failed! Passing on weights')
return df
def getNeighbors(shapefileDir, indexMapping):
# map by blockId
neighborsMap = {}
# map by Index
neighborsMapByIndex = {}
sf = shapefile.Reader(shapefileDir)
records = sf.records()
# choose the "queen" neighbors. This means shapes are neighbors as long as they share a vertex.
# alternative is the "rook", where shapes must share an edge.
w = ps.queen_from_shapefile(shapefileDir + ".shp")
N = w.n
for i in range(N):
# blockId is the field in index 4
# blockId = int(records[i][4])
blockId = int(records[i][3])
index = indexMapping[blockId]
# this var is a map containing the neighbors of block i, where the key is the neighbor, and value is the weight
neighbors = w[i]
# map everything by blockIds instead of indices in this list
# since this ordering is different from the blocksList
neighborList = []
neighborIndexList = []
for n in neighbors.keys():
# neighborId = int(records[n][4])
neighborId = int(records[n][3])
neighborIndex = indexMapping[neighborId]
neighborList.append(neighborId)
neighborIndexList.append(neighborIndex)
neighborsMap[blockId] = neighborList
neighborsMapByIndex[index] = neighborIndexList
return neighborsMap, neighborsMapByIndex
def test_spatial(self):
X = np.array(self.db.by_col("INC"))
X = np.reshape(X, (49, 1))
X = SP.csr_matrix(X)
yd = np.array(self.db.by_col("CRIME"))
yd = np.reshape(yd, (49, 1))
q = np.array(self.db.by_col("DISCBD"))
q = np.reshape(q, (49, 1))
w = pysal.queen_from_shapefile(pysal.examples.get_path('columbus.shp'))
reg = GM_Lag(self.y, X, yd, q, spat_diag=True, w=w)
betas = np.array([[5.46344924e+01], [4.13301682e-01],
[-5.92637442e-01], [-7.40490883e-03]])
np.testing.assert_allclose(reg.betas, betas, RTOL)
vm = np.array([[
4.45202654e+02, -1.50290275e+01, -6.36557072e+00, -5.71403440e-03
], [-1.50290275e+01, 5.93124683e-01, 2.19169508e-01, -6.70675916e-03],
[
-6.36557072e+00, 2.19169508e-01, 1.06577542e-01,
-2.96533875e-03
],
[
-5.71403440e-03, -6.70675916e-03, -2.96533875e-03,
1.15655425e-03
]])
np.testing.assert_allclose(reg.vm, vm, RTOL)
ak_test = np.array([2.52597326, 0.11198567])
np.testing.assert_allclose(reg.ak_test, ak_test, RTOL)
def accept(self):
# look for open shapefile layers, if none
if len(self.comboBox.currentText()) == 0 and self.lineEdit.text(
) == "":
QMessageBox.information(
self, self.tr("Weights from Shapefile"),
self.tr("Please select input polygon vector layer"))
# elif self.outShape.text() == "":
# QMessageBox.information(self, self.tr("Sum Line Lengths In Polyons"), self.tr("Please specify output shapefile"))
else:
# run the PySAL logic
if str(self.comboBox.currentText()) == "":
shapefile = str(self.shapefile)
else:
shapefile = str(self.d[str(self.comboBox.currentText())])
if self.radioButton.isChecked():
w = PS.queen_from_shapefile(shapefile)
abb = 'q'
else:
w = PS.rook_from_shapefile(shapefile)
abb = 'r'
cards = NP.array(w.cardinalities.values())
cards.shape = (len(cards), 1)
galfile = shapefile.split(".")[0] + "_" + abb + ".gal"
gal = PS.open(galfile, 'w')
gal.write(w)
gal.close()
QDialog.accept(self)
def setUp(self):
db=pysal.open(pysal.examples.get_path("columbus.dbf"),"r")
y = np.array(db.by_col("CRIME"))
self.y = np.reshape(y, (49,1))
X = []
X.append(db.by_col("HOVAL"))
X.append(db.by_col("INC"))
self.X = np.array(X).T
X2 = []
X2.append(db.by_col("INC"))
self.X2 = np.array(X2).T
yd = []
yd.append(db.by_col("HOVAL"))
self.yd = np.array(yd).T
q = []
q.append(db.by_col("DISCBD"))
self.q = np.array(q).T
self.w = pysal.queen_from_shapefile(pysal.examples.get_path("columbus.shp"))
self.w.transform = 'r'
self.r_var = 'NSA'
self.regimes = db.by_col(self.r_var)
#Artficial:
n = 256
self.n2 = n/2
self.x_a1 = np.random.uniform(-10,10,(n,1))
self.x_a2 = np.random.uniform(1,5,(n,1))
self.q_a = self.x_a2 + np.random.normal(0,1,(n,1))
self.x_a = np.hstack((self.x_a1,self.x_a2))
self.y_a = np.dot(np.hstack((np.ones((n,1)),self.x_a)),np.array([[1],[0.5],[2]])) + np.random.normal(0,1,(n,1))
latt = int(np.sqrt(n))
self.w_a = pysal.lat2W(latt,latt)
self.w_a.transform='r'
self.regi_a = [0]*(n/2) + [1]*(n/2)
self.w_a1 = pysal.lat2W(latt/2,latt)
self.w_a1.transform='r'
def get(self,shpName='',width=0,height=0):
print shpName,width,height
if not shpName in self.SHPS:
return self.index()
shp = pysal.open(self.SHPS[shpName])
W = None
if 'w' in self.request.GET:
wtype = self.request.GET['w']
if wtype.lower() == 'rook':
W = pysal.rook_from_shapefile(self.SHPS[shpName])
elif wtype.lower() == 'queen':
W = pysal.queen_from_shapefile(self.SHPS[shpName])
else:
try:
k = int(wtype)
W = pysal.knnW_from_shapefile(self.SHPS[shpName],k)
except:
print "No valid W"
print shp
if width and height:
width=int(width)
height=int(height)
if W:
return self.write({'len':len(shp), 'polygons':shift_scale_shp(shp,width,height),'width':width,'height':height,'W':W.neighbors})
else:
return self.write({'len':len(shp), 'polygons':shift_scale_shp(shp,width,height),'width':width,'height':height,'W':'null'})
return self.write({'len':len(shp)})
def setUp(self):
self.w = pysal.queen_from_shapefile(pysal.examples.get_path("columbus.shp"))
self.w.transform = "r"
self.db = pysal.open(pysal.examples.get_path("columbus.dbf"), "r")
y = np.array(self.db.by_col("CRIME"))
self.y = np.reshape(y, (49, 1))
self.r_var = "NSA"
self.regimes = self.db.by_col(self.r_var)
def setUp(self):
self.w = pysal.queen_from_shapefile(pysal.examples.get_path("columbus.shp"))
self.w.transform = 'r'
self.db = pysal.open(pysal.examples.get_path("columbus.dbf"), 'r')
y = np.array(self.db.by_col("CRIME"))
self.y = np.reshape(y, (49,1))
self.r_var = 'NSA'
self.regimes = self.db.by_col(self.r_var)
def get_weight_matrix(self, array, rook=False, shpfile=None):
"""Return the spatial weight matrix based on pysal functionalities
Keyword arguments:
array Numpy array with inventory values.
rook Boolean to select spatial weights matrix as rook or
queen case.
shpfile Name of file used to setup weight matrix.
"""
# Get case name.
if rook:
case = 'rook'
else:
case = 'queen'
# Get grid dimension.
dim = array.shape
if self.sptype == 'vector':
try:
# Create weights based on shapefile topology using defined key.
if shpfile is None:
shpfile = self.invfile
# Differentiat between rook and queen's case.
if rook:
w = pysal.rook_from_shapefile(shpfile, self.invcol)
else:
w = pysal.queen_from_shapefile(shpfile, self.invcol)
except:
msg = "Couldn't build spatial weight matrix for vector "
"inventory <%s>" % (self.name)
raise RuntimeError(msg)
# Match weight index to inventory array index.
w.id_order = list(self.inv_index)
logger.info("Weight matrix in %s's case successfully calculated "
"for vector dataset" % case)
elif self.sptype == 'raster':
try:
# Construct weight matrix in input grid size.
w = pysal.lat2W(*dim, rook=rook)
except:
msg = "Couldn't build spatial weight matrix for raster "
"inventory <%s>" % (self.name)
raise RuntimeError(msg)
logger.info("Weight matrix in %s's case successfully calculated "
"for raster dataset" % case)
# Print imported raster summary.
print("[ WEIGHT NUMBER ] = ", w.n)
print("[ MIN NEIGHBOR ] = ", w.min_neighbors)
print("[ MAX NEIGHBOR ] = ", w.max_neighbors)
print("[ ISLANDS ] = ", *w.islands)
print("[ HISTOGRAM ] = ", *w.histogram)
self._Inventory__modmtime()
return(w)
def test_names(self):
w = pysal.queen_from_shapefile(pysal.examples.get_path('columbus.shp'))
gwk = pysal.kernelW_from_shapefile(
pysal.examples.get_path('columbus.shp'),
k=5,
function='triangular',
fixed=False)
name_x = ['inc']
name_y = 'crime'
name_yend = ['hoval']
name_q = ['discbd']
name_w = 'queen'
name_gwk = 'k=5'
name_ds = 'columbus'
name_regimes = 'nsa'
reg = TSLS_Regimes(self.y,
self.x,
self.yd,
self.q,
self.regimes,
regime_err_sep=False,
spat_diag=True,
w=w,
robust='hac',
gwk=gwk,
name_regimes=name_regimes,
name_x=name_x,
name_y=name_y,
name_q=name_q,
name_w=name_w,
name_yend=name_yend,
name_gwk=name_gwk,
name_ds=name_ds)
betas = np.array([[80.23408166], [5.48218125], [82.98396737],
[0.49775429], [-3.72663211], [-1.27451485]])
np.testing.assert_allclose(reg.betas, betas, RTOL)
vm = np.array([[ 522.75813101, 120.64940697, -15.60303241, -0.976389 ,\
-67.15556574, 0.64553579],\
[ 122.83491674, 122.62303068, -5.52270916, 0.05023488,\
-57.89404902, 0.15750428],\
[ 0.1983661 , -0.03539147, 335.24731378, 17.40764168,\
-0.26447114, -14.3375455 ],\
[ -0.13612426, -0.43622084, 18.46644989, 2.70320508,\
0.20398876, -1.31821991],\
[ -68.0704928 , -58.03685405, 2.66225388, 0.00323082,\
27.68512974, -0.08124602],\
[ -0.08001296, 0.13575504, -14.6998294 , -1.28225201,\
-0.05193056, 0.79845124]])
np.testing.assert_allclose(reg.vm, vm, RTOL)
self.assertEqual(reg.name_x,
['0_CONSTANT', '0_inc', '1_CONSTANT', '1_inc'])
self.assertEqual(reg.name_yend, ['0_hoval', '1_hoval'])
self.assertEqual(reg.name_q, ['0_discbd', '1_discbd'])
self.assertEqual(reg.name_y, name_y)
self.assertEqual(reg.name_w, name_w)
self.assertEqual(reg.name_gwk, name_gwk)
self.assertEqual(reg.name_ds, name_ds)
self.assertEqual(reg.name_regimes, name_regimes)
def neighbors_pysal(blocks, d):
# uses pysal to get all touching neighbors for each shapefile. w.neighbors is the dictionary
# https://stackoverflow.com/a/27993912/5272945
print('calculating queen neighbors')
w = pysal.queen_from_shapefile(blocks, idVariable='GEOID10')
neighbors_dict = w.neighbors
write_csv(neighbors_dict, d)
def calc_shp_adjacencies(shp, sort_id=None):
if sort_id:
adj=ps.queen_from_shapefile(shp,sort_id)
else:
adj=ps.queen_from_shapefile(shp)
sort_order=sorted(adj.id_order)
neigh_dict=adj.neighbors
neigh_list_nest=[neigh_dict[i] for i in sort_order]
neigh_list_flat=[k for sub in neigh_list_nest for k in sorted(sub)]
neigh_id_list=[sort_order.index(X)+1 for X in neigh_list_flat]
num_neigh_list=[len(x) for x in neigh_list_nest]
return {'num_neigh':num_neigh_list, 'neigh_id':neigh_id_list,'shp':shp}
def test_names(self):
X = np.array(self.db.by_col("INC"))
X = np.reshape(X, (49, 1))
X = SP.csr_matrix(X)
yd = np.array(self.db.by_col("CRIME"))
yd = np.reshape(yd, (49, 1))
q = np.array(self.db.by_col("DISCBD"))
q = np.reshape(q, (49, 1))
w = pysal.queen_from_shapefile(pysal.examples.get_path("columbus.shp"))
gwk = pysal.kernelW_from_shapefile(
pysal.examples.get_path("columbus.shp"), k=5, function="triangular", fixed=False
)
name_x = ["inc"]
name_y = "crime"
name_yend = ["crime"]
name_q = ["discbd"]
name_w = "queen"
name_gwk = "k=5"
name_ds = "columbus"
reg = GM_Lag(
self.y,
X,
yd,
q,
spat_diag=True,
w=w,
robust="hac",
gwk=gwk,
name_x=name_x,
name_y=name_y,
name_q=name_q,
name_w=name_w,
name_yend=name_yend,
name_gwk=name_gwk,
name_ds=name_ds,
)
betas = np.array([[5.46344924e01], [4.13301682e-01], [-5.92637442e-01], [-7.40490883e-03]])
np.testing.assert_array_almost_equal(reg.betas, betas, 7)
vm = np.array(
[
[5.70817052e02, -1.83655385e01, -8.36602575e00, 2.37538877e-02],
[-1.85224661e01, 6.53311383e-01, 2.84209566e-01, -6.47694160e-03],
[-8.31105622e00, 2.78772694e-01, 1.38144928e-01, -3.98175246e-03],
[2.66662466e-02, -6.23783104e-03, -4.11092891e-03, 1.10936528e-03],
]
)
np.testing.assert_array_almost_equal(reg.vm, vm, 6)
self.assertListEqual(reg.name_x, ["CONSTANT"] + name_x)
name_yend.append("W_crime")
self.assertListEqual(reg.name_yend, name_yend)
name_q.extend(["W_inc", "W_discbd"])
self.assertListEqual(reg.name_q, name_q)
self.assertEqual(reg.name_y, name_y)
self.assertEqual(reg.name_w, name_w)
self.assertEqual(reg.name_gwk, name_gwk)
self.assertEqual(reg.name_ds, name_ds)
def test_names(self):
X = np.array(self.db.by_col("INC"))
X = np.reshape(X, (49, 1))
X = SP.csr_matrix(X)
yd = np.array(self.db.by_col("CRIME"))
yd = np.reshape(yd, (49, 1))
q = np.array(self.db.by_col("DISCBD"))
q = np.reshape(q, (49, 1))
w = pysal.queen_from_shapefile(pysal.examples.get_path('columbus.shp'))
gwk = pysal.kernelW_from_shapefile(
pysal.examples.get_path('columbus.shp'),
k=5,
function='triangular',
fixed=False)
name_x = ['inc']
name_y = 'crime'
name_yend = ['crime']
name_q = ['discbd']
name_w = 'queen'
name_gwk = 'k=5'
name_ds = 'columbus'
reg = GM_Lag(self.y,
X,
yd,
q,
spat_diag=True,
w=w,
robust='hac',
gwk=gwk,
name_x=name_x,
name_y=name_y,
name_q=name_q,
name_w=name_w,
name_yend=name_yend,
name_gwk=name_gwk,
name_ds=name_ds)
betas = np.array([[5.46344924e+01], [4.13301682e-01],
[-5.92637442e-01], [-7.40490883e-03]])
np.testing.assert_allclose(reg.betas, betas, RTOL)
vm = np.array([
[5.70817052e+02, -1.83655385e+01, -8.36602575e+00, 2.37538877e-02],
[-1.85224661e+01, 6.53311383e-01, 2.84209566e-01, -6.47694160e-03],
[-8.31105622e+00, 2.78772694e-01, 1.38144928e-01, -3.98175246e-03],
[2.66662466e-02, -6.23783104e-03, -4.11092891e-03, 1.10936528e-03]
])
np.testing.assert_allclose(reg.vm, vm, RTOL)
self.assertListEqual(reg.name_x, ['CONSTANT'] + name_x)
name_yend.append('W_crime')
self.assertListEqual(reg.name_yend, name_yend)
name_q.extend(['W_inc', 'W_discbd'])
self.assertListEqual(reg.name_q, name_q)
self.assertEqual(reg.name_y, name_y)
self.assertEqual(reg.name_w, name_w)
self.assertEqual(reg.name_gwk, name_gwk)
self.assertEqual(reg.name_ds, name_ds)
def estimate_elasticity(self, zones):
dummies = pd.get_dummies(zones.county)
zones = pd.concat([zones, dummies], axis=1)
zones['avg_far'] = self.buildings_far.groupby('zone_id').far.mean() #use far_x because Xavier's code adds far to buildings
#zones = zones[zones.residential_sqft_zone>0]
#wrook = py.queen_from_shapefile('C:/users/jmartinez/documents/Test Zones/zones_prj_res2.shp')
wqueen = py.queen_from_shapefile(os.path.join(misc.data_dir(),'shapefiles\\zones.shp'))
w = py.weights.weights.W(wqueen.neighbors, wqueen.weights)
x = zones[['zonal_pop','mean_income']]
x = x.apply(np.log1p)
x['ln_jobs_within_30min'] = zones['ln_jobs_within_30min']
x['zone_contains_park'] = zones['zone_contains_park']
x['Arapahoe'] = zones['Arapahoe']
x['Boulder'] = zones['Boulder']
x['Broomfield'] = zones['Broomfield']
x['Clear Creek'] = zones['Clear Creek']
x['Denver'] = zones['Denver']
x['Douglas'] = zones['Douglas']
x['Elbert'] = zones['Elbert']
x['Gilpin'] = zones['Gilpin']
x['Jefferson'] = zones['Jefferson']
x['Weld'] = zones['Weld']
x=x.fillna(0)
x = x.as_matrix()
imat = zones[['ln_avg_nonres_unit_price_zone','avg_far']]
imat = imat.fillna(0)
imat = imat.as_matrix()
yend = zones['ln_avg_unit_price_zone']
yend = yend.fillna(0)
yend = yend.as_matrix()
yend = np.reshape(yend,(zones.shape[0],1))
y = zones['residential_sqft_zone']
y = y.fillna(0)
y = y.apply(np.log1p)
y = y.as_matrix()
y = np.reshape(y,(zones.shape[0],1))
imat_names = ['non_res_price','avg_far']
x_names = ['zonal_pop', 'mean_income', 'ln_jobs_within_30min', 'zone_contains_park','Arapahoe','Boulder','Broomfield','Clear Creek','Denver','Douglas','Elbert','Gilpin','Jefferson','Weld']
yend_name = ['ln_avg_unit_price_zone']
y_name = 'residential_sqft_zone'
reg_2sls = py.spreg.twosls_sp.GM_Lag(y, x, yend=yend, q=imat, w=w, w_lags=2, robust ='white', name_x = x_names, name_q = imat_names, name_y = y_name, name_yend = yend_name)
demand_elasticity = np.absolute(reg_2sls.betas[15])
demand_elasticity = 1/demand_elasticity[0]
#
return demand_elasticity
def processAlgorithm(self, progress):
field = self.getParameterValue(self.FIELD)
field = field[0:10] # try to handle Shapefile field length limit
filename = self.getParameterValue(self.INPUT)
layer = dataobjects.getObjectFromUri(filename)
filename = dataobjects.exportVectorLayer(layer)
provider = layer.dataProvider()
fields = provider.fields()
fields.append(QgsField('L_G', QVariant.Double))
fields.append(QgsField('L_G_p', QVariant.Double))
fields.append(QgsField('L_G_S', QVariant.Int))
fields.append(QgsField('L_G_ll_hh', QVariant.Int))
writer = self.getOutputFromName(self.OUTPUT).getVectorWriter(
fields, provider.geometryType(), layer.crs() )
contiguity = self.getParameterValue(self.CONTIGUITY)
if self.CONTIGUITY_OPTIONS[contiguity] == 'queen':
print 'INFO: Local G and G* using queen contiguity'
w = pysal.queen_from_shapefile(filename)
elif self.CONTIGUITY_OPTIONS[contiguity] == 'rook':
print 'INFO: Local G and G* using rook contiguity'
w = pysal.rook_from_shapefile(filename)
significance_level = self.getParameterValue(self.SIGNIFICANCE_LEVEL)
if self.SIGNIFICANCE_OPTIONS[significance_level] == '90%':
max_p = 0.10
elif self.SIGNIFICANCE_OPTIONS[significance_level] == '95%':
max_p = 0.05
elif self.SIGNIFICANCE_OPTIONS[significance_level] == '99%':
max_p = 0.01
print 'INFO: significance level ' + self.SIGNIFICANCE_OPTIONS[significance_level]
f = pysal.open(pysal.examples.get_path(filename.replace('.shp','.dbf')))
y = np.array(f.by_col[str(field)])
lg = pysal.G_Local(y,w,transform = "b", permutations = 999)
sig_g = 1.0 * lg.p_sim <= max_p
ll_hh = 1.0 * (lg.Gs > lg.EGs) + 1
sig_ll_hh = sig_g * ll_hh
outFeat = QgsFeature()
i = 0
for inFeat in processing.features(layer):
inGeom = inFeat.geometry()
outFeat.setGeometry(inGeom)
attrs = inFeat.attributes()
attrs.append(float(lg.Gs[i]))
attrs.append(float(lg.p_sim[i]))
attrs.append(int(sig_g[i]))
attrs.append(int(sig_ll_hh[i]))
outFeat.setAttributes(attrs)
writer.addFeature(outFeat)
i+=1
del writer
def test_spatial(self):
w = pysal.queen_from_shapefile(pysal.examples.get_path('columbus.shp'))
reg = TSLS(self.y, self.X, self.yd, self.q, spat_diag=True, w=w)
betas = np.array([[ 88.46579584], [ 0.5200379 ], [ -1.58216593]])
np.testing.assert_array_almost_equal(reg.betas, betas, 7)
vm = np.array([[ 229.05640809, 10.36945783, -9.54463414],
[ 10.36945783, 2.0013142 , -1.01826408],
[ -9.54463414, -1.01826408, 0.62914915]])
np.testing.assert_array_almost_equal(reg.vm, vm, 7)
ak_test = np.array([ 1.16816972, 0.27977763])
np.testing.assert_array_almost_equal(reg.ak_test, ak_test, 7)
def main(perms):
np.random.seed(10)
w = pysal.queen_from_shapefile(pysal.examples.get_path("NAT.shp"))
f = pysal.open(pysal.examples.get_path('NAT.dbf'))
y = np.array(f.by_col('POL90'))
y = np.reshape(y, (3085,))
t1 = time.time()
lm = Moran_Local(y, w, transformation = "r", permutations = perms, cores=None)
t2 = time.time()
print "Local Morans (only) time was {}".format(t2-t1)
print "I value: {}".format(lm.Is)
print "p sum: {}".format(lm.p_z_sim)
def main(perms):
np.random.seed(10)
w = pysal.queen_from_shapefile(pysal.examples.get_path("NAT.shp"))
f = pysal.open(pysal.examples.get_path('NAT.dbf'))
y = np.array(f.by_col('POL90'))
y = np.reshape(y, (3085, ))
t1 = time.time()
lm = Moran_Local(y, w, transformation="r", permutations=perms, cores=None)
t2 = time.time()
print "Local Morans (only) time was {}".format(t2 - t1)
print "I value: {}".format(lm.Is)
print "p sum: {}".format(lm.p_z_sim)
def test_mplot():
link = ps.examples.get_path('columbus.shp')
db = read_files(link)
y = db['HOVAL'].values
w = ps.queen_from_shapefile(link)
w.transform = 'R'
m = ps.Moran(y, w)
fig = mplot(m, xlabel='Response', ylabel='Spatial Lag',
title='Moran Scatterplot', custom=(7,7))
plt.close(fig)
def _contiguity(arg_dict):
"""
General handler for building contiguity weights from shapefiles
Examples
--------
>>> w = wmd_reader('wrook1.wmd')
>>> w.n
49
>>> w.meta_data
{'root': {u'input1': {u'data1': {u'type': u'shp',
u'uri': u'http://toae.org/pub/columbus.shp'}},
u'weight_type': u'rook', u'transform': u'O'}}
"""
input1 = arg_dict['input1']
for key in input1:
input1 = input1[key]
break
uri = input1['uri']
weight_type = arg_dict['weight_type']
weight_type = weight_type.lower()
if weight_type == 'rook':
w = ps.rook_from_shapefile(uri)
elif weight_type == 'queen':
w = ps.queen_from_shapefile(uri)
else:
print "Unsupported contiguity criterion: ", weight_type
return None
if 'parameters' in arg_dict:
order = arg_dict['parameters'].get('order', 1) # default to 1st order
lower = arg_dict['parameters'].get(
'lower', 0) # default to exclude lower orders
if order > 1:
w_orig = w
w = ps.higher_order(w, order)
if lower:
for o in xrange(order - 1, 1, -1):
w = ps.weights.w_union(ps.higher_order(w_orig, o), w)
w = ps.weights.w_union(w, w_orig)
parameters = arg_dict['parameters']
else:
parameters = {'lower': 0, 'order': 1}
w = WMD(w.neighbors, w.weights)
w.meta_data = {}
w.meta_data["input1"] = {"type": 'shp', 'uri': uri}
w.meta_data["transform"] = w.transform
w.meta_data["weight_type"] = weight_type
w.meta_data['parameters'] = parameters
return w
def _contiguity(arg_dict):
"""
General handler for building contiguity weights from shapefiles
Examples
--------
>>> w = wmd_reader('wrook1.wmd')
>>> w.n
49
>>> w.meta_data
{'root': {u'input1': {u'data1': {u'type': u'shp',
u'uri': u'http://toae.org/pub/columbus.shp'}},
u'weight_type': u'rook', u'transform': u'O'}}
"""
input1 = arg_dict['input1']
for key in input1:
input1 = input1[key]
break
uri = input1['uri']
weight_type = arg_dict['weight_type']
weight_type = weight_type.lower()
if weight_type == 'rook':
w = ps.rook_from_shapefile(uri)
elif weight_type == 'queen':
w = ps.queen_from_shapefile(uri)
else:
print "Unsupported contiguity criterion: ", weight_type
return None
if 'parameters' in arg_dict:
order = arg_dict['parameters'].get('order', 1) # default to 1st order
lower = arg_dict['parameters'].get('lower', 0) # default to exclude lower orders
if order > 1:
w_orig = w
w = ps.higher_order(w, order)
if lower:
for o in xrange(order-1, 1, -1):
w = ps.weights.w_union(ps.higher_order(w_orig, o), w)
w = ps.weights.w_union(w, w_orig)
parameters = arg_dict['parameters']
else:
parameters = {'lower': 0, 'order': 1}
w = WMD(w.neighbors, w.weights)
w.meta_data = {}
w.meta_data["input1"] = {"type": 'shp', 'uri': uri}
w.meta_data["transform"] = w.transform
w.meta_data["weight_type"] = weight_type
w.meta_data['parameters'] = parameters
return w
def test_mplot():
link = ps.examples.get_path('columbus.shp')
db = read_files(link)
y = db['HOVAL'].values
w = ps.queen_from_shapefile(link)
w.transform = 'R'
m = ps.Moran(y, w)
fig = mplot(m,
xlabel='Response',
ylabel='Spatial Lag',
title='Moran Scatterplot',
custom=(7, 7))
plt.close(fig)
def BugsAdj(Shp, IDField, OrderList=None):
Adj=pysal.queen_from_shapefile(Shp,IDField)
NeighDict=Adj.neighbors
if OrderList:
SortOrder=OrderList
else:
SortOrder=sorted(Adj.neighbors.keys())
NeighListNest=[NeighDict[i] for i in SortOrder]
NeighListFlat=[Key for SubList in NeighListNest for Key in sorted(SubList)]
NeighList=[SortOrder.index(X)+1 for X in NeighListFlat]
NumNeighList=[len(x) for x in NeighListNest]
return {'NumNeigh':NumNeighList, 'Neigh':NeighList}
def _importArcData(filename):
"""Creates a new Layer from a shapefile (<file>.shp)
This function wraps and extends a core clusterPy function to utilize PySAL
W constructors and dbf readers.
Parameters
==========
filename: string
suffix of shapefile (fileName not fileName.shp)
Returns
=======
layer: clusterpy layer instance
"""
layer = _clusterpy.Layer()
layer.name = filename.split('/')[-1]
#print "Loading " + filename + ".dbf"
dbf = ps.open(filename+".dbf")
fields = dbf.header
#data, fields, specs = importDBF(filename + '.dbf')
data = {}
#print "Loading " + filename + ".shp"
if fields[0] != "ID":
fields = ["ID"] + fields
for y in range(dbf.n_records):
data[y] = [y] + dbf.by_row(y)
else:
for y in range(dbf.n_records):
data[y] = dbf.by_row_(y)
layer.fieldNames = fields
layer.Y = data
shpf = filename+".shp"
layer.shpType = 5
#print 'pysal reader'
layer.Wrook = ps.rook_from_shapefile(filename+".shp").neighbors
layer.Wqueen = ps.queen_from_shapefile(filename+".shp").neighbors
#print "Done"
return layer
def lisa_mapa(variavel, shapefile, p_thres=0.05, **kws):
w = ps.queen_from_shapefile(shapefile)
lisa = ps.Moran_Local(variavel, w)
fig = plt.figure(figsize=(9, 7))
shp = ps.open(shapefile)
base = maps.map_poly_shp(shp)
base = maps.base_lisa_cluster(base, lisa, p_thres=p_thres)
base.set_edgecolor('1')
base.set_linewidth(0.1)
ax = maps.setup_ax([base], [shp.bbox])
boxes, labels = maps.lisa_legend_components(lisa, p_thres=p_thres)
plt.legend(boxes, labels, fancybox=True, **kws)
plt.show();
def _importArcData(filename):
"""Creates a new Layer from a shapefile (<file>.shp)
This function wraps and extends a core clusterPy function to utilize PySAL
W constructors and dbf readers.
Parameters
==========
filename: string
suffix of shapefile (fileName not fileName.shp)
Returns
=======
layer: clusterpy layer instance
"""
layer = _clusterpy.Layer()
layer.name = filename.split('/')[-1]
#print "Loading " + filename + ".dbf"
dbf = ps.open(filename+".dbf")
fields = dbf.header
#data, fields, specs = importDBF(filename + '.dbf')
data = {}
#print "Loading " + filename + ".shp"
if fields[0] != "ID":
fields = ["ID"] + fields
for y in range(dbf.n_records):
data[y] = [y] + dbf.by_row(y)
else:
for y in range(dbf.n_records):
data[y] = dbf.by_row_(y)
layer.fieldNames = fields
layer.Y = data
shpf = filename+".shp"
layer.shpType = 5
#print 'pysal reader'
layer.Wrook = ps.rook_from_shapefile(filename+".shp").neighbors
layer.Wqueen = ps.queen_from_shapefile(filename+".shp").neighbors
#print "Done"
return layer
def lisa_mapa(variavel, shapefile, p_thres=0.05, **kws):
w = ps.queen_from_shapefile(shapefile)
lisa = ps.Moran_Local(variavel, w)
fig = plt.figure(figsize=(9, 7))
shp = ps.open(shapefile)
base = maps.map_poly_shp(shp)
base = maps.base_lisa_cluster(base, lisa, p_thres=p_thres)
base.set_edgecolor('1')
base.set_linewidth(0.1)
ax = maps.setup_ax([base], [shp.bbox])
boxes, labels = maps.lisa_legend_components(lisa, p_thres=p_thres)
plt.legend(boxes, labels, fancybox=True, **kws)
plt.show()
def test_names(self):
w = pysal.queen_from_shapefile(pysal.examples.get_path("columbus.shp"))
gwk = pysal.kernelW_from_shapefile(
pysal.examples.get_path("columbus.shp"), k=5, function="triangular", fixed=False
)
name_x = ["inc"]
name_y = "crime"
name_yend = ["hoval"]
name_q = ["discbd"]
name_w = "queen"
name_gwk = "k=5"
name_ds = "columbus"
reg = TSLS(
self.y,
self.X,
self.yd,
self.q,
spat_diag=True,
w=w,
robust="hac",
gwk=gwk,
name_x=name_x,
name_y=name_y,
name_q=name_q,
name_w=name_w,
name_yend=name_yend,
name_gwk=name_gwk,
name_ds=name_ds,
)
betas = np.array([[88.46579584], [0.5200379], [-1.58216593]])
np.testing.assert_array_almost_equal(reg.betas, betas, 7)
vm = np.array(
[
[225.0795089, 17.11660041, -12.22448566],
[17.67097154, 2.47483461, -1.4183641],
[-12.45093722, -1.40495464, 0.8700441],
]
)
np.testing.assert_array_almost_equal(reg.vm, vm, 7)
self.assertListEqual(reg.name_x, ["CONSTANT"] + name_x)
self.assertListEqual(reg.name_yend, name_yend)
self.assertListEqual(reg.name_q, name_q)
self.assertEqual(reg.name_y, name_y)
self.assertEqual(reg.name_w, name_w)
self.assertEqual(reg.name_gwk, name_gwk)
self.assertEqual(reg.name_ds, name_ds)
def test_spatial(self):
X = np.array(self.db.by_col("INC"))
X = np.reshape(X, (49,1))
yd = np.array(self.db.by_col("CRIME"))
yd = np.reshape(yd, (49,1))
q = np.array(self.db.by_col("DISCBD"))
q = np.reshape(q, (49,1))
w = pysal.queen_from_shapefile(pysal.examples.get_path('columbus.shp'))
reg = GM_Lag(self.y, X, yend=yd, q=q, spat_diag=True, w=w)
betas = np.array([[ 5.46344924e+01], [ 4.13301682e-01], [ -5.92637442e-01], [ -7.40490883e-03]])
np.testing.assert_array_almost_equal(reg.betas, betas, 7)
vm = np.array( [[ 4.45202654e+02, -1.50290275e+01, -6.36557072e+00, -5.71403440e-03],
[ -1.50290275e+01, 5.93124683e-01, 2.19169508e-01, -6.70675916e-03],
[ -6.36557072e+00, 2.19169508e-01, 1.06577542e-01, -2.96533875e-03],
[ -5.71403440e-03, -6.70675916e-03, -2.96533875e-03, 1.15655425e-03]])
np.testing.assert_array_almost_equal(reg.vm, vm, 6)
ak_test = np.array([ 2.52597326, 0.11198567])
np.testing.assert_array_almost_equal(reg.ak_test, ak_test, 7)
def spw_from_shapefile(shapefile, norm=False):
polygons = ps.open(shapefile, 'r').read()
spolygons = list(map(asShape, polygons))
spolygons = [fix_mp(p) for p in spolygons]
perimeters = [p.length if norm else 1. for p in spolygons]
Wsrc = ps.queen_from_shapefile(shapefile)
new_weights, edges = {}, {}
for i in Wsrc.neighbors:
a = spolygons[i]
p = perimeters[i]
new_weights[i] = []
for j in Wsrc.neighbors[i]:
intersect = a.intersection(spolygons[j])
new_weights[i].append(intersect.length)
edges[i] = a.length - sum(new_weights[i]) # /a.length
return edges, ps.W(Wsrc.neighbors, new_weights)
def test_spatial(self):
w = pysal.queen_from_shapefile(pysal.examples.get_path('columbus.shp'))
reg = TSLS_Regimes(self.y, self.x, self.yd, self.q, self.regimes, spat_diag=True, w=w, regime_err_sep=False)
betas = np.array([[ 80.23408166],[ 5.48218125],[ 82.98396737],[ 0.49775429],[ -3.72663211],[ -1.27451485]])
np.testing.assert_array_almost_equal(reg.betas, betas, 7)
vm = np.array([[ 495.16048523, 78.89742341, 0. , 0. ,\
-47.12971066, 0. ],\
[ 78.89742341, 64.58341083, 0. , 0. ,\
-30.74878934, 0. ],\
[ 0. , 0. , 732.05899155, 11.21128921,\
0. , -20.96804956],\
[ 0. , 0. , 11.21128921, 7.48778398,\
0. , -2.56752553],\
[ -47.12971066, -30.74878934, 0. , 0. ,\
14.89456384, 0. ],\
[ 0. , 0. , -20.96804956, -2.56752553,\
0. , 1.3154267 ]])
np.testing.assert_array_almost_equal(reg.vm, vm, 7)
ak_test = np.array([ 0.69774552, 0.40354227])
np.testing.assert_array_almost_equal(reg.ak_test, ak_test, 7)
def _get_spatial_weights(self):
"""
creates the spatial weights object for use in
OLS. This structure defaults to a certain
spatial relationship. We can add more key words
and create different relationships
"""
#this matrix tells the algorithm how to
#look at neighboring features
#queen looks at polygons with shared edges
#queen b/c the way the chess piece moves
if self.spat_weights == "queen":
return pysal.queen_from_shapefile(self.infile)
elif self.spat_weights == "rook":
return pysal.rook_from_shapefile(self.infile)
else:
#won't use spatial weights
return None
def processAlgorithm(self, progress):
variable_field = self.getParameterValue(self.VARIABLE_FIELD)
variable_field = variable_field[
0:10] # try to handle Shapefile field length limit
population_field = self.getParameterValue(self.POPULATION_FIELD)
population_field = population_field[
0:10] # try to handle Shapefile field length limit
filename = self.getParameterValue(self.INPUT)
layer = dataobjects.getObjectFromUri(filename)
filename = dataobjects.exportVectorLayer(layer)
contiguity = self.getParameterValue(self.CONTIGUITY)
if self.CONTIGUITY_OPTIONS[contiguity] == 'queen':
print 'INFO: Moran\'s for rates using queen contiguity'
w = pysal.queen_from_shapefile(filename)
elif self.CONTIGUITY_OPTIONS[contiguity] == 'rook':
print 'INFO: Moran\'s for rates using rook contiguity'
w = pysal.rook_from_shapefile(filename)
f = pysal.open(
pysal.examples.get_path(filename.replace('.shp', '.dbf')))
y = np.array(f.by_col[str(variable_field)])
population = np.array(f.by_col[str(population_field)])
m = pysal.esda.moran.Moran_Rate(y,
population,
w,
transformation="r",
permutations=999)
print "Moran's I: %f" % (m.I)
print "INFO: Moran's I values range from -1 (indicating perfect dispersion) to +1 (perfect correlation). Values close to -1/(n-1) indicate a random spatial pattern."
print "p_norm: %f" % (m.p_norm)
print "p_rand: %f" % (m.p_rand)
print "p_sim: %f" % (m.p_sim)
print "INFO: p values smaller than 0.05 indicate spatial autocorrelation that is significant at the 5% level."
print "z_norm: %f" % (m.z_norm)
print "z_rand: %f" % (m.z_rand)
print "z_sim: %f" % (m.z_sim)
print "INFO: z values greater than 1.96 or smaller than -1.96 indicate spatial autocorrelation that is significant at the 5% level."
def compute(self, vlayer, tfield, idvar,matType):
vlayer=qgis.utils.iface.activeLayer()
idvar=self.idVariable.currentText()
tfield=self.inField.currentText()
provider=vlayer.dataProvider()
allAttrs=provider.attributeIndexes()
caps=vlayer.dataProvider().capabilities()
if caps & QgsVectorDataProvider.AddAttributes:
TestField = idvar[:5]+"_qrr"
res = vlayer.dataProvider().addAttributes([QgsField(TestField, QVariant.Double)])
wp=str(self.dic[str(self.inShape.currentText())])
if matType == "Rook":
w = py.rook_from_shapefile(wp, idVariable=unicode(idvar))
else:
w = py.queen_from_shapefile(wp, idVariable=unicode(idvar))
w1=wp[:-3]+"dbf"
db=py.open(w1)
y=np.array(db.by_col[unicode(tfield)])
mi = py.Moran(y,w)
mg = mi.I
self.SAresult.setText("Global Moran's I index is " + str(mg))
def test_names(self):
w = pysal.queen_from_shapefile(pysal.examples.get_path('columbus.shp'))
gwk = pysal.kernelW_from_shapefile(pysal.examples.get_path('columbus.shp'),k=5,function='triangular', fixed=False)
name_x = ['inc']
name_y = 'crime'
name_yend = ['hoval']
name_q = ['discbd']
name_w = 'queen'
name_gwk = 'k=5'
name_ds = 'columbus'
name_regimes= 'nsa'
reg = TSLS_Regimes(self.y, self.x, self.yd, self.q, self.regimes, regime_err_sep=False,
spat_diag=True, w=w, robust='hac', gwk=gwk, name_regimes=name_regimes,
name_x=name_x, name_y=name_y, name_q=name_q, name_w=name_w,
name_yend=name_yend, name_gwk=name_gwk, name_ds=name_ds)
betas = np.array([[ 80.23408166],[ 5.48218125],[ 82.98396737],[ 0.49775429],[ -3.72663211],[ -1.27451485]])
np.testing.assert_array_almost_equal(reg.betas, betas, 7)
vm = np.array([[ 522.75813101, 120.64940697, -15.60303241, -0.976389 ,\
-67.15556574, 0.64553579],\
[ 122.83491674, 122.62303068, -5.52270916, 0.05023488,\
-57.89404902, 0.15750428],\
[ 0.1983661 , -0.03539147, 335.24731378, 17.40764168,\
-0.26447114, -14.3375455 ],\
[ -0.13612426, -0.43622084, 18.46644989, 2.70320508,\
0.20398876, -1.31821991],\
[ -68.0704928 , -58.03685405, 2.66225388, 0.00323082,\
27.68512974, -0.08124602],\
[ -0.08001296, 0.13575504, -14.6998294 , -1.28225201,\
-0.05193056, 0.79845124]])
np.testing.assert_array_almost_equal(reg.vm, vm, 7)
self.assertEqual(reg.name_x, ['0_CONSTANT', '0_inc', '1_CONSTANT', '1_inc'])
self.assertEqual(reg.name_yend, ['0_hoval', '1_hoval'])
self.assertEqual(reg.name_q, ['0_discbd', '1_discbd'])
self.assertEqual(reg.name_y, name_y)
self.assertEqual(reg.name_w, name_w)
self.assertEqual(reg.name_gwk, name_gwk)
self.assertEqual(reg.name_ds, name_ds)
self.assertEqual(reg.name_regimes, name_regimes)
def compute(self, vlayer, tfield, idvar, matType):
vlayer = qgis.utils.iface.activeLayer()
idvar = self.idVariable.currentText()
tfield = self.inField.currentText()
provider = vlayer.dataProvider()
allAttrs = provider.attributeIndexes()
caps = vlayer.dataProvider().capabilities()
if caps & QgsVectorDataProvider.AddAttributes:
TestField = idvar[:5] + "_qrr"
res = vlayer.dataProvider().addAttributes(
[QgsField(TestField, QVariant.Double)])
wp = str(self.dic[str(self.inShape.currentText())])
if matType == "Rook":
w = py.rook_from_shapefile(wp, idVariable=unicode(idvar))
else:
w = py.queen_from_shapefile(wp, idVariable=unicode(idvar))
w1 = wp[:-3] + "dbf"
db = py.open(w1)
y = np.array(db.by_col[unicode(tfield)])
mi = py.Moran(y, w)
mg = mi.I
self.SAresult.setText("Global Moran's I index is " + str(mg))
def getNeighborPairs(shapefileDir):
neighborPairs = []
tmp = set()
sf = shapefile.Reader(shapefileDir)
records = sf.records()
# choose the "queen" neighbors. This means shapes are neighbors as long as they share a vertex.
# alternative is the "rook", where shapes must share an edge.
w = ps.queen_from_shapefile(shapefileDir + ".shp")
N = w.n
for i in range(N):
# blockId is the field in index 4
blockId = int(records[i][4])
# this var is a map containing the neighbors of block i, where the key is the neighbor, and value is the weight
neighbors = w[i]
for n in neighbors.keys():
neighborId = int(records[n][4])
# pairs are unordered, we don't want to double add pairs
pair = (blockId, neighborId)
revPair = (neighborId, blockId)
if revPair in tmp:
continue
tmp.add(pair)
neighborPairs.append(pair)
return neighborPairs
def parsewmd(jwmd, uploaddir=None):
#Get the URI
uri = jwmd['input1']['data1']['uri']
url = urlparse(uri)
if url.scheme == 'http':
basename = url.path.split('/')[-1]
if uploaddir != None:
basename = os.path.join(uploaddir, basename)
#The WMD never spcifies the .shx, but PySAL needs it
shx = basename.replace('.shp', '.shx')
shxuri = uri.replace('.shp', '.shx')
#Download both files locally
response = urllib.urlretrieve(uri, basename)
response = urllib.urlretrieve(shxuri, shx)
elif url.scheme == 'file':
pass
#Get the generation information
wtype = jwmd['weight_type']
transform = jwmd['transform']
#Populate the W
if wtype.lower() == 'rook':
w = ps.rook_from_shapefile(basename)
elif wtype.lower() == 'queen':
w = ps.queen_from_shapefile(basename)
#Transform
w.transform = transform
return w
return model
def _test():
import doctest
start_suppress = np.get_printoptions()['suppress']
np.set_printoptions(suppress=True)
doctest.testmod()
np.set_printoptions(suppress=start_suppress)
if __name__ == '__main__':
_test()
import numpy as np
import pysal
db = pysal.open(pysal.examples.get_path("columbus.dbf"),'r')
y_var = 'CRIME'
y = np.array([db.by_col(y_var)]).reshape(49,1)
x_var = ['INC']
x = np.array([db.by_col(name) for name in x_var]).T
yd_var = ['HOVAL']
yd = np.array([db.by_col(name) for name in yd_var]).T
q_var = ['DISCBD']
q = np.array([db.by_col(name) for name in q_var]).T
r_var = 'NSA'
regimes = db.by_col(r_var)
w = pysal.queen_from_shapefile(pysal.examples.get_path("columbus.shp"))
w.transform = 'r'
model = GM_Lag_Regimes(y, x, regimes, yend=yd, q=q, w=w, constant_regi='many', spat_diag=True, sig2n_k=False, lag_q=True, name_y=y_var, name_x=x_var, name_yend=yd_var, name_q=q_var, name_regimes=r_var, name_ds='columbus', name_w='columbus.gal')
print model.summary
'''
for i in range(comm.size):
if i == rank:
print i, neighbors
comm.Barrier() # Just to get prints to be pretty
'''
neighbors_list = comm.gather(neighbors, root=0)
if rank == 0:
neighbors = neighbors_list[0]
for n in neighbors_list[1:]:
for k, v in n.iteritems():
try:
neighbors[k] = neighbors[k].union(v)
except KeyError:
neighbors[k] = v
for k, v in neighbors.iteritems():
v.remove(k)
w_mpi = ps.W(neighbors)
w_ps = ps.queen_from_shapefile(sys.argv[1])
for k, v in w_mpi.neighbors.iteritems():
#print k, sorted(v), sorted(w_ps.neighbors[k])
assert(sorted(v) == sorted(w_ps.neighbors[k]))
