def swm2Weights(swmFile, master2Order=None):
swm = WU.SWMReader(swmFile)
numObs = swm.numObs
adjust = False
if master2Order and len(master2Order) < numObs:
msg = ("The spatial attributes have fewer entries than spatial"
"weights! Weights will be adjusted dynamically...")
ARCPY.AddWarning(msg)
adjust = True
neighs = {}
w = {}
rowStandard = swm.rowStandard
for i in range(numObs):
masterID, nn, nhsTemp, weightsTemp, sumUnstandard = swm.swm.readEntry()
if master2Order == None:
# no need adjust when convert directly from weights content
orderID = masterID
nhIDs = nhsTemp
weights = weightsTemp
elif masterID in master2Order:
orderID = master2Order[masterID]
if not adjust:
nhIDs = [master2Order[nh] for nh in nhsTemp]
weights = weightsTemp
else:
# Restandardize Due to Subset/Select
nhIDs = []
weights = []
if nn:
for i in range(nn):
nh = nhsTemp[i]
if nh in master2Order:
nhOrder = master2Order[nh]
nhIDs.append(nhOrder)
nhWeight = weightsTemp[i]
if rowStandard:
# Unstandardize if Necessary
nhWeight = nhWeight * sumUnstandard[0]
weights.append(nhWeight)
# Re-Standardize
if nhIDs:
weights = NUM.array(weights)
if rowStandard:
weights = (1.0 / weights.sum()) * weights
# Add To Dict Structures
neighs[orderID] = nhIDs
w[orderID] = weights
swm.close()
wobj = W(neighs, w)
wobj._varName = swm.masterField
return wobj
def getFeatNumFromWeights(weightsFile):
weightType = returnWeightFileType(weightsFile)
if weightType in ['GAL', 'GWT']:
weightFile = open(weightsFile, 'r')
info = weightFile.readline().strip().split()
if weightType == 'GAL':
if len(info) == 1:
return LOCALE.atoi(info[0])
elif len(info) > 1:
return LOCALE.atoi(info[1])
else:
return LOCALE.atoi(info[1])
elif weightType == 'SWM':
swm = WU.SWMReader(weightsFile)
return swm.numObs
def getIDFieldFromWeights(weightsFile):
weightType = returnWeightFileType(weightsFile)
if weightType == 'SWM':
swm = WU.SWMReader(weightsFile)
if not swm.masterField or swm.masterField == 'UNKNOWN':
return None
return swm.masterField
else:
weightFile = open(weightsFile, 'r')
info = weightFile.readline().strip().split()
weightFile.close()
for item in info:
if not item.isdigit() and item.lower() != "unknown" \
and len(item) > 0:
return item
return None
def construct(self):
"""Constructs the neighborhood structure for each feature and
dispatches the appropriate values for the calculation of the
statistic."""
#### Shorthand Attributes ####
ssdo = self.ssdo
varName = self.varName
concept = self.concept
gaConcept = concept.lower()
threshold = self.threshold
exponent = self.exponent
wType = self.wType
numObs = self.numObs
master2Order = self.master2Order
masterField = ssdo.masterField
weightsFile = self.weightsFile
potentialField = self.potentialField
#### Assure that Variance is Larger than Zero ####
yVar = NUM.var(self.y)
if NUM.isnan(yVar) or yVar <= 0.0:
ARCPY.AddIDMessage("ERROR", 906)
raise SystemExit()
#### Create Summed Variables ####
self.intRange = NUM.arange(numObs)
self.floatN = self.numObs * 1.0
ySum = self.y.sum()
ySum2 = (self.y**2.0).sum()
self.yBar = ySum / self.floatN
self.S = NUM.sqrt((ySum2 / self.floatN) - self.yBar**2.0)
self.nm1 = self.floatN - 1.0
#### Create Base Data Structures/Variables ####
self.gi = NUM.zeros(numObs)
self.pVals = NUM.ones(numObs)
if self.permutations:
self.pseudoPVals = NUM.ones(numObs)
#### Set Neighborhood Structure Type ####
if self.weightsFile:
if self.swmFileBool:
#### Open Spatial Weights and Obtain Chars ####
swm = WU.SWMReader(weightsFile)
N = swm.numObs
rowStandard = swm.rowStandard
self.swm = swm
#### Check to Assure Complete Set of Weights ####
if numObs > N:
ARCPY.AddIDMessage("ERROR", 842, numObs, N)
raise SystemExit()
#### Check if Selection Set ####
isSubSet = False
if numObs < N:
isSubSet = True
iterVals = xrange(N)
else:
#### Warning for GWT with Bad Records/Selection ####
if ssdo.selectionSet or ssdo.badRecords:
ARCPY.AddIDMessage("WARNING", 1029)
#### Build Weights Dictionary ####
weightDict = WU.buildTextWeightDict(weightsFile, master2Order)
iterVals = master2Order.keys()
N = numObs
elif wType in [4, 5]:
#### Polygon Contiguity ####
if wType == 4:
contiguityType = "ROOK"
else:
contiguityType = "QUEEN"
contDict = WU.polygonNeighborDict(ssdo.inputFC,
ssdo.oidName,
contiguityType=contiguityType)
iterVals = master2Order.keys()
N = numObs
else:
gaTable = ssdo.gaTable
gaSearch = GAPY.ga_nsearch(gaTable)
if wType == 7:
#### Zone of Indiff, All Related to All ####
gaSearch.init_nearest(threshold, numObs, gaConcept)
else:
#### Inverse and Fixed Distances ####
gaSearch.init_nearest(threshold, self.numNeighs, gaConcept)
iterVals = range(numObs)
N = numObs
neighWeights = ARC._ss.NeighborWeights(gaTable,
gaSearch,
weight_type=wType,
exponent=exponent,
row_standard=False,
include_self=True)
#### Create Progressor ####
msg = ARCPY.GetIDMessage(84007)
if self.permutations:
msg += ": Using Permutations = %i" % self.permutations
ARCPY.SetProgressor("step", msg, 0, N, 1)
#### Create Neighbor Info Class ####
ni = WU.NeighborInfo(masterField)
#### Calculation For Each Feature ####
for i in iterVals:
if self.swmFileBool:
#### Using SWM File ####
info = swm.swm.readEntry()
masterID = info[0]
if master2Order.has_key(masterID):
rowInfo = WU.getWeightsValuesSWM(info,
master2Order,
self.y,
rowStandard=rowStandard,
potVals=self.potVals)
includeIt = True
else:
includeIt = False
elif self.weightsFile and not self.swmFileBool:
#### Text Weights ####
masterID = i
includeIt = True
rowInfo = WU.getWeightsValuesText(masterID,
master2Order,
weightDict,
self.y,
potVals=self.potVals,
allowSelf=True)
elif wType in [4, 5]:
#### Polygon Contiguity ####
masterID = i
includeIt = True
rowInfo = WU.getWeightsValuesCont(masterID,
master2Order,
contDict,
self.y,
rowStandard=False,
potVals=self.potVals)
else:
#### Distance Based ####
masterID = gaTable[i][0]
includeIt = True
rowInfo = WU.getWeightsValuesOTF_Potent(
neighWeights, i, self.y, self.potVals)
#### Subset Boolean for SWM File ####
if includeIt:
#### Parse Row Info ####
orderID, yiVal, nhIDs, nhVals, weights = rowInfo
#### Assure Neighbors Exist After Selection ####
nn, nhIDs, nhVals, weights = ni.processInfo(
masterID, nhIDs, nhVals, weights)
if nn:
#### Calculate Local G ####
self.calculateGI(orderID, yiVal, nhVals, weights)
ARCPY.SetProgressorPosition()
#### Clean Up ####
if self.swmFileBool:
swm.close()
#### Report on Features with No Neighbors ####
ni.reportNoNeighbors(failAllNoNeighs=False)
self.setNullValues(ni.idsNoNeighs)
#### Report on Features with Large Number of Neighbors ####
ni.reportWarnings()
ni.reportMaximums()
self.neighInfo = ni
#### Set p-values for Gi Bins ####
if self.permutations:
#### Use Pseudo p-values ####
pv = self.pseudoPVals
else:
#### Use Traditional p-values ####
pv = self.pVals
toolMSG = ARCPY.GetIDMessage(84466)
if self.applyFDR:
#### Set Bins Using FDR ####
msg = ARCPY.GetIDMessage(84472).format(toolMSG)
ARCPY.SetProgressor("default", msg)
self.giBins = STATS.fdrTransform(pv, self.gi)
else:
msg = ARCPY.GetIDMessage(84473).format(toolMSG)
ARCPY.SetProgressor("default", msg)
self.giBins = STATS.pValueBins(pv, self.gi)
def swm2Weights(ssdo, swmfile):
"""Converts ArcGIS Sparse Spatial Weights Matrix (*.swm) file to
PySAL Sparse Spatial Weights Class.
INPUTS:
ssdo (class): instance of SSDataObject [1,2]
swmFile (str): full path to swm file
NOTES:
(1) Data must already be obtained using ssdo.obtainData()
(2) The masterField for the swm file and the ssdo object must be
the same and may NOT be the OID/FID/ObjectID
"""
neighbors = {}
weights = {}
#### Create SWM Reader Object ####
swm = WU.SWMReader(swmfile)
#### SWM May NOT be a Subset of the Data ####
if ssdo.numObs > swm.numObs:
ARCPY.AddIDMessage("ERROR", 842, ssdo.numObs, swm.numObs)
raise SystemExit()
if swm.masterField != ssdo.masterField:
ARCPY.AddWarning("ERROR", 938)
raise SystemExit()
#### Parse All SWM Records ####
for r in UTILS.ssRange(swm.numObs):
info = swm.swm.readEntry()
masterID, nn, nhs, w, sumUnstandard = info
#### Must Have at Least One Neighbor ####
if nn:
#### Must be in Selection Set (If Exists) ####
if masterID in ssdo.master2Order:
outNHS = []
outW = []
#### Transform Master ID to Order ID ####
orderID = ssdo.master2Order[masterID]
#### Neighbors and Weights Adjusted for Selection ####
for nhInd, nhVal in enumerate(nhs):
try:
nhOrder = ssdo.master2Order[nhVal]
outNHS.append(nhOrder)
weightVal = w[nhInd]
if swm.rowStandard:
weightVal = weightVal * sumUnstandard[0]
outW.append(weightVal)
except KeyError:
pass
#### Add Selected Neighbors/Weights ####
if len(outNHS):
neighbors[orderID] = outNHS
weights[orderID] = outW
swm.close()
#### Construct PySAL Spatial Weights and Standardize as per SWM ####
w = PYSAL.W(neighbors, weights)
if swm.rowStandard:
w.transform = 'R'
return w
def construct(self):
"""Constructs the neighborhood structure for each feature and
dispatches the appropriate values for the calculation of the
statistic."""
#### Shorthand Attributes ####
ssdo = self.ssdo
varName = self.varName
concept = self.concept
gaConcept = concept.lower()
threshold = self.threshold
exponent = self.exponent
wType = self.wType
rowStandard = self.rowStandard
numObs = self.numObs
master2Order = self.master2Order
masterField = ssdo.masterField
weightsFile = self.weightsFile
#### Check That All Input Values are Positive ####
if NUM.sum(self.y < 0.0) != 0:
ARCPY.AddIDMessage("Error", 915)
raise SystemExit()
#### Assure that Variance is Larger than Zero ####
yVar = NUM.var(self.y)
if NUM.isnan(yVar) or yVar <= 0.0:
ARCPY.AddIDMessage("Error", 906)
raise SystemExit()
#### Create Base Data Structures/Variables ####
self.numer = 0.0
self.denom = 0.0
self.rowSum = NUM.zeros(numObs)
self.colSum = NUM.zeros(numObs)
self.ySum = 0.0
self.y2Sum = 0.0
self.y3Sum = 0.0
self.y4Sum = 0.0
self.s0 = 0
self.s1 = 0
self.wij = {}
#### Set Neighborhood Structure Type ####
if self.weightsFile:
if self.swmFileBool:
#### Open Spatial Weights and Obtain Chars ####
swm = WU.SWMReader(weightsFile)
N = swm.numObs
rowStandard = swm.rowStandard
#### Check to Assure Complete Set of Weights ####
if numObs > N:
ARCPY.AddIDMessage("Error", 842, numObs, N)
raise SystemExit()
#### Check if Selection Set ####
isSubSet = False
if numObs < N:
isSubSet = True
iterVals = xrange(N)
else:
#### Warning for GWT with Bad Records/Selection ####
if ssdo.selectionSet or ssdo.badRecords:
ARCPY.AddIDMessage("WARNING", 1029)
#### Build Weights Dictionary ####
weightDict = WU.buildTextWeightDict(weightsFile, master2Order)
iterVals = master2Order.iterkeys()
N = numObs
elif wType in [4, 5]:
#### Polygon Contiguity ####
if wType == 4:
contiguityType = "ROOK"
else:
contiguityType = "QUEEN"
contDict = WU.polygonNeighborDict(ssdo.inputFC, ssdo.oidName,
contiguityType = contiguityType)
iterVals = master2Order.keys()
N = numObs
else:
gaTable = ssdo.gaTable
gaSearch = GAPY.ga_nsearch(gaTable)
if wType == 7:
#### Zone of Indiff, All Related to All ####
gaSearch.init_nearest(threshold, numObs, gaConcept)
else:
#### Inverse and Fixed Distances ####
gaSearch.init_nearest(threshold, 0, gaConcept)
iterVals = xrange(numObs)
N = numObs
neighWeights = ARC._ss.NeighborWeights(gaTable, gaSearch,
weight_type = wType,
exponent = exponent,
row_standard = rowStandard)
#### Create Progressor ####
ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84007), 0, N, 1)
#### Create Neighbor Info Class ####
ni = WU.NeighborInfo(masterField)
#### Calculation For Each Feature ####
for i in iterVals:
if self.swmFileBool:
#### Using SWM File ####
info = swm.swm.readEntry()
masterID = info[0]
if master2Order.has_key(masterID):
rowInfo = WU.getWeightsValuesSWM(info, master2Order,
self.y,
rowStandard = rowStandard,
isSubSet = isSubSet)
includeIt = True
else:
includeIt = False
elif self.weightsFile and not self.swmFileBool:
#### Text Weights ####
masterID = i
includeIt = True
rowInfo = WU.getWeightsValuesText(masterID, master2Order,
weightDict, self.y)
elif wType in [4, 5]:
#### Polygon Contiguity ####
masterID = i
includeIt = True
rowInfo = WU.getWeightsValuesCont(masterID, master2Order,
contDict, self.y,
rowStandard = rowStandard)
else:
#### Distance Based ####
masterID = gaTable[i][0]
includeIt = True
rowInfo = WU.getWeightsValuesOTF(neighWeights, i, self.y)
#### Subset Boolean for SWM File ####
if includeIt:
#### Parse Row Info ####
orderID, yiVal, nhIDs, nhVals, weights = rowInfo
#### Assure Neighbors Exist After Selection ####
nn, nhIDs, nhVals, weights = ni.processInfo(masterID, nhIDs,
nhVals, weights)
if nn:
#### Process Feature Contribution to General G ####
self.processRow(orderID, yiVal, nhIDs,
nhVals, weights)
#### Reset Progessor ####
ARCPY.SetProgressorPosition()
#### Clean Up ####
if self.swmFileBool:
swm.close()
#### Report on Features with No Neighbors ####
ni.reportNoNeighbors()
#### Report on Features with Large Number of Neighbors ####
ni.reportWarnings()
ni.reportMaximums()
self.neighInfo = ni
def swm2Table(swmFile, outputTable):
"""Places the spatial relationships contained in a given Spatial
Weight Matrix File (*.swm) into a given output table.
INPUTS:
swmFile (str): Path to the input spatial weight matrix file
outputTable (str): Path to the output database table
"""
#### Open Spatial Weights and Obtain Characteristics ####
swm = WU.SWMReader(swmFile)
masterField = swm.masterField
N = swm.numObs
rowStandard = swm.rowStandard
#### Allow Overwrite Output ####
ARCPY.env.overwriteOutput = True
#### Get Output Table Name With Extension if Appropriate ####
outputTable, dbf = UTILS.returnTableName(outputTable)
#### Delete Table If Exists ####
UTILS.passiveDelete(outputTable)
#### Create Table ####
outPath, outName = OS.path.split(outputTable)
try:
DM.CreateTable(outPath, outName)
except:
ARCPY.AddIDMessage("ERROR", 541)
raise SystemExit()
#### Create a List of Required Field Names ####
fn = UTILS.getFieldNames(swm2TabFieldNames, outPath)
neighFieldName, weightFieldName = fn
fieldNames = [masterField, neighFieldName, weightFieldName]
fieldTypes = ["LONG", "LONG", "DOUBLE"]
for ind, field in enumerate(fieldNames):
UTILS.addEmptyField(outputTable, field, fieldTypes[ind])
#### Create Insert Cursor ####
try:
insert = DA.InsertCursor(outputTable, fieldNames)
except:
ARCPY.AddIDMessage("ERROR", 204)
raise SystemExit()
#### Create Progressor ####
ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84117), 0, N, 1)
#### Process Spatial Weights File and Populate Output Table ####
try:
for r in xrange(N):
info = swm.swm.readEntry()
masterID, nn, nhs, weights, sumUnstandard = info
if nn != 0:
for ind, neigh in enumerate(nhs):
row = [masterID, neigh, weights[ind]]
insert.insertRow(row)
ARCPY.SetProgressorPosition()
except:
swm.close()
ARCPY.AddIDMessage("ERROR", 919)
raise SystemExit()
#### Clean Up ####
del insert
swm.close()
#### Report if Any Features Have No Neighbors ####
swm.reportNoNeighbors()
#### Make Table Visable in TOC if *.dbf Had To Be Added ####
if dbf:
ARCPY.SetParameterAsText(1, outputTable)
def construct(self):
"""Constructs the neighborhood structure for each feature and
dispatches the appropriate values for the calculation of the
statistic."""
#### Shorthand Attributes ####
ssdo = self.ssdo
masterField = ssdo.masterField
numObs = len(self.y)
master2Order = self.ssdo.master2Order
yVar = NUM.var(self.y)
if NUM.isnan(yVar) or yVar <= 0.0:
ARCPY.AddIDMessage("Error", 906)
raise SystemExit()
#### Create Deviation Variables ####
self.yBar = NUM.mean(self.y)
self.yDev = self.y - self.yBar
#### Create Base Data Structures/Variables ####
self.numer = 0.0
self.denom = NUM.sum(self.yDev**2.0)
self.rowSum = NUM.zeros(numObs)
self.colSum = NUM.zeros(numObs)
self.s0 = 0
self.s1 = 0
self.wij = {}
#### Open Spatial Weights and Obtain Chars ####
if self.weightsType == "SWM":
swm = WU.SWMReader(self.weightsMatrix)
N = swm.numObs
rowStandard = swm.rowStandard
#### Check to Assure Complete Set of Weights ####
if numObs > N:
ARCPY.AddIDMessage("Error", 842, numObs, N)
raise SystemExit()
#### Check if Selection Set ####
isSubSet = False
if numObs < N:
isSubSet = True
iterVals = xrange(N)
elif self.weightsType == "GWT":
#### Warning for GWT with Bad Records/Selection ####
if ssdo.selectionSet or ssdo.badRecords:
ARCPY.AddIDMessage("WARNING", 1029)
#### Build Weights Dictionary ####
iterVals = master2Order.keys()
N = numObs
else:
#### Use GA Table, 8 Nearest Neighbors ####
iterVals = range(numObs)
N = numObs
neighWeights = ARC._ss.NeighborWeights(ssdo.gaTable,
self.weightsMatrix)
#### Create Neighbor Info Class ####
ni = WU.NeighborInfo(masterField, silent=self.silent)
#### Calculation For Each Feature ####
for i in iterVals:
if self.weightsType == "SWM":
info = swm.swm.readEntry()
masterID = info[0]
if master2Order.has_key(masterID):
rowInfo = WU.getWeightsValuesSWM(info,
master2Order,
self.yDev,
rowStandard=rowStandard,
isSubSet=isSubSet)
includeIt = True
else:
includeIt = False
elif self.weightsType == "GWT":
#### Text Weights ####
masterID = i
includeIt = True
rowInfo = WU.getWeightsValuesText(masterID, master2Order,
self.weightsMatrix,
self.yDev)
else:
#### Distance Based ####
masterID = ssdo.gaTable[i][0]
includeIt = True
rowInfo = WU.getWeightsValuesOTF(neighWeights, i, self.yDev)
#### Subset Boolean for SWM File ####
if includeIt:
#### Parse Row Info ####
orderID, yiDev, nhIDs, nhVals, weights = rowInfo
#### Assure Neighbors Exist After Selection ####
nn, nhIDs, nhVals, weights = ni.processInfo(
masterID, nhIDs, nhVals, weights)
if nn:
#### Process Feature Contribution to Moran's I ####
self.processRow(orderID, yiDev, nhIDs, nhVals, weights)
#### Clean Up ####
if self.weightsType == "SWM":
swm.close()
if not self.silent:
#### Report on Features with No Neighbors ####
ni.reportNoNeighbors()
#### Report on Features with Large Number of Neighbors ####
ni.reportWarnings()
ni.reportMaximums()
self.neighInfo = ni
