def swm2Weights(swmFile, master2Order=None): swm = WU.SWMReader(swmFile) numObs = swm.numObs adjust = False if master2Order and len(master2Order) < numObs: msg = ("The spatial attributes have fewer entries than spatial" "weights! Weights will be adjusted dynamically...") ARCPY.AddWarning(msg) adjust = True neighs = {} w = {} rowStandard = swm.rowStandard for i in range(numObs): masterID, nn, nhsTemp, weightsTemp, sumUnstandard = swm.swm.readEntry() if master2Order == None: # no need adjust when convert directly from weights content orderID = masterID nhIDs = nhsTemp weights = weightsTemp elif masterID in master2Order: orderID = master2Order[masterID] if not adjust: nhIDs = [master2Order[nh] for nh in nhsTemp] weights = weightsTemp else: # Restandardize Due to Subset/Select nhIDs = [] weights = [] if nn: for i in range(nn): nh = nhsTemp[i] if nh in master2Order: nhOrder = master2Order[nh] nhIDs.append(nhOrder) nhWeight = weightsTemp[i] if rowStandard: # Unstandardize if Necessary nhWeight = nhWeight * sumUnstandard[0] weights.append(nhWeight) # Re-Standardize if nhIDs: weights = NUM.array(weights) if rowStandard: weights = (1.0 / weights.sum()) * weights # Add To Dict Structures neighs[orderID] = nhIDs w[orderID] = weights swm.close() wobj = W(neighs, w) wobj._varName = swm.masterField return wobj
def getFeatNumFromWeights(weightsFile): weightType = returnWeightFileType(weightsFile) if weightType in ['GAL', 'GWT']: weightFile = open(weightsFile, 'r') info = weightFile.readline().strip().split() if weightType == 'GAL': if len(info) == 1: return LOCALE.atoi(info[0]) elif len(info) > 1: return LOCALE.atoi(info[1]) else: return LOCALE.atoi(info[1]) elif weightType == 'SWM': swm = WU.SWMReader(weightsFile) return swm.numObs
def getIDFieldFromWeights(weightsFile): weightType = returnWeightFileType(weightsFile) if weightType == 'SWM': swm = WU.SWMReader(weightsFile) if not swm.masterField or swm.masterField == 'UNKNOWN': return None return swm.masterField else: weightFile = open(weightsFile, 'r') info = weightFile.readline().strip().split() weightFile.close() for item in info: if not item.isdigit() and item.lower() != "unknown" \ and len(item) > 0: return item return None
def construct(self): """Constructs the neighborhood structure for each feature and dispatches the appropriate values for the calculation of the statistic.""" #### Shorthand Attributes #### ssdo = self.ssdo varName = self.varName concept = self.concept gaConcept = concept.lower() threshold = self.threshold exponent = self.exponent wType = self.wType numObs = self.numObs master2Order = self.master2Order masterField = ssdo.masterField weightsFile = self.weightsFile potentialField = self.potentialField #### Assure that Variance is Larger than Zero #### yVar = NUM.var(self.y) if NUM.isnan(yVar) or yVar <= 0.0: ARCPY.AddIDMessage("ERROR", 906) raise SystemExit() #### Create Summed Variables #### self.intRange = NUM.arange(numObs) self.floatN = self.numObs * 1.0 ySum = self.y.sum() ySum2 = (self.y**2.0).sum() self.yBar = ySum / self.floatN self.S = NUM.sqrt((ySum2 / self.floatN) - self.yBar**2.0) self.nm1 = self.floatN - 1.0 #### Create Base Data Structures/Variables #### self.gi = NUM.zeros(numObs) self.pVals = NUM.ones(numObs) if self.permutations: self.pseudoPVals = NUM.ones(numObs) #### Set Neighborhood Structure Type #### if self.weightsFile: if self.swmFileBool: #### Open Spatial Weights and Obtain Chars #### swm = WU.SWMReader(weightsFile) N = swm.numObs rowStandard = swm.rowStandard self.swm = swm #### Check to Assure Complete Set of Weights #### if numObs > N: ARCPY.AddIDMessage("ERROR", 842, numObs, N) raise SystemExit() #### Check if Selection Set #### isSubSet = False if numObs < N: isSubSet = True iterVals = xrange(N) else: #### Warning for GWT with Bad Records/Selection #### if ssdo.selectionSet or ssdo.badRecords: ARCPY.AddIDMessage("WARNING", 1029) #### Build Weights Dictionary #### weightDict = WU.buildTextWeightDict(weightsFile, master2Order) iterVals = master2Order.keys() N = numObs elif wType in [4, 5]: #### Polygon Contiguity #### if wType == 4: contiguityType = "ROOK" else: contiguityType = "QUEEN" contDict = WU.polygonNeighborDict(ssdo.inputFC, ssdo.oidName, contiguityType=contiguityType) iterVals = master2Order.keys() N = numObs else: gaTable = ssdo.gaTable gaSearch = GAPY.ga_nsearch(gaTable) if wType == 7: #### Zone of Indiff, All Related to All #### gaSearch.init_nearest(threshold, numObs, gaConcept) else: #### Inverse and Fixed Distances #### gaSearch.init_nearest(threshold, self.numNeighs, gaConcept) iterVals = range(numObs) N = numObs neighWeights = ARC._ss.NeighborWeights(gaTable, gaSearch, weight_type=wType, exponent=exponent, row_standard=False, include_self=True) #### Create Progressor #### msg = ARCPY.GetIDMessage(84007) if self.permutations: msg += ": Using Permutations = %i" % self.permutations ARCPY.SetProgressor("step", msg, 0, N, 1) #### Create Neighbor Info Class #### ni = WU.NeighborInfo(masterField) #### Calculation For Each Feature #### for i in iterVals: if self.swmFileBool: #### Using SWM File #### info = swm.swm.readEntry() masterID = info[0] if master2Order.has_key(masterID): rowInfo = WU.getWeightsValuesSWM(info, master2Order, self.y, rowStandard=rowStandard, potVals=self.potVals) includeIt = True else: includeIt = False elif self.weightsFile and not self.swmFileBool: #### Text Weights #### masterID = i includeIt = True rowInfo = WU.getWeightsValuesText(masterID, master2Order, weightDict, self.y, potVals=self.potVals, allowSelf=True) elif wType in [4, 5]: #### Polygon Contiguity #### masterID = i includeIt = True rowInfo = WU.getWeightsValuesCont(masterID, master2Order, contDict, self.y, rowStandard=False, potVals=self.potVals) else: #### Distance Based #### masterID = gaTable[i][0] includeIt = True rowInfo = WU.getWeightsValuesOTF_Potent( neighWeights, i, self.y, self.potVals) #### Subset Boolean for SWM File #### if includeIt: #### Parse Row Info #### orderID, yiVal, nhIDs, nhVals, weights = rowInfo #### Assure Neighbors Exist After Selection #### nn, nhIDs, nhVals, weights = ni.processInfo( masterID, nhIDs, nhVals, weights) if nn: #### Calculate Local G #### self.calculateGI(orderID, yiVal, nhVals, weights) ARCPY.SetProgressorPosition() #### Clean Up #### if self.swmFileBool: swm.close() #### Report on Features with No Neighbors #### ni.reportNoNeighbors(failAllNoNeighs=False) self.setNullValues(ni.idsNoNeighs) #### Report on Features with Large Number of Neighbors #### ni.reportWarnings() ni.reportMaximums() self.neighInfo = ni #### Set p-values for Gi Bins #### if self.permutations: #### Use Pseudo p-values #### pv = self.pseudoPVals else: #### Use Traditional p-values #### pv = self.pVals toolMSG = ARCPY.GetIDMessage(84466) if self.applyFDR: #### Set Bins Using FDR #### msg = ARCPY.GetIDMessage(84472).format(toolMSG) ARCPY.SetProgressor("default", msg) self.giBins = STATS.fdrTransform(pv, self.gi) else: msg = ARCPY.GetIDMessage(84473).format(toolMSG) ARCPY.SetProgressor("default", msg) self.giBins = STATS.pValueBins(pv, self.gi)
def swm2Weights(ssdo, swmfile): """Converts ArcGIS Sparse Spatial Weights Matrix (*.swm) file to PySAL Sparse Spatial Weights Class. INPUTS: ssdo (class): instance of SSDataObject [1,2] swmFile (str): full path to swm file NOTES: (1) Data must already be obtained using ssdo.obtainData() (2) The masterField for the swm file and the ssdo object must be the same and may NOT be the OID/FID/ObjectID """ neighbors = {} weights = {} #### Create SWM Reader Object #### swm = WU.SWMReader(swmfile) #### SWM May NOT be a Subset of the Data #### if ssdo.numObs > swm.numObs: ARCPY.AddIDMessage("ERROR", 842, ssdo.numObs, swm.numObs) raise SystemExit() if swm.masterField != ssdo.masterField: ARCPY.AddWarning("ERROR", 938) raise SystemExit() #### Parse All SWM Records #### for r in UTILS.ssRange(swm.numObs): info = swm.swm.readEntry() masterID, nn, nhs, w, sumUnstandard = info #### Must Have at Least One Neighbor #### if nn: #### Must be in Selection Set (If Exists) #### if masterID in ssdo.master2Order: outNHS = [] outW = [] #### Transform Master ID to Order ID #### orderID = ssdo.master2Order[masterID] #### Neighbors and Weights Adjusted for Selection #### for nhInd, nhVal in enumerate(nhs): try: nhOrder = ssdo.master2Order[nhVal] outNHS.append(nhOrder) weightVal = w[nhInd] if swm.rowStandard: weightVal = weightVal * sumUnstandard[0] outW.append(weightVal) except KeyError: pass #### Add Selected Neighbors/Weights #### if len(outNHS): neighbors[orderID] = outNHS weights[orderID] = outW swm.close() #### Construct PySAL Spatial Weights and Standardize as per SWM #### w = PYSAL.W(neighbors, weights) if swm.rowStandard: w.transform = 'R' return w
def construct(self): """Constructs the neighborhood structure for each feature and dispatches the appropriate values for the calculation of the statistic.""" #### Shorthand Attributes #### ssdo = self.ssdo varName = self.varName concept = self.concept gaConcept = concept.lower() threshold = self.threshold exponent = self.exponent wType = self.wType rowStandard = self.rowStandard numObs = self.numObs master2Order = self.master2Order masterField = ssdo.masterField weightsFile = self.weightsFile #### Check That All Input Values are Positive #### if NUM.sum(self.y < 0.0) != 0: ARCPY.AddIDMessage("Error", 915) raise SystemExit() #### Assure that Variance is Larger than Zero #### yVar = NUM.var(self.y) if NUM.isnan(yVar) or yVar <= 0.0: ARCPY.AddIDMessage("Error", 906) raise SystemExit() #### Create Base Data Structures/Variables #### self.numer = 0.0 self.denom = 0.0 self.rowSum = NUM.zeros(numObs) self.colSum = NUM.zeros(numObs) self.ySum = 0.0 self.y2Sum = 0.0 self.y3Sum = 0.0 self.y4Sum = 0.0 self.s0 = 0 self.s1 = 0 self.wij = {} #### Set Neighborhood Structure Type #### if self.weightsFile: if self.swmFileBool: #### Open Spatial Weights and Obtain Chars #### swm = WU.SWMReader(weightsFile) N = swm.numObs rowStandard = swm.rowStandard #### Check to Assure Complete Set of Weights #### if numObs > N: ARCPY.AddIDMessage("Error", 842, numObs, N) raise SystemExit() #### Check if Selection Set #### isSubSet = False if numObs < N: isSubSet = True iterVals = xrange(N) else: #### Warning for GWT with Bad Records/Selection #### if ssdo.selectionSet or ssdo.badRecords: ARCPY.AddIDMessage("WARNING", 1029) #### Build Weights Dictionary #### weightDict = WU.buildTextWeightDict(weightsFile, master2Order) iterVals = master2Order.iterkeys() N = numObs elif wType in [4, 5]: #### Polygon Contiguity #### if wType == 4: contiguityType = "ROOK" else: contiguityType = "QUEEN" contDict = WU.polygonNeighborDict(ssdo.inputFC, ssdo.oidName, contiguityType = contiguityType) iterVals = master2Order.keys() N = numObs else: gaTable = ssdo.gaTable gaSearch = GAPY.ga_nsearch(gaTable) if wType == 7: #### Zone of Indiff, All Related to All #### gaSearch.init_nearest(threshold, numObs, gaConcept) else: #### Inverse and Fixed Distances #### gaSearch.init_nearest(threshold, 0, gaConcept) iterVals = xrange(numObs) N = numObs neighWeights = ARC._ss.NeighborWeights(gaTable, gaSearch, weight_type = wType, exponent = exponent, row_standard = rowStandard) #### Create Progressor #### ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84007), 0, N, 1) #### Create Neighbor Info Class #### ni = WU.NeighborInfo(masterField) #### Calculation For Each Feature #### for i in iterVals: if self.swmFileBool: #### Using SWM File #### info = swm.swm.readEntry() masterID = info[0] if master2Order.has_key(masterID): rowInfo = WU.getWeightsValuesSWM(info, master2Order, self.y, rowStandard = rowStandard, isSubSet = isSubSet) includeIt = True else: includeIt = False elif self.weightsFile and not self.swmFileBool: #### Text Weights #### masterID = i includeIt = True rowInfo = WU.getWeightsValuesText(masterID, master2Order, weightDict, self.y) elif wType in [4, 5]: #### Polygon Contiguity #### masterID = i includeIt = True rowInfo = WU.getWeightsValuesCont(masterID, master2Order, contDict, self.y, rowStandard = rowStandard) else: #### Distance Based #### masterID = gaTable[i][0] includeIt = True rowInfo = WU.getWeightsValuesOTF(neighWeights, i, self.y) #### Subset Boolean for SWM File #### if includeIt: #### Parse Row Info #### orderID, yiVal, nhIDs, nhVals, weights = rowInfo #### Assure Neighbors Exist After Selection #### nn, nhIDs, nhVals, weights = ni.processInfo(masterID, nhIDs, nhVals, weights) if nn: #### Process Feature Contribution to General G #### self.processRow(orderID, yiVal, nhIDs, nhVals, weights) #### Reset Progessor #### ARCPY.SetProgressorPosition() #### Clean Up #### if self.swmFileBool: swm.close() #### Report on Features with No Neighbors #### ni.reportNoNeighbors() #### Report on Features with Large Number of Neighbors #### ni.reportWarnings() ni.reportMaximums() self.neighInfo = ni
def swm2Table(swmFile, outputTable): """Places the spatial relationships contained in a given Spatial Weight Matrix File (*.swm) into a given output table. INPUTS: swmFile (str): Path to the input spatial weight matrix file outputTable (str): Path to the output database table """ #### Open Spatial Weights and Obtain Characteristics #### swm = WU.SWMReader(swmFile) masterField = swm.masterField N = swm.numObs rowStandard = swm.rowStandard #### Allow Overwrite Output #### ARCPY.env.overwriteOutput = True #### Get Output Table Name With Extension if Appropriate #### outputTable, dbf = UTILS.returnTableName(outputTable) #### Delete Table If Exists #### UTILS.passiveDelete(outputTable) #### Create Table #### outPath, outName = OS.path.split(outputTable) try: DM.CreateTable(outPath, outName) except: ARCPY.AddIDMessage("ERROR", 541) raise SystemExit() #### Create a List of Required Field Names #### fn = UTILS.getFieldNames(swm2TabFieldNames, outPath) neighFieldName, weightFieldName = fn fieldNames = [masterField, neighFieldName, weightFieldName] fieldTypes = ["LONG", "LONG", "DOUBLE"] for ind, field in enumerate(fieldNames): UTILS.addEmptyField(outputTable, field, fieldTypes[ind]) #### Create Insert Cursor #### try: insert = DA.InsertCursor(outputTable, fieldNames) except: ARCPY.AddIDMessage("ERROR", 204) raise SystemExit() #### Create Progressor #### ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84117), 0, N, 1) #### Process Spatial Weights File and Populate Output Table #### try: for r in xrange(N): info = swm.swm.readEntry() masterID, nn, nhs, weights, sumUnstandard = info if nn != 0: for ind, neigh in enumerate(nhs): row = [masterID, neigh, weights[ind]] insert.insertRow(row) ARCPY.SetProgressorPosition() except: swm.close() ARCPY.AddIDMessage("ERROR", 919) raise SystemExit() #### Clean Up #### del insert swm.close() #### Report if Any Features Have No Neighbors #### swm.reportNoNeighbors() #### Make Table Visable in TOC if *.dbf Had To Be Added #### if dbf: ARCPY.SetParameterAsText(1, outputTable)
def construct(self): """Constructs the neighborhood structure for each feature and dispatches the appropriate values for the calculation of the statistic.""" #### Shorthand Attributes #### ssdo = self.ssdo masterField = ssdo.masterField numObs = len(self.y) master2Order = self.ssdo.master2Order yVar = NUM.var(self.y) if NUM.isnan(yVar) or yVar <= 0.0: ARCPY.AddIDMessage("Error", 906) raise SystemExit() #### Create Deviation Variables #### self.yBar = NUM.mean(self.y) self.yDev = self.y - self.yBar #### Create Base Data Structures/Variables #### self.numer = 0.0 self.denom = NUM.sum(self.yDev**2.0) self.rowSum = NUM.zeros(numObs) self.colSum = NUM.zeros(numObs) self.s0 = 0 self.s1 = 0 self.wij = {} #### Open Spatial Weights and Obtain Chars #### if self.weightsType == "SWM": swm = WU.SWMReader(self.weightsMatrix) N = swm.numObs rowStandard = swm.rowStandard #### Check to Assure Complete Set of Weights #### if numObs > N: ARCPY.AddIDMessage("Error", 842, numObs, N) raise SystemExit() #### Check if Selection Set #### isSubSet = False if numObs < N: isSubSet = True iterVals = xrange(N) elif self.weightsType == "GWT": #### Warning for GWT with Bad Records/Selection #### if ssdo.selectionSet or ssdo.badRecords: ARCPY.AddIDMessage("WARNING", 1029) #### Build Weights Dictionary #### iterVals = master2Order.keys() N = numObs else: #### Use GA Table, 8 Nearest Neighbors #### iterVals = range(numObs) N = numObs neighWeights = ARC._ss.NeighborWeights(ssdo.gaTable, self.weightsMatrix) #### Create Neighbor Info Class #### ni = WU.NeighborInfo(masterField, silent=self.silent) #### Calculation For Each Feature #### for i in iterVals: if self.weightsType == "SWM": info = swm.swm.readEntry() masterID = info[0] if master2Order.has_key(masterID): rowInfo = WU.getWeightsValuesSWM(info, master2Order, self.yDev, rowStandard=rowStandard, isSubSet=isSubSet) includeIt = True else: includeIt = False elif self.weightsType == "GWT": #### Text Weights #### masterID = i includeIt = True rowInfo = WU.getWeightsValuesText(masterID, master2Order, self.weightsMatrix, self.yDev) else: #### Distance Based #### masterID = ssdo.gaTable[i][0] includeIt = True rowInfo = WU.getWeightsValuesOTF(neighWeights, i, self.yDev) #### Subset Boolean for SWM File #### if includeIt: #### Parse Row Info #### orderID, yiDev, nhIDs, nhVals, weights = rowInfo #### Assure Neighbors Exist After Selection #### nn, nhIDs, nhVals, weights = ni.processInfo( masterID, nhIDs, nhVals, weights) if nn: #### Process Feature Contribution to Moran's I #### self.processRow(orderID, yiDev, nhIDs, nhVals, weights) #### Clean Up #### if self.weightsType == "SWM": swm.close() if not self.silent: #### Report on Features with No Neighbors #### ni.reportNoNeighbors() #### Report on Features with Large Number of Neighbors #### ni.reportWarnings() ni.reportMaximums() self.neighInfo = ni