def spaceTime2SWM(inputFC, swmFile, masterField, concept = "EUCLIDEAN", threshold = None, rowStandard = True, timeField = None, timeType = None, timeValue = None): """ inputFC (str): path to the input feature class swmFile (str): path to the SWM file. masterField (str): field in table that serves as the mapping. concept: {str, EUCLIDEAN}: EUCLIDEAN or MANHATTAN threshold {float, None}: distance threshold rowStandard {bool, True}: row standardize weights? timeField {str, None}: name of the date-time field timeType {str, None}: ESRI enumeration of date-time intervals timeValue {float, None}: value forward and backward in time """ #### Assure Temporal Parameters are Set #### if not timeField: ARCPY.AddIDMessage("ERROR", 1320) raise SystemExit() if not timeType: ARCPY.AddIDMessage("ERROR", 1321) raise SystemExit() if not timeValue or timeValue <= 0: ARCPY.AddIDMessage("ERROR", 1322) raise SystemExit() #### Create SSDataObject #### ssdo = SSDO.SSDataObject(inputFC, templateFC = inputFC, useChordal = True) cnt = UTILS.getCount(inputFC) ERROR.errorNumberOfObs(cnt, minNumObs = 2) ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84001), 0, cnt, 1) #### Validation of Master Field #### verifyMaster = ERROR.checkField(ssdo.allFields, masterField, types = [0,1]) badIDs = [] #### Create Temporal Hash #### timeInfo = {} xyCoords = NUM.empty((cnt, 2), float) #### Process Field Values #### fieldList = [masterField, "SHAPE@XY", timeField] try: rows = DA.SearchCursor(ssdo.catPath, fieldList, "", ssdo.spatialRefString) except: ARCPY.AddIDMessage("ERROR", 204) raise SystemExit() #### Add Data to GATable and Time Dictionary #### c = 0 for row in rows: badRow = False #### Assure Masterfield is Valid #### masterID = row[0] if masterID == None or masterID == "": badRow = True #### Assure Date/Time is Valid #### timeStamp = row[-1] if timeStamp == None or timeStamp == "": badRow = True #### Assure Centroid is Valid #### badXY = row[1].count(None) if not badXY: x,y = row[1] xyCoords[c] = (x,y) else: badRow = True #### Process Data #### if not badRow: if timeInfo.has_key(masterID): #### Assure Uniqueness #### ARCPY.AddIDMessage("Error", 644, masterField) ARCPY.AddIDMessage("Error", 643) raise SystemExit() else: #### Fill Date/Time Dict #### startDT, endDT = TUTILS.calculateTimeWindow(timeStamp, timeValue, timeType) timeInfo[masterID] = (timeStamp, startDT, endDT) else: badIDs.append(masterID) #### Set Progress #### c += 1 ARCPY.SetProgressorPosition() #### Clean Up #### del rows #### Get Set of Bad IDs #### numBadObs = len(badIDs) badIDs = list(set(badIDs)) badIDs.sort() badIDs = [ str(i) for i in badIDs ] #### Process any bad records encountered #### if numBadObs: ERROR.reportBadRecords(cnt, numBadObs, badIDs, label = masterField) #### Load Neighbor Table #### gaTable, gaInfo = WU.gaTable(ssdo.inputFC, fieldNames = [masterField, timeField], spatRef = ssdo.spatialRefString) numObs = len(gaTable) xyCoords = xyCoords[0:numObs] #### Set the Distance Threshold #### concept, gaConcept = WU.validateDistanceMethod(concept, ssdo.spatialRef) if threshold == None: #### Set Progressor for Search #### ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84144)) #### Create k-Nearest Neighbor Search Type #### gaSearch = GAPY.ga_nsearch(gaTable) gaSearch.init_nearest(0.0, 1, gaConcept) neighDist = ARC._ss.NeighborDistances(gaTable, gaSearch) N = len(neighDist) threshold = 0.0 sumDist = 0.0 #### Find Maximum Nearest Neighbor Distance #### for row in xrange(N): dij = neighDist[row][-1][0] if dij > threshold: threshold = dij sumDist += dij ARCPY.SetProgressorPosition() #### Increase For Rounding Error #### threshold = threshold * 1.0001 avgDist = sumDist / (N * 1.0) #### Add Linear/Angular Units #### thresholdStr = ssdo.distanceInfo.printDistance(threshold) ARCPY.AddIDMessage("Warning", 853, thresholdStr) #### Chordal Default Check #### if ssdo.useChordal: hardMaxExtent = ARC._ss.get_max_gcs_distance(ssdo.spatialRef) if threshold > hardMaxExtent: ARCPY.AddIDMessage("ERROR", 1609) raise SystemExit() #### Clean Up #### del gaSearch #### Create Missing SSDO Info #### extent = UTILS.resetExtent(xyCoords) #### Reset Coordinates for Chordal #### if ssdo.useChordal: sliceInfo = UTILS.SpheroidSlice(extent, ssdo.spatialRef) maxExtent = sliceInfo.maxExtent else: env = UTILS.Envelope(extent) maxExtent = env.maxExtent threshold = checkDistanceThresholdSWM(ssdo, threshold, maxExtent) #### Set Default Progressor for Neigborhood Structure #### ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84143)) #### Create Distance Neighbor Search Type #### gaSearch = GAPY.ga_nsearch(gaTable) gaSearch.init_nearest(threshold, 0, gaConcept) neighSearch = ARC._ss.NeighborSearch(gaTable, gaSearch) #### Set Progressor for Weights Writing #### ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84127), 0, numObs, 1) #### Initialize Spatial Weights Matrix File #### swmWriter = WU.SWMWriter(swmFile, masterField, ssdo.spatialRefName, numObs, rowStandard, inputFC = inputFC, wType = 9, distanceMethod = concept, threshold = threshold, timeField = timeField, timeType = timeType, timeValue = timeValue) for row in xrange(numObs): masterID = gaTable[row][2] #### Get Date/Time Info #### dt0, startDT0, endDT0 = timeInfo[masterID] nhs = neighSearch[row] neighs = [] weights = [] for nh in nhs: #### Search Through Spatial Neighbors #### neighID = gaTable[nh][2] #### Get Date/Time Info #### dt1, startDT1, endDT1 = timeInfo[neighID] #### Filter Based on Date/Time #### insideTimeWindow = TUTILS.isTimeNeighbor(startDT0, endDT0, dt1) if insideTimeWindow: neighs.append(neighID) weights.append(1.0) #### Add Spatial Weights Matrix Entry #### swmWriter.swm.writeEntry(masterID, neighs, weights) #### Set Progress #### ARCPY.SetProgressorPosition() swmWriter.close() del gaTable #### Report Warning/Max Neighbors #### swmWriter.reportNeighInfo() #### Report Spatial Weights Summary #### swmWriter.report() #### Report SWM File is Large #### swmWriter.reportLargeSWM()
def kNearest2SWM(inputFC, swmFile, masterField, concept = "EUCLIDEAN", kNeighs = 1, rowStandard = True): """Creates a sparse spatial weights matrix (SWM) based on k-nearest neighbors. INPUTS: inputFC (str): path to the input feature class swmFile (str): path to the SWM file. masterField (str): field in table that serves as the mapping. concept: {str, EUCLIDEAN}: EUCLIDEAN or MANHATTAN kNeighs {int, 1}: number of neighbors to return rowStandard {bool, True}: row standardize weights? """ #### Assure that kNeighs is Non-Zero #### if kNeighs <= 0: ARCPY.AddIDMessage("ERROR", 976) raise SystemExit() #### Set Default Progressor for Neigborhood Structure #### ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84143)) #### Create SSDataObject #### ssdo = SSDO.SSDataObject(inputFC, templateFC = inputFC, useChordal = True) cnt = UTILS.getCount(inputFC) ERROR.errorNumberOfObs(cnt, minNumObs = 2) #### Validation of Master Field #### verifyMaster = ERROR.checkField(ssdo.allFields, masterField, types = [0,1]) #### Create GA Data Structure #### gaTable, gaInfo = WU.gaTable(ssdo.catPath, [masterField], spatRef = ssdo.spatialRefString) #### Assure Enough Observations #### N = gaInfo[0] ERROR.errorNumberOfObs(N, minNumObs = 2) #### Process any bad records encountered #### numBadRecs = cnt - N if numBadRecs: badRecs = WU.parseGAWarnings(gaTable.warnings) err = ERROR.reportBadRecords(cnt, numBadRecs, badRecs, label = ssdo.oidName) #### Assure k-Nearest is Less Than Number of Features #### if kNeighs >= N: ARCPY.AddIDMessage("ERROR", 975) raise SystemExit() #### Create k-Nearest Neighbor Search Type #### gaSearch = GAPY.ga_nsearch(gaTable) concept, gaConcept = WU.validateDistanceMethod(concept, ssdo.spatialRef) gaSearch.init_nearest(0.0, kNeighs, gaConcept) neighWeights = ARC._ss.NeighborWeights(gaTable, gaSearch, weight_type = 1, row_standard = False) #### Set Progressor for Weights Writing #### ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84127), 0, N, 1) #### Initialize Spatial Weights Matrix File #### swmWriter = WU.SWMWriter(swmFile, masterField, ssdo.spatialRefName, N, rowStandard, inputFC = inputFC, wType = 2, distanceMethod = concept, numNeighs = kNeighs) #### Unique Master ID Dictionary #### masterSet = set([]) for row in xrange(N): masterID = int(gaTable[row][2]) if masterID in masterSet: ARCPY.AddIDMessage("Error", 644, masterField) ARCPY.AddIDMessage("Error", 643) raise SystemExit() else: masterSet.add(masterID) neighs, weights = neighWeights[row] neighs = [ gaTable[nh][2] for nh in neighs ] #### Add Spatial Weights Matrix Entry #### swmWriter.swm.writeEntry(masterID, neighs, weights) #### Set Progress #### ARCPY.SetProgressorPosition() swmWriter.close() del gaTable #### Report Warning/Max Neighbors #### swmWriter.reportNeighInfo() #### Report Spatial Weights Summary #### swmWriter.report() #### Report SWM File is Large #### swmWriter.reportLargeSWM()
def distance2SWM(inputFC, swmFile, masterField, fixed = 0, concept = "EUCLIDEAN", exponent = 1.0, threshold = None, kNeighs = 1, rowStandard = True): """Creates a sparse spatial weights matrix (SWM) based on k-nearest neighbors. INPUTS: inputFC (str): path to the input feature class swmFile (str): path to the SWM file. masterField (str): field in table that serves as the mapping. fixed (boolean): fixed (1) or inverse (0) distance? concept: {str, EUCLIDEAN}: EUCLIDEAN or MANHATTAN exponent {float, 1.0}: distance decay threshold {float, None}: distance threshold kNeighs (int): number of neighbors to return rowStandard {bool, True}: row standardize weights? """ #### Create SSDataObject #### ssdo = SSDO.SSDataObject(inputFC, templateFC = inputFC, useChordal = True) #### Validation of Master Field #### verifyMaster = ERROR.checkField(ssdo.allFields, masterField, types = [0,1]) #### Read Data #### ssdo.obtainDataGA(masterField, minNumObs = 2) N = ssdo.numObs gaTable = ssdo.gaTable if fixed: wType = 1 else: wType = 0 #### Set Default Progressor for Neigborhood Structure #### ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84143)) #### Set the Distance Threshold #### concept, gaConcept = WU.validateDistanceMethod(concept, ssdo.spatialRef) if threshold == None: threshold, avgDist = WU.createThresholdDist(ssdo, concept = concept) #### Assures that the Threshold is Appropriate #### gaExtent = UTILS.get92Extent(ssdo.extent) threshold, maxSet = WU.checkDistanceThreshold(ssdo, threshold, weightType = wType) #### If the Threshold is Set to the Max #### #### Set to Zero for Script Logic #### if maxSet: #### All Locations are Related #### threshold = SYS.maxint if N > 500: ARCPY.AddIDMessage("Warning", 717) #### Assure k-Nearest is Less Than Number of Features #### if kNeighs >= N and fixed: ARCPY.AddIDMessage("ERROR", 975) raise SystemExit() #### Create Distance/k-Nearest Neighbor Search Type #### gaSearch = GAPY.ga_nsearch(gaTable) gaSearch.init_nearest(threshold, kNeighs, gaConcept) neighWeights = ARC._ss.NeighborWeights(gaTable, gaSearch, weight_type = wType, exponent = exponent, row_standard = False) #### Set Progressor for Weights Writing #### ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84127), 0, N, 1) #### Initialize Spatial Weights Matrix File #### swmWriter = WU.SWMWriter(swmFile, masterField, ssdo.spatialRefName, N, rowStandard, inputFC = inputFC, wType = wType, distanceMethod = concept, exponent = exponent, threshold = threshold) #### Unique Master ID Dictionary #### masterDict = {} #### Unique Master ID Dictionary #### masterSet = set([]) for row in xrange(N): masterID = int(gaTable[row][2]) if masterID in masterSet: ARCPY.AddIDMessage("Error", 644, masterField) ARCPY.AddIDMessage("Error", 643) raise SystemExit() else: masterSet.add(masterID) neighs, weights = neighWeights[row] neighs = [ gaTable[nh][2] for nh in neighs ] #### Add Spatial Weights Matrix Entry #### swmWriter.swm.writeEntry(masterID, neighs, weights) #### Set Progress #### ARCPY.SetProgressorPosition() swmWriter.close() del gaTable #### Report Warning/Max Neighbors #### swmWriter.reportNeighInfo() #### Add Linear/Angular Unit (Distance Based Only) #### distanceOut = ssdo.distanceInfo.outputString distanceOut = [ARCPY.GetIDMessage(84344).format(distanceOut)] #### Report Spatial Weights Summary #### swmWriter.report(additionalInfo = distanceOut) #### Report SWM File is Large #### swmWriter.reportLargeSWM()
def polygon2SWM(inputFC, swmFile, masterField, concept = "EUCLIDEAN", kNeighs = 0, rowStandard = True, contiguityType = "ROOK"): """Creates a sparse spatial weights matrix (SWM) based on polygon contiguity. INPUTS: inputFC (str): path to the input feature class swmFile (str): path to the SWM file. masterField (str): field in table that serves as the mapping. concept: {str, EUCLIDEAN}: EUCLIDEAN or MANHATTAN kNeighs {int, 0}: number of neighbors to return (1) rowStandard {bool, True}: row standardize weights? contiguityType {str, Rook}: {Rook = Edges Only, Queen = Edges/Vertices} NOTES: (1) kNeighs is used if polygon is not contiguous. E.g. Islands """ #### Set Default Progressor for Neigborhood Structure #### ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84143)) #### Create SSDataObject #### ssdo = SSDO.SSDataObject(inputFC, templateFC = inputFC, useChordal = True) cnt = UTILS.getCount(inputFC) ERROR.errorNumberOfObs(cnt, minNumObs = 2) #### Validation of Master Field #### verifyMaster = ERROR.checkField(ssdo.allFields, masterField, types = [0,1]) #### Create GA Data Structure #### gaTable, gaInfo = WU.gaTable(ssdo.catPath, [masterField], spatRef = ssdo.spatialRefString) #### Assure Enough Observations #### N = gaInfo[0] ERROR.errorNumberOfObs(N, minNumObs = 2) #### Assure k-Nearest is Less Than Number of Features #### if kNeighs >= N: ARCPY.AddIDMessage("ERROR", 975) raise SystemExit() #### Create Nearest Neighbor Search Type For Islands #### if kNeighs > 0: gaSearch = GAPY.ga_nsearch(gaTable) concept, gaConcept = WU.validateDistanceMethod(concept, ssdo.spatialRef) gaSearch.init_nearest(0.0, kNeighs, gaConcept) forceNeighbor = True neighWeights = ARC._ss.NeighborWeights(gaTable, gaSearch, weight_type = 1, row_standard = False) else: forceNeighbor = False neighSearch = None #### Create Polygon Neighbors #### polyNeighborDict = WU.polygonNeighborDict(inputFC, masterField, contiguityType = contiguityType) #### Write Poly Neighbor List (Dict) #### #### Set Progressor for SWM Writing #### ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84127), 0, N, 1) #### Initialize Spatial Weights Matrix File #### if contiguityType == "ROOK": wType = 4 else: wType = 5 swmWriter = WU.SWMWriter(swmFile, masterField, ssdo.spatialRefName, N, rowStandard, inputFC = inputFC, wType = wType, distanceMethod = concept, numNeighs = kNeighs) #### Keep Track of Polygons w/o Neighbors #### islandPolys = [] #### Write Polygon Contiguity to SWM File #### for row in xrange(N): rowInfo = gaTable[row] oid = rowInfo[0] masterID = rowInfo[2] neighs = polyNeighborDict[masterID] nn = len(neighs) if forceNeighbor: if nn < kNeighs: #### Only Force KNN If Specified & Contiguity is Less #### islandPolys.append(oid) flag = True knnNeighs, knnWeights = neighWeights[row] c = 0 while flag: try: neighID = gaTable[knnNeighs[c]][2] if neighID not in neighs: neighs.append(neighID) nn += 1 if nn == kNeighs: flag = False c += 1 except: flag = False weights = NUM.ones(nn) #### Add Weights Entry #### swmWriter.swm.writeEntry(masterID, neighs, weights) #### Set Progress #### ARCPY.SetProgressorPosition() #### Report on Features with No Neighbors #### countIslands = len(islandPolys) if countIslands: islandPolys.sort() if countIslands > 30: islandPolys = islandPolys[0:30] ERROR.warningNoNeighbors(N, countIslands, islandPolys, ssdo.oidName, forceNeighbor = forceNeighbor, contiguity = True) #### Clean Up #### swmWriter.close() del gaTable #### Report Spatial Weights Summary #### swmWriter.report() #### Report SWM File is Large #### swmWriter.reportLargeSWM() del polyNeighborDict
def delaunay2SWM(inputFC, swmFile, masterField, rowStandard = True): """Creates a sparse spatial weights matrix (SWM) based on Delaunay Triangulation. INPUTS: inputFC (str): path to the input feature class swmFile (str): path to the SWM file. masterField (str): field in table that serves as the mapping. rowStandard {bool, True}: row standardize weights? """ #### Set Default Progressor for Neigborhood Structure #### ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84143)) #### Create SSDataObject #### ssdo = SSDO.SSDataObject(inputFC, templateFC = inputFC, useChordal = True) cnt = UTILS.getCount(inputFC) ERROR.errorNumberOfObs(cnt, minNumObs = 2) #### Validation of Master Field #### verifyMaster = ERROR.checkField(ssdo.allFields, masterField, types = [0,1]) #### Create GA Data Structure #### gaTable, gaInfo = WU.gaTable(ssdo.catPath, [masterField], spatRef = ssdo.spatialRefString) #### Assure Enough Observations #### N = gaInfo[0] ERROR.errorNumberOfObs(N, minNumObs = 2) #### Process any bad records encountered #### numBadRecs = cnt - N if numBadRecs: badRecs = WU.parseGAWarnings(gaTable.warnings) err = ERROR.reportBadRecords(cnt, numBadRecs, badRecs, label = ssdo.oidName) #### Create Delaunay Neighbor Search Type #### gaSearch = GAPY.ga_nsearch(gaTable) gaSearch.init_delaunay() neighWeights = ARC._ss.NeighborWeights(gaTable, gaSearch, weight_type = 1, row_standard = False) #### Set Progressor for Weights Writing #### ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84127), 0, N, 1) #### Initialize Spatial Weights Matrix File #### swmWriter = WU.SWMWriter(swmFile, masterField, ssdo.spatialRefName, N, rowStandard, inputFC = inputFC, wType = 3) #### Unique Master ID Dictionary #### masterSet = set([]) for row in xrange(N): masterID = int(gaTable[row][2]) if masterID in masterSet: ARCPY.AddIDMessage("Error", 644, masterField) ARCPY.AddIDMessage("Error", 643) raise SystemExit() else: masterSet.add(masterID) neighs, weights = neighWeights[row] neighs = [ gaTable[nh][2] for nh in neighs ] #### Add Spatial Weights Matrix Entry #### swmWriter.swm.writeEntry(masterID, neighs, weights) #### Set Progress #### ARCPY.SetProgressorPosition() #### Clean Up #### swmWriter.close() del gaTable #### Report if Any Features Have No Neighbors #### swmWriter.reportNoNeighbors() #### Report Spatial Weights Summary #### swmWriter.report() #### Report SWM File is Large #### swmWriter.reportLargeSWM()
def obtainDataGA(self, masterField, fields = [], types = [0,1,2,3,5,6], minNumObs = 0, warnNumObs = 0): """Takes a list of field names and returns it in a dictionary structure. INPUTS: masterField (str): name of field being used as the master fields {list, []}: name(s) of the field to be returned types (list): types of data allowed to be returned (1) minNumObs {int, 0}: minimum number of observations for error warnNumObs {int, 0}: minimum number of observations for warning ATTRIBUTES: gaTable (structure): instance of the GA Table fields (dict): fieldName = instance of FCField master2Order (dict): masterID = order in lists order2Master (dict): order in lists = masterID masterField (str): field that serves as the master badRecords (list): master IDs that could not be read xyCoords (array, nunObs x 2): xy-coordinates for feature centroids NOTES: (1) No Text Fields; short [0], long [1], float [2], double[3] """ #### Validation of Master Field #### verifyMaster = ERROR.checkField(self.allFields, masterField, types = [0,1,5]) #### Set MasterIsOID Boolean #### self.masterIsOID = masterField == self.oidName #### Set Master and Data Indices #### if self.masterIsOID: self.masterColumnIndex = 0 self.dataColumnIndex = 2 fieldList = [] else: self.masterColumnIndex = 2 self.dataColumnIndex = 3 fieldList = [masterField] #### Validation and Initialization of Data Fields #### numFields = len(fields) for field in fields: fType = ERROR.checkField(self.allFields, field, types = types) fieldList.append(field) self.fields[field] = self.allFields[field] #### ZCoords Are Last #### getZBool = self.hasZ and (not self.renderType) if getZBool: fieldList.append("SHAPE&Z") #### Create GA Data Structure #### cnt = UTILS.getCount(self.inputFC) fieldList = tuple(fieldList) gaTable, gaInfo = WU.gaTable(self.inputFC, fieldNames = fieldList, spatRef = self.spatialRefString) #### Check Whether the Number of Features is Appropriate #### numObs = gaInfo[0] ERROR.checkNumberOfObs(numObs, minNumObs = minNumObs, warnNumObs = warnNumObs, silentWarnings = self.silentWarnings) #### Process any bad records encountered #### numBadIDs = cnt - numObs if numBadIDs: badIDs = WU.parseGAWarnings(gaTable.warnings) if not self.silentWarnings: ERROR.reportBadRecords(cnt, numBadIDs, badIDs, label = self.oidName) else: badIDs = [] #### Initialization of Centroids #### xyCoords = NUM.empty((numObs, 2), float) #### Z Coords #### if self.hasZ: zCoords = NUM.empty((numObs, ), float) #### Create Empty Data Arrays #### for fieldName, fieldObj in self.fields.iteritems(): fieldObj.createDataArray(numObs) #### Populate SSDataObject #### ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84001), 0, numObs, 1) for row in xrange(numObs): rowInfo = gaTable[row] x,y = rowInfo[1] masterID = int(rowInfo[self.masterColumnIndex]) if self.master2Order.has_key(masterID): ARCPY.AddIDMessage("ERROR", 644, masterField) ARCPY.AddIDMessage("ERROR", 643) raise SystemExit() else: self.master2Order[masterID] = row self.order2Master[row] = masterID xyCoords[row] = (x, y) if numFields: restFields = rowInfo[self.dataColumnIndex:] for fieldInd, fieldName in enumerate(fields): self.fields[fieldName].data[row] = restFields[fieldInd] if self.hasZ: if getZBool: zCoords[row] = rowInfo[-1] else: zCoords[row] = NUM.nan ARCPY.SetProgressorPosition() #### Set the Hidden Fields (E.g. Not in Use) #### self.setHiddenFields() #### Reset Extent to Honor Env and Subsets #### try: self.extent = UTILS.resetExtent(xyCoords) except: pass #### Reset Coordinates for Chordal #### if self.useChordal: #### Project to XY on Spheroid #### self.spheroidCoords = ARC._ss.lonlat_to_xy(xyCoords, self.spatialRef) self.sliceInfo = UTILS.SpheroidSlice(self.extent, self.spatialRef) else: self.spheroidCoords = None self.sliceInfo = None #### Set Further Attributes #### self.badRecords = badIDs self.xyCoords = xyCoords self.masterField = masterField self.gaTable = gaTable self.numObs = numObs if self.hasZ: self.zCoords = zCoords else: self.zCoords = None
def obtainData(self, masterField, fields = [], types = [0,1,2,3,4,5,6], minNumObs = 0, warnNumObs = 0, dateStr = False, explicitBadRecordID = None): """Takes a list of field names and returns it in a dictionary structure. INPUTS: masterField (str): name of field being used as the master fields {list, []}: name(s) of the field to be returned types (list): types of data allowed to be returned (1) minNumObs {int, 0}: minimum number of observations for error warnNumObs {int, 0}: minimum number of observations for warning OID {bool, False}: OID field allowed to be master field? ATTRIBUTES: gaTable (structure): instance of the GA Table fields (dict): fieldName = instance of FCField master2Order (dict): masterID = order in lists order2Master (dict): order in lists = masterID masterField (str): field that serves as the master badRecords (list): master IDs that could not be read xyCoords (array, nunObs x 2): xy-coordinates for feature centroids """ #### Get Base Count, May Include Bad Records #### cnt = UTILS.getCount(self.inputFC) #### Validation of Master Field #### verifyMaster = ERROR.checkField(self.allFields, masterField, types = [0,1,5]) #### Set MasterIsOID Boolean #### self.masterIsOID = masterField == self.oidName #### Set Master and Data Indices #### if self.masterIsOID: self.masterColumnIndex = 0 self.dataColumnIndex = 2 fieldList = [self.oidName, "shape@XY"] else: self.masterColumnIndex = 2 self.dataColumnIndex = 3 fieldList = [self.oidName, "shape@XY", masterField] #### Initialization of Centroids #### xyCoords = NUM.empty((cnt, 2), float) #### Validation and Initialization of Data Fields #### numFields = len(fields) fieldTypes = {} hasDate = False for field in fields: fieldType = ERROR.checkField(self.allFields, field, types = types) fieldTypes[field] = fieldType fieldList.append(field) self.fields[field] = self.allFields[field] if fieldType.upper() == "DATE": hasDate = True nowTime = DT.datetime.now() #### Create Empty Data Arrays #### for fieldName, fieldObj in self.fields.iteritems(): fieldObj.createDataArray(cnt, dateStr = dateStr) #### Z Coords #### if self.hasZ: zCoords = NUM.empty((cnt, ), float) fieldList.append("shape@Z") #### Keep track of Invalid Fields #### badIDs = [] badRecord = 0 #### Create Progressor Bar #### ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84001), 0, cnt, 1) #### Process Field Values #### try: rows = DA.SearchCursor(self.inputFC, fieldList, "", self.spatialRefString) except: ARCPY.AddIDMessage("ERROR", 204) raise SystemExit() c = 0 for row in rows: oid = row[0] badXY = row[1].count(None) if self.hasZ: badValues = row[0:-1].count(None) else: badValues = row.count(None) #### Check Bad Record #### if badXY or badValues: badRow = 1 badRecord = 1 badIDs.append(oid) else: #### Get Centroid and Master ID #### xyCoords[c] = row[1] masterID = row[self.masterColumnIndex] #### Add Field Values #### if numFields: restFields = row[self.dataColumnIndex:] for fieldInd, fieldName in enumerate(fields): fieldValue = restFields[fieldInd] fieldType = fieldTypes[fieldName] if fieldType.upper() == "DATE": if dateStr: fieldValue = str(fieldValue) else: fieldValue = (nowTime - fieldValue).total_seconds() self.fields[fieldName].data[c] = fieldValue if self.hasZ: zCoords[c] = row[-1] #### Check uniqueness of masterID field #### if self.master2Order.has_key(masterID): del rows ARCPY.AddIDMessage("ERROR", 644, masterField) ARCPY.AddIDMessage("ERROR", 643) raise SystemExit() else: self.master2Order[masterID] = c self.order2Master[c] = masterID c += 1 ARCPY.SetProgressorPosition() del rows #### Check Whether the Number of Features is Appropriate #### numObs = len(self.master2Order) ERROR.checkNumberOfObs(numObs, minNumObs = minNumObs, warnNumObs = warnNumObs, silentWarnings = self.silentWarnings) #### Get Set of Bad IDs #### badIDs = list(set(badIDs)) badIDs.sort() badIDs = [ str(i) for i in badIDs ] #### Process any bad records encountered #### if badRecord != 0: bn = len(badIDs) if not self.silentWarnings: ERROR.reportBadRecords(cnt, bn, badIDs, label = self.oidName, explicitBadRecordID = explicitBadRecordID) #### Prune Data Arrays #### xyCoords = xyCoords[0:numObs] self.resizeDataArrays(numObs) if self.hasZ: zCoords = zCoords[0:numObs] #### Set the Hidden Fields (E.g. Not in Use) #### self.setHiddenFields() #### Reset Extent to Honor Env and Subsets #### try: self.extent = UTILS.resetExtent(xyCoords) except: pass #### Reset Coordinates for Chordal #### if self.useChordal: #### Project to XY on Spheroid #### self.spheroidCoords = ARC._ss.lonlat_to_xy(xyCoords, self.spatialRef) self.sliceInfo = UTILS.SpheroidSlice(self.extent, self.spatialRef) else: self.spheroidCoords = None self.sliceInfo = None #### Set Further Attributes #### self.badRecords = badIDs self.xyCoords = xyCoords self.masterField = masterField self.gaTable = None self.numObs = numObs if self.hasZ: self.zCoords = zCoords else: self.zCoords = None
def obtainDataGA(self, masterField, fields=[], types=[0, 1, 2, 3, 5, 6], minNumObs=0, warnNumObs=0): """Takes a list of field names and returns it in a dictionary structure. INPUTS: masterField (str): name of field being used as the master fields {list, []}: name(s) of the field to be returned types (list): types of data allowed to be returned (1) minNumObs {int, 0}: minimum number of observations for error warnNumObs {int, 0}: minimum number of observations for warning ATTRIBUTES: gaTable (structure): instance of the GA Table fields (dict): fieldName = instance of FCField master2Order (dict): masterID = order in lists order2Master (dict): order in lists = masterID masterField (str): field that serves as the master badRecords (list): master IDs that could not be read xyCoords (array, nunObs x 2): xy-coordinates for feature centroids NOTES: (1) No Text Fields; short [0], long [1], float [2], double[3] """ #### Validation of Master Field #### verifyMaster = ERROR.checkField(self.allFields, masterField, types=[0, 1, 5]) #### Set MasterIsOID Boolean #### self.masterIsOID = masterField == self.oidName #### Set Master and Data Indices #### if self.masterIsOID: self.masterColumnIndex = 0 self.dataColumnIndex = 2 fieldList = [] else: self.masterColumnIndex = 2 self.dataColumnIndex = 3 fieldList = [masterField] #### Validation and Initialization of Data Fields #### numFields = len(fields) for field in fields: fType = ERROR.checkField(self.allFields, field, types=types) fieldList.append(field) self.fields[field] = self.allFields[field] #### ZCoords Are Last #### getZBool = self.hasZ and (not self.renderType) if getZBool: fieldList.append("SHAPE&Z") #### Create GA Data Structure #### cnt = UTILS.getCount(self.inputFC) fieldList = tuple(fieldList) gaTable, gaInfo = WU.gaTable(self.inputFC, fieldNames=fieldList, spatRef=self.spatialRefString) #### Check Whether the Number of Features is Appropriate #### numObs = gaInfo[0] ERROR.checkNumberOfObs(numObs, minNumObs=minNumObs, warnNumObs=warnNumObs, silentWarnings=self.silentWarnings) #### Process any bad records encountered #### numBadIDs = cnt - numObs if numBadIDs: badIDs = WU.parseGAWarnings(gaTable.warnings) if not self.silentWarnings: ERROR.reportBadRecords(cnt, numBadIDs, badIDs, label=self.oidName) else: badIDs = [] #### Initialization of Centroids #### xyCoords = NUM.empty((numObs, 2), float) #### Z Coords #### if self.hasZ: zCoords = NUM.empty((numObs, ), float) #### Create Empty Data Arrays #### for fieldName, fieldObj in self.fields.iteritems(): fieldObj.createDataArray(numObs) #### Populate SSDataObject #### ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84001), 0, numObs, 1) for row in xrange(numObs): rowInfo = gaTable[row] x, y = rowInfo[1] masterID = int(rowInfo[self.masterColumnIndex]) if self.master2Order.has_key(masterID): ARCPY.AddIDMessage("ERROR", 644, masterField) ARCPY.AddIDMessage("ERROR", 643) raise SystemExit() else: self.master2Order[masterID] = row self.order2Master[row] = masterID xyCoords[row] = (x, y) if numFields: restFields = rowInfo[self.dataColumnIndex:] for fieldInd, fieldName in enumerate(fields): self.fields[fieldName].data[row] = restFields[fieldInd] if self.hasZ: if getZBool: zCoords[row] = rowInfo[-1] else: zCoords[row] = NUM.nan ARCPY.SetProgressorPosition() #### Set the Hidden Fields (E.g. Not in Use) #### self.setHiddenFields() #### Reset Extent to Honor Env and Subsets #### try: self.extent = UTILS.resetExtent(xyCoords) except: pass #### Reset Coordinates for Chordal #### if self.useChordal: #### Project to XY on Spheroid #### self.spheroidCoords = ARC._ss.lonlat_to_xy(xyCoords, self.spatialRef) self.sliceInfo = UTILS.SpheroidSlice(self.extent, self.spatialRef) else: self.spheroidCoords = None self.sliceInfo = None #### Set Further Attributes #### self.badRecords = badIDs self.xyCoords = xyCoords self.masterField = masterField self.gaTable = gaTable self.numObs = numObs if self.hasZ: self.zCoords = zCoords else: self.zCoords = None
def obtainData(self, masterField, fields=[], types=[0, 1, 2, 3, 4, 5, 6], minNumObs=0, warnNumObs=0, dateStr=False, explicitBadRecordID=None): """Takes a list of field names and returns it in a dictionary structure. INPUTS: masterField (str): name of field being used as the master fields {list, []}: name(s) of the field to be returned types (list): types of data allowed to be returned (1) minNumObs {int, 0}: minimum number of observations for error warnNumObs {int, 0}: minimum number of observations for warning OID {bool, False}: OID field allowed to be master field? ATTRIBUTES: gaTable (structure): instance of the GA Table fields (dict): fieldName = instance of FCField master2Order (dict): masterID = order in lists order2Master (dict): order in lists = masterID masterField (str): field that serves as the master badRecords (list): master IDs that could not be read xyCoords (array, nunObs x 2): xy-coordinates for feature centroids """ #### Get Base Count, May Include Bad Records #### cnt = UTILS.getCount(self.inputFC) #### Validation of Master Field #### verifyMaster = ERROR.checkField(self.allFields, masterField, types=[0, 1, 5]) #### Set MasterIsOID Boolean #### self.masterIsOID = masterField == self.oidName #### Set Master and Data Indices #### if self.masterIsOID: self.masterColumnIndex = 0 self.dataColumnIndex = 2 fieldList = [self.oidName, "shape@XY"] else: self.masterColumnIndex = 2 self.dataColumnIndex = 3 fieldList = [self.oidName, "shape@XY", masterField] #### Initialization of Centroids #### xyCoords = NUM.empty((cnt, 2), float) #### Validation and Initialization of Data Fields #### numFields = len(fields) fieldTypes = {} hasDate = False for field in fields: fieldType = ERROR.checkField(self.allFields, field, types=types) fieldTypes[field] = fieldType fieldList.append(field) self.fields[field] = self.allFields[field] if fieldType.upper() == "DATE": hasDate = True nowTime = DT.datetime.now() #### Create Empty Data Arrays #### for fieldName, fieldObj in self.fields.iteritems(): fieldObj.createDataArray(cnt, dateStr=dateStr) #### Z Coords #### if self.hasZ: zCoords = NUM.empty((cnt, ), float) fieldList.append("shape@Z") #### Keep track of Invalid Fields #### badIDs = [] badRecord = 0 #### Create Progressor Bar #### ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84001), 0, cnt, 1) #### Process Field Values #### try: rows = DA.SearchCursor(self.inputFC, fieldList, "", self.spatialRefString) except: ARCPY.AddIDMessage("ERROR", 204) raise SystemExit() c = 0 for row in rows: oid = row[0] badXY = row[1].count(None) if self.hasZ: badValues = row[0:-1].count(None) else: badValues = row.count(None) #### Check Bad Record #### if badXY or badValues: badRow = 1 badRecord = 1 badIDs.append(oid) else: #### Get Centroid and Master ID #### xyCoords[c] = row[1] masterID = row[self.masterColumnIndex] #### Add Field Values #### if numFields: restFields = row[self.dataColumnIndex:] for fieldInd, fieldName in enumerate(fields): fieldValue = restFields[fieldInd] fieldType = fieldTypes[fieldName] if fieldType.upper() == "DATE": if dateStr: fieldValue = str(fieldValue) else: fieldValue = (nowTime - fieldValue).total_seconds() self.fields[fieldName].data[c] = fieldValue if self.hasZ: zCoords[c] = row[-1] #### Check uniqueness of masterID field #### if self.master2Order.has_key(masterID): del rows ARCPY.AddIDMessage("ERROR", 644, masterField) ARCPY.AddIDMessage("ERROR", 643) raise SystemExit() else: self.master2Order[masterID] = c self.order2Master[c] = masterID c += 1 ARCPY.SetProgressorPosition() del rows #### Check Whether the Number of Features is Appropriate #### numObs = len(self.master2Order) ERROR.checkNumberOfObs(numObs, minNumObs=minNumObs, warnNumObs=warnNumObs, silentWarnings=self.silentWarnings) #### Get Set of Bad IDs #### badIDs = list(set(badIDs)) badIDs.sort() badIDs = [str(i) for i in badIDs] #### Process any bad records encountered #### if badRecord != 0: bn = len(badIDs) if not self.silentWarnings: ERROR.reportBadRecords(cnt, bn, badIDs, label=self.oidName, explicitBadRecordID=explicitBadRecordID) #### Prune Data Arrays #### xyCoords = xyCoords[0:numObs] self.resizeDataArrays(numObs) if self.hasZ: zCoords = zCoords[0:numObs] #### Set the Hidden Fields (E.g. Not in Use) #### self.setHiddenFields() #### Reset Extent to Honor Env and Subsets #### try: self.extent = UTILS.resetExtent(xyCoords) except: pass #### Reset Coordinates for Chordal #### if self.useChordal: #### Project to XY on Spheroid #### self.spheroidCoords = ARC._ss.lonlat_to_xy(xyCoords, self.spatialRef) self.sliceInfo = UTILS.SpheroidSlice(self.extent, self.spatialRef) else: self.spheroidCoords = None self.sliceInfo = None #### Set Further Attributes #### self.badRecords = badIDs self.xyCoords = xyCoords self.masterField = masterField self.gaTable = None self.numObs = numObs if self.hasZ: self.zCoords = zCoords else: self.zCoords = None
def polygon2SWM(inputFC, swmFile, masterField, concept = "EUCLIDEAN", kNeighs = 0, rowStandard = True, contiguityType = "ROOK"): """Creates a sparse spatial weights matrix (SWM) based on polygon contiguity. INPUTS: inputFC (str): path to the input feature class swmFile (str): path to the SWM file. masterField (str): field in table that serves as the mapping. concept: {str, EUCLIDEAN}: EUCLIDEAN or MANHATTAN kNeighs {int, 0}: number of neighbors to return (1) rowStandard {bool, True}: row standardize weights? contiguityType {str, Rook}: {Rook = Edges Only, Queen = Edges/Vertices} NOTES: (1) kNeighs is used if polygon is not contiguous. E.g. Islands """ #### Set Default Progressor for Neigborhood Structure #### ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84143)) #### Create SSDataObject #### ssdo = SSDO.SSDataObject(inputFC, templateFC = inputFC, useChordal = True) cnt = UTILS.getCount(inputFC) ERROR.errorNumberOfObs(cnt, minNumObs = 2) #### Validation of Master Field #### verifyMaster = ERROR.checkField(ssdo.allFields, masterField, types = [0,1]) #### Create GA Data Structure #### gaTable, gaInfo = WU.gaTable(ssdo.catPath, [masterField], spatRef = ssdo.spatialRefString) #### Assure Enough Observations #### N = gaInfo[0] ERROR.errorNumberOfObs(N, minNumObs = 2) #### Assure k-Nearest is Less Than Number of Features #### if kNeighs >= N: ARCPY.AddIDMessage("ERROR", 975) raise SystemExit() #### Create Nearest Neighbor Search Type For Islands #### gaSearch = GAPY.ga_nsearch(gaTable) concept, gaConcept = WU.validateDistanceMethod(concept, ssdo.spatialRef) gaSearch.init_nearest(0.0, kNeighs, gaConcept) if kNeighs > 0: forceNeighbor = True neighWeights = ARC._ss.NeighborWeights(gaTable, gaSearch, weight_type = 1, row_standard = False) else: forceNeighbor = False neighSearch = None #### Create Polygon Neighbors #### polyNeighborDict = WU.polygonNeighborDict(inputFC, masterField, contiguityType = contiguityType) #### Write Poly Neighbor List (Dict) #### #### Set Progressor for SWM Writing #### ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84127), 0, N, 1) #### Initialize Spatial Weights Matrix File #### if contiguityType == "ROOK": wType = 4 else: wType = 5 swmWriter = WU.SWMWriter(swmFile, masterField, ssdo.spatialRefName, N, rowStandard, inputFC = inputFC, wType = wType, distanceMethod = concept, numNeighs = kNeighs) #### Keep Track of Polygons w/o Neighbors #### islandPolys = [] #### Write Polygon Contiguity to SWM File #### for row in xrange(N): rowInfo = gaTable[row] oid = rowInfo[0] masterID = rowInfo[2] neighs = polyNeighborDict[masterID] if neighs: weights = [ 1. for nh in neighs ] isIsland = False else: isIsland = True islandPolys.append(oid) weights = [] #### Get Nearest Neighbor Based On Centroid Distance #### if isIsland and forceNeighbor: neighs, weights = neighWeights[row] neighs = [ gaTable[nh][2] for nh in neighs ] #### Add Weights Entry #### swmWriter.swm.writeEntry(masterID, neighs, weights) #### Set Progress #### ARCPY.SetProgressorPosition() #### Report on Features with No Neighbors #### countIslands = len(islandPolys) if countIslands: islandPolys.sort() if countIslands > 30: islandPolys = islandPolys[0:30] ERROR.warningNoNeighbors(N, countIslands, islandPolys, ssdo.oidName, forceNeighbor = forceNeighbor, contiguity = True) #### Clean Up #### swmWriter.close() del gaTable #### Report Spatial Weights Summary #### swmWriter.report() #### Report SWM File is Large #### swmWriter.reportLargeSWM() del polyNeighborDict