def distance2Weights(ssdo, neighborType = 1, distanceBand = 0.0, numNeighs = 0,
distanceType = "euclidean", exponent = 1.0, rowStandard = True,
includeSelf = False):
"""Uses ArcGIS Neighborhood Searching Structure to create a PySAL Sparse Spatial
Weights Matrix.
INPUTS:
ssdo (class): instance of SSDataObject [1]
neighborType {int, 1}: 0 = inverse distance, 1 = fixed distance,
2 = k-nearest-neighbors, 3 = delaunay
distanceBand {float, 0.0}: return all neighbors within this distance for
inverse/fixed distance
numNeighs {int, 0}: number of neighbors for k-nearest-neighbor, can also
be used to set a minimum number of neighbors for
inverse/fixed distance
distanceType {str, euclidean}: manhattan or euclidean distance [2]
exponent {float, 1.0}: distance decay factor for inverse distance
rowStandard {bool, True}: whether to row standardize the spatial weights
includeSelf {bool, False}: whether to return self as a neighbor
NOTES:
(1) Data must already be obtained using ssdo.obtainDataGA()
(2) Chordal Distance is used for GCS Data
"""
neighbors = {}
weights = {}
gaSearch = GAPY.ga_nsearch(ssdo.gaTable)
if neighborType == 3:
gaSearch.init_delaunay()
neighSearch = ARC._ss.NeighborWeights(ssdo.gaTable, gaSearch, weight_type = 1)
else:
if neighborType == 2:
distanceBand = 0.0
weightType = 1
else:
weightType = neighborType
concept, gaConcept = WU.validateDistanceMethod(distanceType.upper(), ssdo.spatialRef)
gaSearch.init_nearest(distanceBand, numNeighs, gaConcept)
neighSearch = ARC._ss.NeighborWeights(ssdo.gaTable, gaSearch, weight_type = weightType,
exponent = exponent, include_self = includeSelf)
for i in range(len(neighSearch)):
neighOrderIDs, neighWeights = neighSearch[i]
neighbors[i] = neighOrderIDs
weights[i] = neighWeights
w = PYSAL.W(neighbors, weights)
if rowStandard:
w.transform = 'R'
return w
def distance2SWM(inputFC, swmFile, masterField, fixed = 0,
concept = "EUCLIDEAN", exponent = 1.0, threshold = None,
kNeighs = 1, rowStandard = True):
"""Creates a sparse spatial weights matrix (SWM) based on k-nearest
neighbors.
INPUTS:
inputFC (str): path to the input feature class
swmFile (str): path to the SWM file.
masterField (str): field in table that serves as the mapping.
fixed (boolean): fixed (1) or inverse (0) distance?
concept: {str, EUCLIDEAN}: EUCLIDEAN or MANHATTAN
exponent {float, 1.0}: distance decay
threshold {float, None}: distance threshold
kNeighs (int): number of neighbors to return
rowStandard {bool, True}: row standardize weights?
"""
#### Create SSDataObject ####
ssdo = SSDO.SSDataObject(inputFC, templateFC = inputFC,
useChordal = True)
#### Validation of Master Field ####
verifyMaster = ERROR.checkField(ssdo.allFields, masterField, types = [0,1])
#### Read Data ####
ssdo.obtainDataGA(masterField, minNumObs = 2)
N = ssdo.numObs
gaTable = ssdo.gaTable
if fixed:
wType = 1
else:
wType = 0
#### Set Default Progressor for Neigborhood Structure ####
ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84143))
#### Set the Distance Threshold ####
concept, gaConcept = WU.validateDistanceMethod(concept, ssdo.spatialRef)
if threshold == None:
threshold, avgDist = WU.createThresholdDist(ssdo,
concept = concept)
#### Assures that the Threshold is Appropriate ####
gaExtent = UTILS.get92Extent(ssdo.extent)
threshold, maxSet = WU.checkDistanceThreshold(ssdo, threshold,
weightType = wType)
#### If the Threshold is Set to the Max ####
#### Set to Zero for Script Logic ####
if maxSet:
#### All Locations are Related ####
threshold = SYS.maxint
if N > 500:
ARCPY.AddIDMessage("Warning", 717)
#### Assure k-Nearest is Less Than Number of Features ####
if kNeighs >= N and fixed:
ARCPY.AddIDMessage("ERROR", 975)
raise SystemExit()
#### Create Distance/k-Nearest Neighbor Search Type ####
gaSearch = GAPY.ga_nsearch(gaTable)
gaSearch.init_nearest(threshold, kNeighs, gaConcept)
neighWeights = ARC._ss.NeighborWeights(gaTable, gaSearch,
weight_type = wType,
exponent = exponent,
row_standard = False)
#### Set Progressor for Weights Writing ####
ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84127), 0, N, 1)
#### Initialize Spatial Weights Matrix File ####
swmWriter = WU.SWMWriter(swmFile, masterField, ssdo.spatialRefName,
N, rowStandard, inputFC = inputFC,
wType = wType, distanceMethod = concept,
exponent = exponent, threshold = threshold)
#### Unique Master ID Dictionary ####
masterDict = {}
#### Unique Master ID Dictionary ####
masterSet = set([])
for row in xrange(N):
masterID = int(gaTable[row][2])
if masterID in masterSet:
ARCPY.AddIDMessage("Error", 644, masterField)
ARCPY.AddIDMessage("Error", 643)
raise SystemExit()
else:
masterSet.add(masterID)
neighs, weights = neighWeights[row]
neighs = [ gaTable[nh][2] for nh in neighs ]
#### Add Spatial Weights Matrix Entry ####
swmWriter.swm.writeEntry(masterID, neighs, weights)
#### Set Progress ####
ARCPY.SetProgressorPosition()
swmWriter.close()
del gaTable
#### Report Warning/Max Neighbors ####
swmWriter.reportNeighInfo()
#### Add Linear/Angular Unit (Distance Based Only) ####
distanceOut = ssdo.distanceInfo.outputString
distanceOut = [ARCPY.GetIDMessage(84344).format(distanceOut)]
#### Report Spatial Weights Summary ####
swmWriter.report(additionalInfo = distanceOut)
#### Report SWM File is Large ####
swmWriter.reportLargeSWM()
def spaceTime2SWM(inputFC, swmFile, masterField, concept = "EUCLIDEAN",
threshold = None, rowStandard = True,
timeField = None, timeType = None,
timeValue = None):
"""
inputFC (str): path to the input feature class
swmFile (str): path to the SWM file.
masterField (str): field in table that serves as the mapping.
concept: {str, EUCLIDEAN}: EUCLIDEAN or MANHATTAN
threshold {float, None}: distance threshold
rowStandard {bool, True}: row standardize weights?
timeField {str, None}: name of the date-time field
timeType {str, None}: ESRI enumeration of date-time intervals
timeValue {float, None}: value forward and backward in time
"""
#### Assure Temporal Parameters are Set ####
if not timeField:
ARCPY.AddIDMessage("ERROR", 1320)
raise SystemExit()
if not timeType:
ARCPY.AddIDMessage("ERROR", 1321)
raise SystemExit()
if not timeValue or timeValue <= 0:
ARCPY.AddIDMessage("ERROR", 1322)
raise SystemExit()
#### Create SSDataObject ####
ssdo = SSDO.SSDataObject(inputFC, templateFC = inputFC,
useChordal = True)
cnt = UTILS.getCount(inputFC)
ERROR.errorNumberOfObs(cnt, minNumObs = 2)
ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84001), 0, cnt, 1)
#### Validation of Master Field ####
verifyMaster = ERROR.checkField(ssdo.allFields, masterField, types = [0,1])
badIDs = []
#### Create Temporal Hash ####
timeInfo = {}
xyCoords = NUM.empty((cnt, 2), float)
#### Process Field Values ####
fieldList = [masterField, "SHAPE@XY", timeField]
try:
rows = DA.SearchCursor(ssdo.catPath, fieldList, "",
ssdo.spatialRefString)
except:
ARCPY.AddIDMessage("ERROR", 204)
raise SystemExit()
#### Add Data to GATable and Time Dictionary ####
c = 0
for row in rows:
badRow = False
#### Assure Masterfield is Valid ####
masterID = row[0]
if masterID == None or masterID == "":
badRow = True
#### Assure Date/Time is Valid ####
timeStamp = row[-1]
if timeStamp == None or timeStamp == "":
badRow = True
#### Assure Centroid is Valid ####
badXY = row[1].count(None)
if not badXY:
x,y = row[1]
xyCoords[c] = (x,y)
else:
badRow = True
#### Process Data ####
if not badRow:
if timeInfo.has_key(masterID):
#### Assure Uniqueness ####
ARCPY.AddIDMessage("Error", 644, masterField)
ARCPY.AddIDMessage("Error", 643)
raise SystemExit()
else:
#### Fill Date/Time Dict ####
startDT, endDT = TUTILS.calculateTimeWindow(timeStamp,
timeValue,
timeType)
timeInfo[masterID] = (timeStamp, startDT, endDT)
else:
badIDs.append(masterID)
#### Set Progress ####
c += 1
ARCPY.SetProgressorPosition()
#### Clean Up ####
del rows
#### Get Set of Bad IDs ####
numBadObs = len(badIDs)
badIDs = list(set(badIDs))
badIDs.sort()
badIDs = [ str(i) for i in badIDs ]
#### Process any bad records encountered ####
if numBadObs:
ERROR.reportBadRecords(cnt, numBadObs, badIDs, label = masterField)
#### Load Neighbor Table ####
gaTable, gaInfo = WU.gaTable(ssdo.inputFC,
fieldNames = [masterField, timeField],
spatRef = ssdo.spatialRefString)
numObs = len(gaTable)
xyCoords = xyCoords[0:numObs]
#### Set the Distance Threshold ####
concept, gaConcept = WU.validateDistanceMethod(concept, ssdo.spatialRef)
if threshold == None:
#### Set Progressor for Search ####
ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84144))
#### Create k-Nearest Neighbor Search Type ####
gaSearch = GAPY.ga_nsearch(gaTable)
gaSearch.init_nearest(0.0, 1, gaConcept)
neighDist = ARC._ss.NeighborDistances(gaTable, gaSearch)
N = len(neighDist)
threshold = 0.0
sumDist = 0.0
#### Find Maximum Nearest Neighbor Distance ####
for row in xrange(N):
dij = neighDist[row][-1][0]
if dij > threshold:
threshold = dij
sumDist += dij
ARCPY.SetProgressorPosition()
#### Increase For Rounding Error ####
threshold = threshold * 1.0001
avgDist = sumDist / (N * 1.0)
#### Add Linear/Angular Units ####
thresholdStr = ssdo.distanceInfo.printDistance(threshold)
ARCPY.AddIDMessage("Warning", 853, thresholdStr)
#### Chordal Default Check ####
if ssdo.useChordal:
hardMaxExtent = ARC._ss.get_max_gcs_distance(ssdo.spatialRef)
if threshold > hardMaxExtent:
ARCPY.AddIDMessage("ERROR", 1609)
raise SystemExit()
#### Clean Up ####
del gaSearch
#### Create Missing SSDO Info ####
extent = UTILS.resetExtent(xyCoords)
#### Reset Coordinates for Chordal ####
if ssdo.useChordal:
sliceInfo = UTILS.SpheroidSlice(extent, ssdo.spatialRef)
maxExtent = sliceInfo.maxExtent
else:
env = UTILS.Envelope(extent)
maxExtent = env.maxExtent
threshold = checkDistanceThresholdSWM(ssdo, threshold, maxExtent)
#### Set Default Progressor for Neigborhood Structure ####
ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84143))
#### Create Distance Neighbor Search Type ####
gaSearch = GAPY.ga_nsearch(gaTable)
gaSearch.init_nearest(threshold, 0, gaConcept)
neighSearch = ARC._ss.NeighborSearch(gaTable, gaSearch)
#### Set Progressor for Weights Writing ####
ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84127), 0, numObs, 1)
#### Initialize Spatial Weights Matrix File ####
swmWriter = WU.SWMWriter(swmFile, masterField, ssdo.spatialRefName,
numObs, rowStandard, inputFC = inputFC,
wType = 9, distanceMethod = concept,
threshold = threshold, timeField = timeField,
timeType = timeType, timeValue = timeValue)
for row in xrange(numObs):
masterID = gaTable[row][2]
#### Get Date/Time Info ####
dt0, startDT0, endDT0 = timeInfo[masterID]
nhs = neighSearch[row]
neighs = []
weights = []
for nh in nhs:
#### Search Through Spatial Neighbors ####
neighID = gaTable[nh][2]
#### Get Date/Time Info ####
dt1, startDT1, endDT1 = timeInfo[neighID]
#### Filter Based on Date/Time ####
insideTimeWindow = TUTILS.isTimeNeighbor(startDT0, endDT0, dt1)
if insideTimeWindow:
neighs.append(neighID)
weights.append(1.0)
#### Add Spatial Weights Matrix Entry ####
swmWriter.swm.writeEntry(masterID, neighs, weights)
#### Set Progress ####
ARCPY.SetProgressorPosition()
swmWriter.close()
del gaTable
#### Report Warning/Max Neighbors ####
swmWriter.reportNeighInfo()
#### Report Spatial Weights Summary ####
swmWriter.report()
#### Report SWM File is Large ####
swmWriter.reportLargeSWM()
def polygon2SWM(inputFC, swmFile, masterField,
concept = "EUCLIDEAN", kNeighs = 0,
rowStandard = True, contiguityType = "ROOK"):
"""Creates a sparse spatial weights matrix (SWM) based on polygon
contiguity.
INPUTS:
inputFC (str): path to the input feature class
swmFile (str): path to the SWM file.
masterField (str): field in table that serves as the mapping.
concept: {str, EUCLIDEAN}: EUCLIDEAN or MANHATTAN
kNeighs {int, 0}: number of neighbors to return (1)
rowStandard {bool, True}: row standardize weights?
contiguityType {str, Rook}: {Rook = Edges Only, Queen = Edges/Vertices}
NOTES:
(1) kNeighs is used if polygon is not contiguous. E.g. Islands
"""
#### Set Default Progressor for Neigborhood Structure ####
ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84143))
#### Create SSDataObject ####
ssdo = SSDO.SSDataObject(inputFC, templateFC = inputFC,
useChordal = True)
cnt = UTILS.getCount(inputFC)
ERROR.errorNumberOfObs(cnt, minNumObs = 2)
#### Validation of Master Field ####
verifyMaster = ERROR.checkField(ssdo.allFields, masterField,
types = [0,1])
#### Create GA Data Structure ####
gaTable, gaInfo = WU.gaTable(ssdo.catPath, [masterField],
spatRef = ssdo.spatialRefString)
#### Assure Enough Observations ####
N = gaInfo[0]
ERROR.errorNumberOfObs(N, minNumObs = 2)
#### Assure k-Nearest is Less Than Number of Features ####
if kNeighs >= N:
ARCPY.AddIDMessage("ERROR", 975)
raise SystemExit()
#### Create Nearest Neighbor Search Type For Islands ####
if kNeighs > 0:
gaSearch = GAPY.ga_nsearch(gaTable)
concept, gaConcept = WU.validateDistanceMethod(concept, ssdo.spatialRef)
gaSearch.init_nearest(0.0, kNeighs, gaConcept)
forceNeighbor = True
neighWeights = ARC._ss.NeighborWeights(gaTable, gaSearch,
weight_type = 1,
row_standard = False)
else:
forceNeighbor = False
neighSearch = None
#### Create Polygon Neighbors ####
polyNeighborDict = WU.polygonNeighborDict(inputFC, masterField,
contiguityType = contiguityType)
#### Write Poly Neighbor List (Dict) ####
#### Set Progressor for SWM Writing ####
ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84127), 0, N, 1)
#### Initialize Spatial Weights Matrix File ####
if contiguityType == "ROOK":
wType = 4
else:
wType = 5
swmWriter = WU.SWMWriter(swmFile, masterField, ssdo.spatialRefName,
N, rowStandard, inputFC = inputFC,
wType = wType, distanceMethod = concept,
numNeighs = kNeighs)
#### Keep Track of Polygons w/o Neighbors ####
islandPolys = []
#### Write Polygon Contiguity to SWM File ####
for row in xrange(N):
rowInfo = gaTable[row]
oid = rowInfo[0]
masterID = rowInfo[2]
neighs = polyNeighborDict[masterID]
nn = len(neighs)
if forceNeighbor:
if nn < kNeighs:
#### Only Force KNN If Specified & Contiguity is Less ####
islandPolys.append(oid)
flag = True
knnNeighs, knnWeights = neighWeights[row]
c = 0
while flag:
try:
neighID = gaTable[knnNeighs[c]][2]
if neighID not in neighs:
neighs.append(neighID)
nn += 1
if nn == kNeighs:
flag = False
c += 1
except:
flag = False
weights = NUM.ones(nn)
#### Add Weights Entry ####
swmWriter.swm.writeEntry(masterID, neighs, weights)
#### Set Progress ####
ARCPY.SetProgressorPosition()
#### Report on Features with No Neighbors ####
countIslands = len(islandPolys)
if countIslands:
islandPolys.sort()
if countIslands > 30:
islandPolys = islandPolys[0:30]
ERROR.warningNoNeighbors(N, countIslands, islandPolys, ssdo.oidName,
forceNeighbor = forceNeighbor,
contiguity = True)
#### Clean Up ####
swmWriter.close()
del gaTable
#### Report Spatial Weights Summary ####
swmWriter.report()
#### Report SWM File is Large ####
swmWriter.reportLargeSWM()
del polyNeighborDict
def kNearest2SWM(inputFC, swmFile, masterField, concept = "EUCLIDEAN",
kNeighs = 1, rowStandard = True):
"""Creates a sparse spatial weights matrix (SWM) based on k-nearest
neighbors.
INPUTS:
inputFC (str): path to the input feature class
swmFile (str): path to the SWM file.
masterField (str): field in table that serves as the mapping.
concept: {str, EUCLIDEAN}: EUCLIDEAN or MANHATTAN
kNeighs {int, 1}: number of neighbors to return
rowStandard {bool, True}: row standardize weights?
"""
#### Assure that kNeighs is Non-Zero ####
if kNeighs <= 0:
ARCPY.AddIDMessage("ERROR", 976)
raise SystemExit()
#### Set Default Progressor for Neigborhood Structure ####
ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84143))
#### Create SSDataObject ####
ssdo = SSDO.SSDataObject(inputFC, templateFC = inputFC,
useChordal = True)
cnt = UTILS.getCount(inputFC)
ERROR.errorNumberOfObs(cnt, minNumObs = 2)
#### Validation of Master Field ####
verifyMaster = ERROR.checkField(ssdo.allFields, masterField, types = [0,1])
#### Create GA Data Structure ####
gaTable, gaInfo = WU.gaTable(ssdo.catPath, [masterField],
spatRef = ssdo.spatialRefString)
#### Assure Enough Observations ####
N = gaInfo[0]
ERROR.errorNumberOfObs(N, minNumObs = 2)
#### Process any bad records encountered ####
numBadRecs = cnt - N
if numBadRecs:
badRecs = WU.parseGAWarnings(gaTable.warnings)
err = ERROR.reportBadRecords(cnt, numBadRecs, badRecs,
label = ssdo.oidName)
#### Assure k-Nearest is Less Than Number of Features ####
if kNeighs >= N:
ARCPY.AddIDMessage("ERROR", 975)
raise SystemExit()
#### Create k-Nearest Neighbor Search Type ####
gaSearch = GAPY.ga_nsearch(gaTable)
concept, gaConcept = WU.validateDistanceMethod(concept, ssdo.spatialRef)
gaSearch.init_nearest(0.0, kNeighs, gaConcept)
neighWeights = ARC._ss.NeighborWeights(gaTable, gaSearch,
weight_type = 1,
row_standard = False)
#### Set Progressor for Weights Writing ####
ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84127), 0, N, 1)
#### Initialize Spatial Weights Matrix File ####
swmWriter = WU.SWMWriter(swmFile, masterField, ssdo.spatialRefName,
N, rowStandard, inputFC = inputFC,
wType = 2, distanceMethod = concept,
numNeighs = kNeighs)
#### Unique Master ID Dictionary ####
masterSet = set([])
for row in xrange(N):
masterID = int(gaTable[row][2])
if masterID in masterSet:
ARCPY.AddIDMessage("Error", 644, masterField)
ARCPY.AddIDMessage("Error", 643)
raise SystemExit()
else:
masterSet.add(masterID)
neighs, weights = neighWeights[row]
neighs = [ gaTable[nh][2] for nh in neighs ]
#### Add Spatial Weights Matrix Entry ####
swmWriter.swm.writeEntry(masterID, neighs, weights)
#### Set Progress ####
ARCPY.SetProgressorPosition()
swmWriter.close()
del gaTable
#### Report Warning/Max Neighbors ####
swmWriter.reportNeighInfo()
#### Report Spatial Weights Summary ####
swmWriter.report()
#### Report SWM File is Large ####
swmWriter.reportLargeSWM()
def distance2Weights(ssdo,
neighborType=1,
distanceBand=0.0,
numNeighs=0,
distanceType="euclidean",
exponent=1.0,
rowStandard=True,
includeSelf=False):
"""Uses ArcGIS Neighborhood Searching Structure to create a PySAL Sparse Spatial
Weights Matrix.
INPUTS:
ssdo (class): instance of SSDataObject [1]
neighborType {int, 1}: 0 = inverse distance, 1 = fixed distance,
2 = k-nearest-neighbors, 3 = delaunay
distanceBand {float, 0.0}: return all neighbors within this distance for
inverse/fixed distance
numNeighs {int, 0}: number of neighbors for k-nearest-neighbor, can also
be used to set a minimum number of neighbors for
inverse/fixed distance
distanceType {str, euclidean}: manhattan or euclidean distance [2]
exponent {float, 1.0}: distance decay factor for inverse distance
rowStandard {bool, True}: whether to row standardize the spatial weights
includeSelf {bool, False}: whether to return self as a neighbor
NOTES:
(1) Data must already be obtained using ssdo.obtainDataGA()
(2) Chordal Distance is used for GCS Data
"""
neighbors = {}
weights = {}
gaSearch = GAPY.ga_nsearch(ssdo.gaTable)
if neighborType == 3:
gaSearch.init_delaunay()
neighSearch = ARC._ss.NeighborWeights(ssdo.gaTable,
gaSearch,
weight_type=1)
else:
if neighborType == 2:
distanceBand = 0.0
weightType = 1
else:
weightType = neighborType
concept, gaConcept = WU.validateDistanceMethod(distanceType.upper(),
ssdo.spatialRef)
gaSearch.init_nearest(distanceBand, numNeighs, gaConcept)
neighSearch = ARC._ss.NeighborWeights(ssdo.gaTable,
gaSearch,
weight_type=weightType,
exponent=exponent,
include_self=includeSelf)
for i in range(len(neighSearch)):
neighOrderIDs, neighWeights = neighSearch[i]
neighbors[i] = neighOrderIDs
weights[i] = neighWeights
w = PYSAL.W(neighbors, weights)
if rowStandard:
w.transform = 'R'
return w
def spaceTime2SWM(inputFC, swmFile, masterField, concept = "EUCLIDEAN",
threshold = None, rowStandard = True,
timeField = None, timeType = None,
timeValue = None):
"""
inputFC (str): path to the input feature class
swmFile (str): path to the SWM file.
masterField (str): field in table that serves as the mapping.
concept: {str, EUCLIDEAN}: EUCLIDEAN or MANHATTAN
threshold {float, None}: distance threshold
rowStandard {bool, True}: row standardize weights?
timeField {str, None}: name of the date-time field
timeType {str, None}: ESRI enumeration of date-time intervals
timeValue {float, None}: value forward and backward in time
"""
#### Assure Temporal Parameters are Set ####
if not timeField:
ARCPY.AddIDMessage("ERROR", 1320)
raise SystemExit()
if not timeType:
ARCPY.AddIDMessage("ERROR", 1321)
raise SystemExit()
if not timeValue or timeValue <= 0:
ARCPY.AddIDMessage("ERROR", 1322)
raise SystemExit()
#### Create SSDataObject ####
ssdo = SSDO.SSDataObject(inputFC, templateFC = inputFC,
useChordal = True)
cnt = UTILS.getCount(inputFC)
ERROR.errorNumberOfObs(cnt, minNumObs = 2)
ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84001), 0, cnt, 1)
#### Validation of Master Field ####
verifyMaster = ERROR.checkField(ssdo.allFields, masterField, types = [0,1])
badIDs = []
#### Create Temporal Hash ####
timeInfo = {}
xyCoords = NUM.empty((cnt, 2), float)
#### Process Field Values ####
fieldList = [masterField, "SHAPE@XY", timeField]
try:
rows = DA.SearchCursor(ssdo.catPath, fieldList, "",
ssdo.spatialRefString)
except:
ARCPY.AddIDMessage("ERROR", 204)
raise SystemExit()
#### Add Data to GATable and Time Dictionary ####
c = 0
for row in rows:
badRow = False
#### Assure Masterfield is Valid ####
masterID = row[0]
if masterID == None or masterID == "":
badRow = True
#### Assure Date/Time is Valid ####
timeStamp = row[-1]
if timeStamp == None or timeStamp == "":
badRow = True
#### Assure Centroid is Valid ####
badXY = row[1].count(None)
if not badXY:
x,y = row[1]
xyCoords[c] = (x,y)
else:
badRow = True
#### Process Data ####
if not badRow:
if timeInfo.has_key(masterID):
#### Assure Uniqueness ####
ARCPY.AddIDMessage("Error", 644, masterField)
ARCPY.AddIDMessage("Error", 643)
raise SystemExit()
else:
#### Fill Date/Time Dict ####
startDT, endDT = TUTILS.calculateTimeWindow(timeStamp,
timeValue,
timeType)
timeInfo[masterID] = (timeStamp, startDT, endDT)
else:
badIDs.append(masterID)
#### Set Progress ####
c += 1
ARCPY.SetProgressorPosition()
#### Clean Up ####
del rows
#### Get Set of Bad IDs ####
numBadObs = len(badIDs)
badIDs = list(set(badIDs))
badIDs.sort()
badIDs = [ str(i) for i in badIDs ]
#### Process any bad records encountered ####
if numBadObs:
ERROR.reportBadRecords(cnt, numBadObs, badIDs, label = masterField)
#### Load Neighbor Table ####
gaTable, gaInfo = WU.gaTable(ssdo.inputFC,
fieldNames = [masterField, timeField],
spatRef = ssdo.spatialRefString)
numObs = len(gaTable)
xyCoords = xyCoords[0:numObs]
#### Set the Distance Threshold ####
concept, gaConcept = WU.validateDistanceMethod(concept, ssdo.spatialRef)
if threshold == None:
#### Set Progressor for Search ####
ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84144))
#### Create k-Nearest Neighbor Search Type ####
gaSearch = GAPY.ga_nsearch(gaTable)
gaSearch.init_nearest(0.0, 1, gaConcept)
neighDist = ARC._ss.NeighborDistances(gaTable, gaSearch)
N = len(neighDist)
threshold = 0.0
sumDist = 0.0
#### Find Maximum Nearest Neighbor Distance ####
for row in xrange(N):
dij = neighDist[row][-1][0]
if dij > threshold:
threshold = dij
sumDist += dij
ARCPY.SetProgressorPosition()
#### Increase For Rounding Error ####
threshold = threshold * 1.0001
avgDist = sumDist / (N * 1.0)
#### Add Linear/Angular Units ####
thresholdStr = ssdo.distanceInfo.printDistance(threshold)
ARCPY.AddIDMessage("Warning", 853, thresholdStr)
#### Chordal Default Check ####
if ssdo.useChordal:
hardMaxExtent = ARC._ss.get_max_gcs_distance(ssdo.spatialRef)
if threshold > hardMaxExtent:
ARCPY.AddIDMessage("ERROR", 1609)
raise SystemExit()
#### Clean Up ####
del gaSearch
#### Create Missing SSDO Info ####
extent = UTILS.resetExtent(xyCoords)
#### Reset Coordinates for Chordal ####
if ssdo.useChordal:
sliceInfo = UTILS.SpheroidSlice(extent, ssdo.spatialRef)
maxExtent = sliceInfo.maxExtent
else:
env = UTILS.Envelope(extent)
maxExtent = env.maxExtent
threshold = checkDistanceThresholdSWM(ssdo, threshold, maxExtent)
#### Set Default Progressor for Neigborhood Structure ####
ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84143))
#### Create Distance Neighbor Search Type ####
gaSearch = GAPY.ga_nsearch(gaTable)
gaSearch.init_nearest(threshold, 0, gaConcept)
neighSearch = ARC._ss.NeighborSearch(gaTable, gaSearch)
#### Set Progressor for Weights Writing ####
ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84127), 0, numObs, 1)
#### Initialize Spatial Weights Matrix File ####
swmWriter = WU.SWMWriter(swmFile, masterField, ssdo.spatialRefName,
numObs, rowStandard, inputFC = inputFC,
wType = 9, distanceMethod = concept,
threshold = threshold, timeField = timeField,
timeType = timeType, timeValue = timeValue)
for row in xrange(numObs):
masterID = gaTable[row][2]
#### Get Date/Time Info ####
dt0, startDT0, endDT0 = timeInfo[masterID]
nhs = neighSearch[row]
neighs = []
weights = []
for nh in nhs:
#### Search Through Spatial Neighbors ####
neighID = gaTable[nh][2]
#### Get Date/Time Info ####
dt1, startDT1, endDT1 = timeInfo[neighID]
#### Filter Based on Date/Time ####
insideTimeWindow = TUTILS.isTimeNeighbor(startDT0, endDT0, dt1)
if insideTimeWindow:
neighs.append(neighID)
weights.append(1.0)
#### Add Spatial Weights Matrix Entry ####
swmWriter.swm.writeEntry(masterID, neighs, weights)
#### Set Progress ####
ARCPY.SetProgressorPosition()
swmWriter.close()
del gaTable
#### Report Warning/Max Neighbors ####
swmWriter.reportNeighInfo()
#### Report Spatial Weights Summary ####
swmWriter.report()
#### Report SWM File is Large ####
swmWriter.reportLargeSWM()
def distance2SWM(inputFC, swmFile, masterField, fixed = 0,
concept = "EUCLIDEAN", exponent = 1.0, threshold = None,
kNeighs = 1, rowStandard = True):
"""Creates a sparse spatial weights matrix (SWM) based on k-nearest
neighbors.
INPUTS:
inputFC (str): path to the input feature class
swmFile (str): path to the SWM file.
masterField (str): field in table that serves as the mapping.
fixed (boolean): fixed (1) or inverse (0) distance?
concept: {str, EUCLIDEAN}: EUCLIDEAN or MANHATTAN
exponent {float, 1.0}: distance decay
threshold {float, None}: distance threshold
kNeighs (int): number of neighbors to return
rowStandard {bool, True}: row standardize weights?
"""
#### Create SSDataObject ####
ssdo = SSDO.SSDataObject(inputFC, templateFC = inputFC,
useChordal = True)
#### Validation of Master Field ####
verifyMaster = ERROR.checkField(ssdo.allFields, masterField, types = [0,1])
#### Read Data ####
ssdo.obtainDataGA(masterField, minNumObs = 2)
N = ssdo.numObs
gaTable = ssdo.gaTable
if fixed:
wType = 1
else:
wType = 0
#### Set Default Progressor for Neigborhood Structure ####
ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84143))
#### Set the Distance Threshold ####
concept, gaConcept = WU.validateDistanceMethod(concept, ssdo.spatialRef)
if threshold == None:
threshold, avgDist = WU.createThresholdDist(ssdo,
concept = concept)
#### Assures that the Threshold is Appropriate ####
gaExtent = UTILS.get92Extent(ssdo.extent)
threshold, maxSet = WU.checkDistanceThreshold(ssdo, threshold,
weightType = wType)
#### If the Threshold is Set to the Max ####
#### Set to Zero for Script Logic ####
if maxSet:
#### All Locations are Related ####
threshold = SYS.maxint
if N > 500:
ARCPY.AddIDMessage("Warning", 717)
#### Assure k-Nearest is Less Than Number of Features ####
if kNeighs >= N and fixed:
ARCPY.AddIDMessage("ERROR", 975)
raise SystemExit()
#### Create Distance/k-Nearest Neighbor Search Type ####
gaSearch = GAPY.ga_nsearch(gaTable)
gaSearch.init_nearest(threshold, kNeighs, gaConcept)
neighWeights = ARC._ss.NeighborWeights(gaTable, gaSearch,
weight_type = wType,
exponent = exponent,
row_standard = False)
#### Set Progressor for Weights Writing ####
ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84127), 0, N, 1)
#### Initialize Spatial Weights Matrix File ####
swmWriter = WU.SWMWriter(swmFile, masterField, ssdo.spatialRefName,
N, rowStandard, inputFC = inputFC,
wType = wType, distanceMethod = concept,
exponent = exponent, threshold = threshold)
#### Unique Master ID Dictionary ####
masterDict = {}
#### Unique Master ID Dictionary ####
masterSet = set([])
for row in xrange(N):
masterID = int(gaTable[row][2])
if masterID in masterSet:
ARCPY.AddIDMessage("Error", 644, masterField)
ARCPY.AddIDMessage("Error", 643)
raise SystemExit()
else:
masterSet.add(masterID)
neighs, weights = neighWeights[row]
neighs = [ gaTable[nh][2] for nh in neighs ]
#### Add Spatial Weights Matrix Entry ####
swmWriter.swm.writeEntry(masterID, neighs, weights)
#### Set Progress ####
ARCPY.SetProgressorPosition()
swmWriter.close()
del gaTable
#### Report Warning/Max Neighbors ####
swmWriter.reportNeighInfo()
#### Add Linear/Angular Unit (Distance Based Only) ####
distanceOut = ssdo.distanceInfo.outputString
distanceOut = [ARCPY.GetIDMessage(84344).format(distanceOut)]
#### Report Spatial Weights Summary ####
swmWriter.report(additionalInfo = distanceOut)
#### Report SWM File is Large ####
swmWriter.reportLargeSWM()
def kNearest2SWM(inputFC, swmFile, masterField, concept = "EUCLIDEAN",
kNeighs = 1, rowStandard = True):
"""Creates a sparse spatial weights matrix (SWM) based on k-nearest
neighbors.
INPUTS:
inputFC (str): path to the input feature class
swmFile (str): path to the SWM file.
masterField (str): field in table that serves as the mapping.
concept: {str, EUCLIDEAN}: EUCLIDEAN or MANHATTAN
kNeighs {int, 1}: number of neighbors to return
rowStandard {bool, True}: row standardize weights?
"""
#### Assure that kNeighs is Non-Zero ####
if kNeighs <= 0:
ARCPY.AddIDMessage("ERROR", 976)
raise SystemExit()
#### Set Default Progressor for Neigborhood Structure ####
ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84143))
#### Create SSDataObject ####
ssdo = SSDO.SSDataObject(inputFC, templateFC = inputFC,
useChordal = True)
cnt = UTILS.getCount(inputFC)
ERROR.errorNumberOfObs(cnt, minNumObs = 2)
#### Validation of Master Field ####
verifyMaster = ERROR.checkField(ssdo.allFields, masterField, types = [0,1])
#### Create GA Data Structure ####
gaTable, gaInfo = WU.gaTable(ssdo.catPath, [masterField],
spatRef = ssdo.spatialRefString)
#### Assure Enough Observations ####
N = gaInfo[0]
ERROR.errorNumberOfObs(N, minNumObs = 2)
#### Process any bad records encountered ####
numBadRecs = cnt - N
if numBadRecs:
badRecs = WU.parseGAWarnings(gaTable.warnings)
err = ERROR.reportBadRecords(cnt, numBadRecs, badRecs,
label = ssdo.oidName)
#### Assure k-Nearest is Less Than Number of Features ####
if kNeighs >= N:
ARCPY.AddIDMessage("ERROR", 975)
raise SystemExit()
#### Create k-Nearest Neighbor Search Type ####
gaSearch = GAPY.ga_nsearch(gaTable)
concept, gaConcept = WU.validateDistanceMethod(concept, ssdo.spatialRef)
gaSearch.init_nearest(0.0, kNeighs, gaConcept)
neighWeights = ARC._ss.NeighborWeights(gaTable, gaSearch,
weight_type = 1,
row_standard = False)
#### Set Progressor for Weights Writing ####
ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84127), 0, N, 1)
#### Initialize Spatial Weights Matrix File ####
swmWriter = WU.SWMWriter(swmFile, masterField, ssdo.spatialRefName,
N, rowStandard, inputFC = inputFC,
wType = 2, distanceMethod = concept,
numNeighs = kNeighs)
#### Unique Master ID Dictionary ####
masterSet = set([])
for row in xrange(N):
masterID = int(gaTable[row][2])
if masterID in masterSet:
ARCPY.AddIDMessage("Error", 644, masterField)
ARCPY.AddIDMessage("Error", 643)
raise SystemExit()
else:
masterSet.add(masterID)
neighs, weights = neighWeights[row]
neighs = [ gaTable[nh][2] for nh in neighs ]
#### Add Spatial Weights Matrix Entry ####
swmWriter.swm.writeEntry(masterID, neighs, weights)
#### Set Progress ####
ARCPY.SetProgressorPosition()
swmWriter.close()
del gaTable
#### Report Warning/Max Neighbors ####
swmWriter.reportNeighInfo()
#### Report Spatial Weights Summary ####
swmWriter.report()
#### Report SWM File is Large ####
swmWriter.reportLargeSWM()
def polygon2SWM(inputFC, swmFile, masterField,
concept = "EUCLIDEAN", kNeighs = 0,
rowStandard = True, contiguityType = "ROOK"):
"""Creates a sparse spatial weights matrix (SWM) based on polygon
contiguity.
INPUTS:
inputFC (str): path to the input feature class
swmFile (str): path to the SWM file.
masterField (str): field in table that serves as the mapping.
concept: {str, EUCLIDEAN}: EUCLIDEAN or MANHATTAN
kNeighs {int, 0}: number of neighbors to return (1)
rowStandard {bool, True}: row standardize weights?
contiguityType {str, Rook}: {Rook = Edges Only, Queen = Edges/Vertices}
NOTES:
(1) kNeighs is used if polygon is not contiguous. E.g. Islands
"""
#### Set Default Progressor for Neigborhood Structure ####
ARCPY.SetProgressor("default", ARCPY.GetIDMessage(84143))
#### Create SSDataObject ####
ssdo = SSDO.SSDataObject(inputFC, templateFC = inputFC,
useChordal = True)
cnt = UTILS.getCount(inputFC)
ERROR.errorNumberOfObs(cnt, minNumObs = 2)
#### Validation of Master Field ####
verifyMaster = ERROR.checkField(ssdo.allFields, masterField,
types = [0,1])
#### Create GA Data Structure ####
gaTable, gaInfo = WU.gaTable(ssdo.catPath, [masterField],
spatRef = ssdo.spatialRefString)
#### Assure Enough Observations ####
N = gaInfo[0]
ERROR.errorNumberOfObs(N, minNumObs = 2)
#### Assure k-Nearest is Less Than Number of Features ####
if kNeighs >= N:
ARCPY.AddIDMessage("ERROR", 975)
raise SystemExit()
#### Create Nearest Neighbor Search Type For Islands ####
gaSearch = GAPY.ga_nsearch(gaTable)
concept, gaConcept = WU.validateDistanceMethod(concept, ssdo.spatialRef)
gaSearch.init_nearest(0.0, kNeighs, gaConcept)
if kNeighs > 0:
forceNeighbor = True
neighWeights = ARC._ss.NeighborWeights(gaTable, gaSearch,
weight_type = 1,
row_standard = False)
else:
forceNeighbor = False
neighSearch = None
#### Create Polygon Neighbors ####
polyNeighborDict = WU.polygonNeighborDict(inputFC, masterField,
contiguityType = contiguityType)
#### Write Poly Neighbor List (Dict) ####
#### Set Progressor for SWM Writing ####
ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84127), 0, N, 1)
#### Initialize Spatial Weights Matrix File ####
if contiguityType == "ROOK":
wType = 4
else:
wType = 5
swmWriter = WU.SWMWriter(swmFile, masterField, ssdo.spatialRefName,
N, rowStandard, inputFC = inputFC,
wType = wType, distanceMethod = concept,
numNeighs = kNeighs)
#### Keep Track of Polygons w/o Neighbors ####
islandPolys = []
#### Write Polygon Contiguity to SWM File ####
for row in xrange(N):
rowInfo = gaTable[row]
oid = rowInfo[0]
masterID = rowInfo[2]
neighs = polyNeighborDict[masterID]
if neighs:
weights = [ 1. for nh in neighs ]
isIsland = False
else:
isIsland = True
islandPolys.append(oid)
weights = []
#### Get Nearest Neighbor Based On Centroid Distance ####
if isIsland and forceNeighbor:
neighs, weights = neighWeights[row]
neighs = [ gaTable[nh][2] for nh in neighs ]
#### Add Weights Entry ####
swmWriter.swm.writeEntry(masterID, neighs, weights)
#### Set Progress ####
ARCPY.SetProgressorPosition()
#### Report on Features with No Neighbors ####
countIslands = len(islandPolys)
if countIslands:
islandPolys.sort()
if countIslands > 30:
islandPolys = islandPolys[0:30]
ERROR.warningNoNeighbors(N, countIslands, islandPolys, ssdo.oidName,
forceNeighbor = forceNeighbor,
contiguity = True)
#### Clean Up ####
swmWriter.close()
del gaTable
#### Report Spatial Weights Summary ####
swmWriter.report()
#### Report SWM File is Large ####
swmWriter.reportLargeSWM()
del polyNeighborDict
