def obtainData(self, masterField, fields = [], types = [0,1,2,3,4,5,6], minNumObs = 0, warnNumObs = 0, dateStr = False, explicitBadRecordID = None): """Takes a list of field names and returns it in a dictionary structure. INPUTS: masterField (str): name of field being used as the master fields {list, []}: name(s) of the field to be returned types (list): types of data allowed to be returned (1) minNumObs {int, 0}: minimum number of observations for error warnNumObs {int, 0}: minimum number of observations for warning OID {bool, False}: OID field allowed to be master field? ATTRIBUTES: gaTable (structure): instance of the GA Table fields (dict): fieldName = instance of FCField master2Order (dict): masterID = order in lists order2Master (dict): order in lists = masterID masterField (str): field that serves as the master badRecords (list): master IDs that could not be read xyCoords (array, nunObs x 2): xy-coordinates for feature centroids """ #### Get Base Count, May Include Bad Records #### cnt = UTILS.getCount(self.inputFC) #### Validation of Master Field #### verifyMaster = ERROR.checkField(self.allFields, masterField, types = [0,1,5]) #### Set MasterIsOID Boolean #### self.masterIsOID = masterField == self.oidName #### Set Master and Data Indices #### if self.masterIsOID: self.masterColumnIndex = 0 self.dataColumnIndex = 2 fieldList = [self.oidName, "shape@XY"] else: self.masterColumnIndex = 2 self.dataColumnIndex = 3 fieldList = [self.oidName, "shape@XY", masterField] #### Initialization of Centroids #### xyCoords = NUM.empty((cnt, 2), float) #### Validation and Initialization of Data Fields #### numFields = len(fields) fieldTypes = {} hasDate = False for field in fields: fieldType = ERROR.checkField(self.allFields, field, types = types) fieldTypes[field] = fieldType fieldList.append(field) self.fields[field] = self.allFields[field] if fieldType.upper() == "DATE": hasDate = True nowTime = DT.datetime.now() #### Create Empty Data Arrays #### for fieldName, fieldObj in self.fields.iteritems(): fieldObj.createDataArray(cnt, dateStr = dateStr) #### Z Coords #### if self.hasZ: zCoords = NUM.empty((cnt, ), float) fieldList.append("shape@Z") #### Keep track of Invalid Fields #### badIDs = [] badRecord = 0 #### Create Progressor Bar #### ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84001), 0, cnt, 1) #### Process Field Values #### try: rows = DA.SearchCursor(self.inputFC, fieldList, "", self.spatialRefString) except: ARCPY.AddIDMessage("ERROR", 204) raise SystemExit() c = 0 for row in rows: oid = row[0] badXY = row[1].count(None) if self.hasZ: badValues = row[0:-1].count(None) else: badValues = row.count(None) #### Check Bad Record #### if badXY or badValues: badRow = 1 badRecord = 1 badIDs.append(oid) else: #### Get Centroid and Master ID #### xyCoords[c] = row[1] masterID = row[self.masterColumnIndex] #### Add Field Values #### if numFields: restFields = row[self.dataColumnIndex:] for fieldInd, fieldName in enumerate(fields): fieldValue = restFields[fieldInd] fieldType = fieldTypes[fieldName] if fieldType.upper() == "DATE": if dateStr: fieldValue = str(fieldValue) else: fieldValue = (nowTime - fieldValue).total_seconds() self.fields[fieldName].data[c] = fieldValue if self.hasZ: zCoords[c] = row[-1] #### Check uniqueness of masterID field #### if self.master2Order.has_key(masterID): del rows ARCPY.AddIDMessage("ERROR", 644, masterField) ARCPY.AddIDMessage("ERROR", 643) raise SystemExit() else: self.master2Order[masterID] = c self.order2Master[c] = masterID c += 1 ARCPY.SetProgressorPosition() del rows #### Check Whether the Number of Features is Appropriate #### numObs = len(self.master2Order) ERROR.checkNumberOfObs(numObs, minNumObs = minNumObs, warnNumObs = warnNumObs, silentWarnings = self.silentWarnings) #### Get Set of Bad IDs #### badIDs = list(set(badIDs)) badIDs.sort() badIDs = [ str(i) for i in badIDs ] #### Process any bad records encountered #### if badRecord != 0: bn = len(badIDs) if not self.silentWarnings: ERROR.reportBadRecords(cnt, bn, badIDs, label = self.oidName, explicitBadRecordID = explicitBadRecordID) #### Prune Data Arrays #### xyCoords = xyCoords[0:numObs] self.resizeDataArrays(numObs) if self.hasZ: zCoords = zCoords[0:numObs] #### Set the Hidden Fields (E.g. Not in Use) #### self.setHiddenFields() #### Reset Extent to Honor Env and Subsets #### try: self.extent = UTILS.resetExtent(xyCoords) except: pass #### Reset Coordinates for Chordal #### if self.useChordal: #### Project to XY on Spheroid #### self.spheroidCoords = ARC._ss.lonlat_to_xy(xyCoords, self.spatialRef) self.sliceInfo = UTILS.SpheroidSlice(self.extent, self.spatialRef) else: self.spheroidCoords = None self.sliceInfo = None #### Set Further Attributes #### self.badRecords = badIDs self.xyCoords = xyCoords self.masterField = masterField self.gaTable = None self.numObs = numObs if self.hasZ: self.zCoords = zCoords else: self.zCoords = None
def obtainData(self, masterField, fields=[], types=[0, 1, 2, 3, 4, 5, 6], minNumObs=0, warnNumObs=0, dateStr=False, explicitBadRecordID=None): """Takes a list of field names and returns it in a dictionary structure. INPUTS: masterField (str): name of field being used as the master fields {list, []}: name(s) of the field to be returned types (list): types of data allowed to be returned (1) minNumObs {int, 0}: minimum number of observations for error warnNumObs {int, 0}: minimum number of observations for warning OID {bool, False}: OID field allowed to be master field? ATTRIBUTES: gaTable (structure): instance of the GA Table fields (dict): fieldName = instance of FCField master2Order (dict): masterID = order in lists order2Master (dict): order in lists = masterID masterField (str): field that serves as the master badRecords (list): master IDs that could not be read xyCoords (array, nunObs x 2): xy-coordinates for feature centroids """ #### Get Base Count, May Include Bad Records #### cnt = UTILS.getCount(self.inputFC) #### Validation of Master Field #### verifyMaster = ERROR.checkField(self.allFields, masterField, types=[0, 1, 5]) #### Set MasterIsOID Boolean #### self.masterIsOID = masterField == self.oidName #### Set Master and Data Indices #### if self.masterIsOID: self.masterColumnIndex = 0 self.dataColumnIndex = 2 fieldList = [self.oidName, "shape@XY"] else: self.masterColumnIndex = 2 self.dataColumnIndex = 3 fieldList = [self.oidName, "shape@XY", masterField] #### Initialization of Centroids #### xyCoords = NUM.empty((cnt, 2), float) #### Validation and Initialization of Data Fields #### numFields = len(fields) fieldTypes = {} hasDate = False for field in fields: fieldType = ERROR.checkField(self.allFields, field, types=types) fieldTypes[field] = fieldType fieldList.append(field) self.fields[field] = self.allFields[field] if fieldType.upper() == "DATE": hasDate = True nowTime = DT.datetime.now() #### Create Empty Data Arrays #### for fieldName, fieldObj in self.fields.iteritems(): fieldObj.createDataArray(cnt, dateStr=dateStr) #### Z Coords #### if self.hasZ: zCoords = NUM.empty((cnt, ), float) fieldList.append("shape@Z") #### Keep track of Invalid Fields #### badIDs = [] badRecord = 0 #### Create Progressor Bar #### ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84001), 0, cnt, 1) #### Process Field Values #### try: rows = DA.SearchCursor(self.inputFC, fieldList, "", self.spatialRefString) except: ARCPY.AddIDMessage("ERROR", 204) raise SystemExit() c = 0 for row in rows: oid = row[0] badXY = row[1].count(None) if self.hasZ: badValues = row[0:-1].count(None) else: badValues = row.count(None) #### Check Bad Record #### if badXY or badValues: badRow = 1 badRecord = 1 badIDs.append(oid) else: #### Get Centroid and Master ID #### xyCoords[c] = row[1] masterID = row[self.masterColumnIndex] #### Add Field Values #### if numFields: restFields = row[self.dataColumnIndex:] for fieldInd, fieldName in enumerate(fields): fieldValue = restFields[fieldInd] fieldType = fieldTypes[fieldName] if fieldType.upper() == "DATE": if dateStr: fieldValue = str(fieldValue) else: fieldValue = (nowTime - fieldValue).total_seconds() self.fields[fieldName].data[c] = fieldValue if self.hasZ: zCoords[c] = row[-1] #### Check uniqueness of masterID field #### if self.master2Order.has_key(masterID): del rows ARCPY.AddIDMessage("ERROR", 644, masterField) ARCPY.AddIDMessage("ERROR", 643) raise SystemExit() else: self.master2Order[masterID] = c self.order2Master[c] = masterID c += 1 ARCPY.SetProgressorPosition() del rows #### Check Whether the Number of Features is Appropriate #### numObs = len(self.master2Order) ERROR.checkNumberOfObs(numObs, minNumObs=minNumObs, warnNumObs=warnNumObs, silentWarnings=self.silentWarnings) #### Get Set of Bad IDs #### badIDs = list(set(badIDs)) badIDs.sort() badIDs = [str(i) for i in badIDs] #### Process any bad records encountered #### if badRecord != 0: bn = len(badIDs) if not self.silentWarnings: ERROR.reportBadRecords(cnt, bn, badIDs, label=self.oidName, explicitBadRecordID=explicitBadRecordID) #### Prune Data Arrays #### xyCoords = xyCoords[0:numObs] self.resizeDataArrays(numObs) if self.hasZ: zCoords = zCoords[0:numObs] #### Set the Hidden Fields (E.g. Not in Use) #### self.setHiddenFields() #### Reset Extent to Honor Env and Subsets #### try: self.extent = UTILS.resetExtent(xyCoords) except: pass #### Reset Coordinates for Chordal #### if self.useChordal: #### Project to XY on Spheroid #### self.spheroidCoords = ARC._ss.lonlat_to_xy(xyCoords, self.spatialRef) self.sliceInfo = UTILS.SpheroidSlice(self.extent, self.spatialRef) else: self.spheroidCoords = None self.sliceInfo = None #### Set Further Attributes #### self.badRecords = badIDs self.xyCoords = xyCoords self.masterField = masterField self.gaTable = None self.numObs = numObs if self.hasZ: self.zCoords = zCoords else: self.zCoords = None
def obtainDataGA(self, masterField, fields = [], types = [0,1,2,3,5,6], minNumObs = 0, warnNumObs = 0): """Takes a list of field names and returns it in a dictionary structure. INPUTS: masterField (str): name of field being used as the master fields {list, []}: name(s) of the field to be returned types (list): types of data allowed to be returned (1) minNumObs {int, 0}: minimum number of observations for error warnNumObs {int, 0}: minimum number of observations for warning ATTRIBUTES: gaTable (structure): instance of the GA Table fields (dict): fieldName = instance of FCField master2Order (dict): masterID = order in lists order2Master (dict): order in lists = masterID masterField (str): field that serves as the master badRecords (list): master IDs that could not be read xyCoords (array, nunObs x 2): xy-coordinates for feature centroids NOTES: (1) No Text Fields; short [0], long [1], float [2], double[3] """ #### Validation of Master Field #### verifyMaster = ERROR.checkField(self.allFields, masterField, types = [0,1,5]) #### Set MasterIsOID Boolean #### self.masterIsOID = masterField == self.oidName #### Set Master and Data Indices #### if self.masterIsOID: self.masterColumnIndex = 0 self.dataColumnIndex = 2 fieldList = [] else: self.masterColumnIndex = 2 self.dataColumnIndex = 3 fieldList = [masterField] #### Validation and Initialization of Data Fields #### numFields = len(fields) for field in fields: fType = ERROR.checkField(self.allFields, field, types = types) fieldList.append(field) self.fields[field] = self.allFields[field] #### ZCoords Are Last #### getZBool = self.hasZ and (not self.renderType) if getZBool: fieldList.append("SHAPE&Z") #### Create GA Data Structure #### cnt = UTILS.getCount(self.inputFC) fieldList = tuple(fieldList) gaTable, gaInfo = WU.gaTable(self.inputFC, fieldNames = fieldList, spatRef = self.spatialRefString) #### Check Whether the Number of Features is Appropriate #### numObs = gaInfo[0] ERROR.checkNumberOfObs(numObs, minNumObs = minNumObs, warnNumObs = warnNumObs, silentWarnings = self.silentWarnings) #### Process any bad records encountered #### numBadIDs = cnt - numObs if numBadIDs: badIDs = WU.parseGAWarnings(gaTable.warnings) if not self.silentWarnings: ERROR.reportBadRecords(cnt, numBadIDs, badIDs, label = self.oidName) else: badIDs = [] #### Initialization of Centroids #### xyCoords = NUM.empty((numObs, 2), float) #### Z Coords #### if self.hasZ: zCoords = NUM.empty((numObs, ), float) #### Create Empty Data Arrays #### for fieldName, fieldObj in self.fields.iteritems(): fieldObj.createDataArray(numObs) #### Populate SSDataObject #### ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84001), 0, numObs, 1) for row in xrange(numObs): rowInfo = gaTable[row] x,y = rowInfo[1] masterID = int(rowInfo[self.masterColumnIndex]) if self.master2Order.has_key(masterID): ARCPY.AddIDMessage("ERROR", 644, masterField) ARCPY.AddIDMessage("ERROR", 643) raise SystemExit() else: self.master2Order[masterID] = row self.order2Master[row] = masterID xyCoords[row] = (x, y) if numFields: restFields = rowInfo[self.dataColumnIndex:] for fieldInd, fieldName in enumerate(fields): self.fields[fieldName].data[row] = restFields[fieldInd] if self.hasZ: if getZBool: zCoords[row] = rowInfo[-1] else: zCoords[row] = NUM.nan ARCPY.SetProgressorPosition() #### Set the Hidden Fields (E.g. Not in Use) #### self.setHiddenFields() #### Reset Extent to Honor Env and Subsets #### try: self.extent = UTILS.resetExtent(xyCoords) except: pass #### Reset Coordinates for Chordal #### if self.useChordal: #### Project to XY on Spheroid #### self.spheroidCoords = ARC._ss.lonlat_to_xy(xyCoords, self.spatialRef) self.sliceInfo = UTILS.SpheroidSlice(self.extent, self.spatialRef) else: self.spheroidCoords = None self.sliceInfo = None #### Set Further Attributes #### self.badRecords = badIDs self.xyCoords = xyCoords self.masterField = masterField self.gaTable = gaTable self.numObs = numObs if self.hasZ: self.zCoords = zCoords else: self.zCoords = None
def obtainDataGA(self, masterField, fields=[], types=[0, 1, 2, 3, 5, 6], minNumObs=0, warnNumObs=0): """Takes a list of field names and returns it in a dictionary structure. INPUTS: masterField (str): name of field being used as the master fields {list, []}: name(s) of the field to be returned types (list): types of data allowed to be returned (1) minNumObs {int, 0}: minimum number of observations for error warnNumObs {int, 0}: minimum number of observations for warning ATTRIBUTES: gaTable (structure): instance of the GA Table fields (dict): fieldName = instance of FCField master2Order (dict): masterID = order in lists order2Master (dict): order in lists = masterID masterField (str): field that serves as the master badRecords (list): master IDs that could not be read xyCoords (array, nunObs x 2): xy-coordinates for feature centroids NOTES: (1) No Text Fields; short [0], long [1], float [2], double[3] """ #### Validation of Master Field #### verifyMaster = ERROR.checkField(self.allFields, masterField, types=[0, 1, 5]) #### Set MasterIsOID Boolean #### self.masterIsOID = masterField == self.oidName #### Set Master and Data Indices #### if self.masterIsOID: self.masterColumnIndex = 0 self.dataColumnIndex = 2 fieldList = [] else: self.masterColumnIndex = 2 self.dataColumnIndex = 3 fieldList = [masterField] #### Validation and Initialization of Data Fields #### numFields = len(fields) for field in fields: fType = ERROR.checkField(self.allFields, field, types=types) fieldList.append(field) self.fields[field] = self.allFields[field] #### ZCoords Are Last #### getZBool = self.hasZ and (not self.renderType) if getZBool: fieldList.append("SHAPE&Z") #### Create GA Data Structure #### cnt = UTILS.getCount(self.inputFC) fieldList = tuple(fieldList) gaTable, gaInfo = WU.gaTable(self.inputFC, fieldNames=fieldList, spatRef=self.spatialRefString) #### Check Whether the Number of Features is Appropriate #### numObs = gaInfo[0] ERROR.checkNumberOfObs(numObs, minNumObs=minNumObs, warnNumObs=warnNumObs, silentWarnings=self.silentWarnings) #### Process any bad records encountered #### numBadIDs = cnt - numObs if numBadIDs: badIDs = WU.parseGAWarnings(gaTable.warnings) if not self.silentWarnings: ERROR.reportBadRecords(cnt, numBadIDs, badIDs, label=self.oidName) else: badIDs = [] #### Initialization of Centroids #### xyCoords = NUM.empty((numObs, 2), float) #### Z Coords #### if self.hasZ: zCoords = NUM.empty((numObs, ), float) #### Create Empty Data Arrays #### for fieldName, fieldObj in self.fields.iteritems(): fieldObj.createDataArray(numObs) #### Populate SSDataObject #### ARCPY.SetProgressor("step", ARCPY.GetIDMessage(84001), 0, numObs, 1) for row in xrange(numObs): rowInfo = gaTable[row] x, y = rowInfo[1] masterID = int(rowInfo[self.masterColumnIndex]) if self.master2Order.has_key(masterID): ARCPY.AddIDMessage("ERROR", 644, masterField) ARCPY.AddIDMessage("ERROR", 643) raise SystemExit() else: self.master2Order[masterID] = row self.order2Master[row] = masterID xyCoords[row] = (x, y) if numFields: restFields = rowInfo[self.dataColumnIndex:] for fieldInd, fieldName in enumerate(fields): self.fields[fieldName].data[row] = restFields[fieldInd] if self.hasZ: if getZBool: zCoords[row] = rowInfo[-1] else: zCoords[row] = NUM.nan ARCPY.SetProgressorPosition() #### Set the Hidden Fields (E.g. Not in Use) #### self.setHiddenFields() #### Reset Extent to Honor Env and Subsets #### try: self.extent = UTILS.resetExtent(xyCoords) except: pass #### Reset Coordinates for Chordal #### if self.useChordal: #### Project to XY on Spheroid #### self.spheroidCoords = ARC._ss.lonlat_to_xy(xyCoords, self.spatialRef) self.sliceInfo = UTILS.SpheroidSlice(self.extent, self.spatialRef) else: self.spheroidCoords = None self.sliceInfo = None #### Set Further Attributes #### self.badRecords = badIDs self.xyCoords = xyCoords self.masterField = masterField self.gaTable = gaTable self.numObs = numObs if self.hasZ: self.zCoords = zCoords else: self.zCoords = None