def runDatasetChecks(dataset,table,qaRulesDataset): qaRules = qaRulesDataset.split(",") success = True for rule in qaRules: if rule == "RepairGeometry": i = 0 count = 1 gzSupport.addMessage("Running " + rule + " for " + table) while i < 3 and count > 0: arcpy.RepairGeometry_management(table) count = checkGeometry(table) i += 1 if count > 0: err = str(count) + " Geometry Errors found after repairing " + str(i) + " times" gzSupport.addError(err) gzSupport.logProcessError(table,rule,rule,str(count),err) success = False else: gzSupport.addMessage("Geometry successfully repaired") elif rule == "CheckGeometry": gzSupport.addMessage("Running " + rule + " for " + table) count = checkGeometry(table) if count > 0: success = False gzSupport.logProcessError(table,rule,rule,str(count),"Geometry Errors Found") return success
def exportDataset(sourceLayer, targetName, dataset): result = True targetTable = os.path.join(gzSupport.workspace, targetName) gzSupport.addMessage("Exporting Layer from " + sourceLayer) whereClause = "" try: try: whereClause = gzSupport.getNodeValue(dataset, "WhereClause") except: whereClause = '' gzSupport.addMessage("Where '" + whereClause + "'") sourceName = sourceLayer[sourceLayer.rfind(os.sep) + 1:sourceLayer.lower().rfind(".lyr")] viewName = sourceName + "_View" xmlFields = xmlDoc.getElementsByTagName("Field") view = gzSupport.makeFeatureViewForLayer(gzSupport.workspace, sourceLayer, viewName, whereClause, xmlFields) count = arcpy.GetCount_management(view).getOutput(0) gzSupport.addMessage(str(count) + " source rows") arcpy.FeatureClassToFeatureClass_conversion(view, gzSupport.workspace, targetName) except: err = "Failed to create new dataset " + targetName gzSupport.showTraceback() gzSupport.addError(err) gzSupport.logProcessError(sourceLayer, gzSupport.sourceIDField, sourceLayer, targetName, err) result = False return result
def checkValueMaps(dataset,table,field,fieldName,mapName): global valueMaps method = gzSupport.getNodeValue(field,"Method") success = True if method == "ValueMap": fieldMapName = gzSupport.getNodeValue(field,"ValueMapName") otherwise = gzSupport.getNodeValue(field,"ValueMapOtherwise") found = False for map in valueMaps: mapNodeName = map.getAttributeNode("name").nodeValue if mapNodeName == fieldMapName and not found: found = True # it is possible for the same value map to be present in multiple gizinta project files, just use the first one. mapValues = gzSupport.getNodeValue(map,mapName).split(",") if otherwise != None and otherwise != '' and otherwise not in mapValues and not otherwise.count(" ") > 2: mapValues.append(otherwise) values = gzSupport.getFieldValues("Unique",[fieldName],[dataset]) uniqueValues = values[0] #delta = len(uniqueValues[0]) - len(mapValues) mismatch = [] for uVal in uniqueValues: if uVal not in mapValues: mismatch.append(uVal) if len(mismatch) > 0 and not otherwise.count(" ") > 2: gzSupport.addError(str(len(mismatch)) + " mismatches for " + fieldName + ", results located in " + gzSupport.errorTableName) for uVal in mismatch: gzSupport.addError("'" + str(uVal) + "' not found in value map " + str(fieldMapName)) gzSupport.logProcessError(table,gzSupport.sourceIDField,"",fieldName,"Mismatched Value Map:" + str(uVal)) success = False elif len(mismatch) == 0: gzSupport.addMessage("No mismatches found for ValueMaps") return success
def checkValueMaps(dataset,table,field,fieldName,mapName): global valueMaps method = gzSupport.getNodeValue(field,"Method") success = True if method == "ValueMap": fieldMapName = gzSupport.getNodeValue(field,"ValueMapName") otherwise = gzSupport.getNodeValue(field,"ValueMapOtherwise") for map in valueMaps: mapNodeName = map.getAttributeNode("name").nodeValue if mapNodeName == fieldMapName: mapValues = gzSupport.getNodeValue(map,mapName).split(",") if otherwise != None and otherwise != '' and otherwise not in mapValues and not otherwise.count(" ") > 2: mapValues.append(otherwise) values = gzSupport.getFieldValues("Unique",[fieldName],[dataset]) uniqueValues = values[0] #delta = len(uniqueValues[0]) - len(mapValues) mismatch = [] for uVal in uniqueValues: if uVal not in mapValues: mismatch.append(uVal) if len(mismatch) > 0 and not otherwise.count(" ") > 2: gzSupport.addError(str(len(mismatch)) + " mismatches for " + fieldName + ", results located in " + gzSupport.errorTableName) for uVal in mismatch: gzSupport.addError("'" + str(uVal) + "' not found in value map " + str(fieldMapName)) gzSupport.logProcessError(table,gzSupport.sourceIDField,"",fieldName,"Mismatched Value Map:" + str(uVal)) success = False elif len(mismatch) == 0: gzSupport.addMessage("No mismatches found for ValueMaps") return success
def exportDataset(sourceLayer, targetName, dataset): result = True targetTable = os.path.join(gzSupport.workspace, targetName) gzSupport.addMessage("Exporting Layer from " + sourceLayer) whereClause = "" try: try: whereClause = gzSupport.getNodeValue(dataset, "WhereClause") except: whereClause = "" gzSupport.addMessage("Where '" + whereClause + "'") sourceName = sourceLayer[sourceLayer.rfind(os.sep) + 1 : sourceLayer.lower().rfind(".lyr")] viewName = sourceName + "_View" xmlFields = xmlDoc.getElementsByTagName("Field") view = gzSupport.makeFeatureViewForLayer(gzSupport.workspace, sourceLayer, viewName, whereClause, xmlFields) count = arcpy.GetCount_management(view).getOutput(0) gzSupport.addMessage(str(count) + " source rows") arcpy.FeatureClassToFeatureClass_conversion(view, gzSupport.workspace, targetName) except: err = "Failed to create new dataset " + targetName gzSupport.showTraceback() gzSupport.addError(err) gzSupport.logProcessError(sourceLayer, gzSupport.sourceIDField, sourceLayer, targetName, err) result = False return result
def importLayer(cadPath, cadName, dataset): result = False try: name = dataset.getAttributeNode("targetName").nodeValue except: name = dataset.getAttributeNode("name").nodeValue table = os.path.join(gzSupport.workspace, name) layerName = dataset.getAttributeNode("sourceName").nodeValue layer = os.path.join(cadPath, cadName, layerName) gzSupport.addMessage("Importing Layer " + layer) try: whereClause = gzSupport.getNodeValue(dataset, "WhereClause") xmlFields = dataset.getElementsByTagName("Field") gzSupport.addMessage("Where " + whereClause) if not arcpy.Exists(table): err = "Feature Class " + name + " does not exist" gzSupport.addError(err) gzSupport.logProcessError(cadName, gzSupport.sourceIDField, name, name, err) return False if whereClause != '': view = gzSupport.makeFeatureView(gzSupport.workspace, layer, layerName + "_View", whereClause, xmlFields) else: view = layer count = arcpy.GetCount_management(view).getOutput(0) gzSupport.addMessage(str(count) + " source Features for " + name) if hasJoinTo(dataset) == True: res = joinToCsv(view, dataset, cadPath, cadName) result = res[0] view = res[1] else: view = view result = True if result == True and count > 0: arcpy.Append_management([view], table, "NO_TEST", "", "") arcpy.ClearWorkspaceCache_management(gzSupport.workspace) except: err = "Failed to import layer " + name gzSupport.addError(err) gzSupport.showTraceback() gzSupport.logProcessError(cadName, gzSupport.sourceIDField, name, layerName, err) gzSupport.cleanupGarbage() try: del view except: gzSupport.addMessage("") return result
def importLayer(cadPath, cadName, dataset): result = False try: name = dataset.getAttributeNode("targetName").nodeValue except: name = dataset.getAttributeNode("name").nodeValue table = os.path.join(gzSupport.workspace, name) layerName = dataset.getAttributeNode("sourceName").nodeValue layer = os.path.join(cadPath, cadName, layerName) gzSupport.addMessage("Importing Layer " + layer) try: whereClause = gzSupport.getNodeValue(dataset, "WhereClause") xmlFields = dataset.getElementsByTagName("Field") gzSupport.addMessage("Where " + whereClause) if not arcpy.Exists(table): err = "Feature Class " + name + " does not exist" gzSupport.addError(err) gzSupport.logProcessError(cadName, gzSupport.sourceIDField, name, name, err) return False if whereClause != "": view = gzSupport.makeFeatureView(gzSupport.workspace, layer, layerName + "_View", whereClause, xmlFields) else: view = layer count = arcpy.GetCount_management(view).getOutput(0) gzSupport.addMessage(str(count) + " source Features for " + name) if hasJoinTo(dataset) == True: res = joinToCsv(view, dataset, cadPath, cadName) result = res[0] view = res[1] else: view = view result = True if result == True and count > 0: arcpy.Append_management([view], table, "NO_TEST", "", "") arcpy.ClearWorkspaceCache_management(gzSupport.workspace) except: err = "Failed to import layer " + name gzSupport.addError(err) gzSupport.showTraceback() gzSupport.logProcessError(cadName, gzSupport.sourceIDField, name, layerName, err) gzSupport.cleanupGarbage() try: del view except: gzSupport.addMessage("") return result
def findDuplicates(dataset,table,field): success = True uValues = gzSupport.getFieldValues("Unique",[field],[dataset]) uniqueValues = uValues[0] diffValues = uValues[1] fieldValues = gzSupport.getFieldValues("All",[field],[dataset])[0] delta = len(diffValues) if delta > 0: gzSupport.addMessage(str(len(fieldValues)) + " All : " + str(len(uniqueValues)) + " Unique") gzSupport.addError(str(delta) + " Duplicates found, results located in " + gzSupport.errorTableName) for x in diffValues: gzSupport.logProcessError(table,field,str(x),field,"Duplicate Value:" + str(x)) success = False elif delta == 0: gzSupport.addMessage("No Duplicates found") return success
def findDuplicates(dataset,table,field): success = True uValues = gzSupport.getFieldValues("Unique",[field],[dataset]) uniqueValues = uValues[0] diffValues = uValues[1] #fieldValues = gzSupport.getFieldValues("All",[field],[dataset])[0] delta = len(diffValues) if delta > 0: count = int(arcpy.GetCount_management(table).getOutput(0)) gzSupport.addMessage(str(count) + " rows : " + str(len(uniqueValues)) + " Unique") gzSupport.addError(str(delta) + " Duplicates found, results located in " + gzSupport.errorTableName) for x in diffValues: gzSupport.logProcessError(table,field,str(x),field,"Duplicate Value:" + str(x)) success = False elif delta == 0: gzSupport.addMessage("No Duplicates found") return success
def runOneFieldCheck(dataset,table,field,fieldTag): success = True fieldVal = field.getElementsByTagName(fieldTag)[0] qaRulesField = fieldVal.getAttributeNode("qa").nodeValue fieldName = gzSupport.getNodeValue(field,fieldTag) gzSupport.addMessage("Field QA (" + qaRulesField + ")for " + fieldName) if fieldTag == "SourceName": mapName = "SourceValues" else: mapName = "TargetValues" if qaRulesField.find("Unique") > -1: retVal = findDuplicates(dataset,table,fieldName) if retVal == False: success = False gzSupport.logProcessError(table,"FieldName",fieldName,"","Duplicate values found") if qaRulesField.find("Required") > -1 and qaRulesField.find("Unique") == -1: retVal = getCountNullBlank(table,fieldName,"") if retVal == False: success = False gzSupport.logProcessError(table,"FieldName",fieldName,"","Null or blank values found") if qaRulesField.find("ValueMaps") > -1: retVal = checkValueMaps(dataset,table,field,fieldName,mapName) if retVal == False: success = False gzSupport.logProcessError(table,"FieldName",fieldName,"","Values found that do not match ValueMaps") if qaRulesField.find("Check") > -1: retVal = getCountNullBlank(table,fieldName,"") #retVal = findDuplicates(dataset,table,fieldName) retVal = checkValueMaps(dataset,table,field,fieldName,mapName) return success
def joinToCsv(view, dataset, cadPath, cadName): retVal = False joinTo = "" if hasJoinTo(dataset) == True: try: joinTo = dataset.getAttributeNode("joinTo").nodeValue cadPart0 = cadName.split(".dwg")[0] csvFile = os.path.join(cadPath, cadPart0, cadPart0 + joinTo) if joinTo and joinTo != "": cadKey = dataset.getAttributeNode("cadKey").nodeValue csvKey = dataset.getAttributeNode("csvKey").nodeValue prefix = dataset.getAttributeNode("fieldPrefix").nodeValue tempTable = os.path.join(gzSupport.workspace, prefix) # Create temporary table if arcpy.Exists(tempTable): arcpy.Delete_management(tempTable) if os.path.isfile(csvFile) == True: arcpy.CopyRows_management(csvFile, tempTable) arcpy.AddJoin_management(view, cadKey, tempTable, csvKey) retVal = True else: err = "Missing csv file - " + csvFile gzSupport.addError(err) gzSupport.logProcessError(cadName, gzSupport.sourceIDField, name, csvFile, err) retVal = False except: err = "Unable to create join for " + name + ", " + csvFile gzSupport.logProcessError(cadName, gzSupport.sourceIDField, name, csvFile, err) gzSupport.addError(err) gzSupport.showTraceback() retVal = False #finally: # if arcpy.Exists(tempTable): # arcpy.Delete_management(tempTable) return [retVal, view]
def joinToCsv(view, dataset, cadPath, cadName): retVal = False joinTo = "" if hasJoinTo(dataset) == True: try: joinTo = dataset.getAttributeNode("joinTo").nodeValue cadPart0 = cadName.split(".dwg")[0] csvFile = os.path.join(cadPath, cadPart0, cadPart0 + joinTo) if joinTo and joinTo != "": cadKey = dataset.getAttributeNode("cadKey").nodeValue csvKey = dataset.getAttributeNode("csvKey").nodeValue prefix = dataset.getAttributeNode("fieldPrefix").nodeValue tempTable = os.path.join(gzSupport.workspace, prefix) # Create temporary table if arcpy.Exists(tempTable): arcpy.Delete_management(tempTable) if os.path.isfile(csvFile) == True: arcpy.CopyRows_management(csvFile, tempTable) arcpy.AddJoin_management(view, cadKey, tempTable, csvKey) retVal = True else: err = "Missing csv file - " + csvFile gzSupport.addError(err) gzSupport.logProcessError(cadName, gzSupport.sourceIDField, name, csvFile, err) retVal = False except: err = "Unable to create join for " + name + ", " + csvFile gzSupport.logProcessError(cadName, gzSupport.sourceIDField, name, csvFile, err) gzSupport.addError(err) gzSupport.showTraceback() retVal = False # finally: # if arcpy.Exists(tempTable): # arcpy.Delete_management(tempTable) return [retVal, view]
if mapExpr and mapExpr != "": currentValue = calcValue(row,attrs,mapExpr) if currentValue == sourceTest or currentValue == sourceValue: # this will check numeric and non-numeric equivalency for current values in value maps found = True try: idx = sourceValues.index(sourceValue) newValue = targetValues[idx] row.setValue(targetName,newValue) except: errCount += 1 row.setValue(targetName,currentValue) success = False err = "Unable to map values for " + targetName + ", value = " + str(newValue) gzSupport.showTraceback() gzSupport.addError(err) gzSupport.logProcessError(row.getValue(gzSupport.sourceNameField),gzSupport.sourceIDField,row.getValue(gzSupport.sourceIDField),targetName,err) if not found: if otherwise and str(otherwise) != "None": otherwise = str(otherwise) if otherwise.count(" ") > 2 or otherwise.count("!") > 1: otherwise = calcValue(row,attrs,otherwise) #gzSupport.addMessage(otherwise) row.setValue(targetName,otherwise) else: errCount += 1 success = False err = "Unable to find map value (otherwise) for " + str(targetName) + ", value = " + str(currentValue) gzSupport.addError(err) gzSupport.logProcessError(row.getValue(gzSupport.sourceNameField),gzSupport.sourceIDField,row.getValue(gzSupport.sourceIDField),targetName,err)
sourceValue) newValue = targetValues[idx] row.setValue(targetName, newValue) except: errCount += 1 row.setValue( targetName, currentValue) success = False err = "Unable to map values for " + targetName + ", value = " + str( newValue) gzSupport.showTraceback() gzSupport.addError(err) gzSupport.logProcessError( row.getValue( gzSupport.sourceNameField), gzSupport.sourceIDField, row.getValue( gzSupport.sourceIDField), targetName, err) if not found: if otherwise and str(otherwise) != "None": otherwise = str(otherwise) if otherwise.count(" ") > 2 or otherwise.count( "!") > 1: otherwise = calcValue(row, attrs, otherwise) #gzSupport.addMessage(otherwise) row.setValue(targetName, otherwise) else: errCount += 1 success = False