def runDatasetChecks(dataset,table,qaRulesDataset):
qaRules = qaRulesDataset.split(",")
success = True
for rule in qaRules:
if rule == "RepairGeometry":
i = 0
count = 1
gzSupport.addMessage("Running " + rule + " for " + table)
while i < 3 and count > 0:
arcpy.RepairGeometry_management(table)
count = checkGeometry(table)
i += 1
if count > 0:
err = str(count) + " Geometry Errors found after repairing " + str(i) + " times"
gzSupport.addError(err)
gzSupport.logProcessError(table,rule,rule,str(count),err)
success = False
else:
gzSupport.addMessage("Geometry successfully repaired")
elif rule == "CheckGeometry":
gzSupport.addMessage("Running " + rule + " for " + table)
count = checkGeometry(table)
if count > 0:
success = False
gzSupport.logProcessError(table,rule,rule,str(count),"Geometry Errors Found")
return success
def exportDataset(sourceLayer, targetName, dataset):
result = True
targetTable = os.path.join(gzSupport.workspace, targetName)
gzSupport.addMessage("Exporting Layer from " + sourceLayer)
whereClause = ""
try:
try:
whereClause = gzSupport.getNodeValue(dataset, "WhereClause")
except:
whereClause = ''
gzSupport.addMessage("Where '" + whereClause + "'")
sourceName = sourceLayer[sourceLayer.rfind(os.sep) +
1:sourceLayer.lower().rfind(".lyr")]
viewName = sourceName + "_View"
xmlFields = xmlDoc.getElementsByTagName("Field")
view = gzSupport.makeFeatureViewForLayer(gzSupport.workspace,
sourceLayer, viewName,
whereClause, xmlFields)
count = arcpy.GetCount_management(view).getOutput(0)
gzSupport.addMessage(str(count) + " source rows")
arcpy.FeatureClassToFeatureClass_conversion(view, gzSupport.workspace,
targetName)
except:
err = "Failed to create new dataset " + targetName
gzSupport.showTraceback()
gzSupport.addError(err)
gzSupport.logProcessError(sourceLayer, gzSupport.sourceIDField,
sourceLayer, targetName, err)
result = False
return result
def checkValueMaps(dataset,table,field,fieldName,mapName):
global valueMaps
method = gzSupport.getNodeValue(field,"Method")
success = True
if method == "ValueMap":
fieldMapName = gzSupport.getNodeValue(field,"ValueMapName")
otherwise = gzSupport.getNodeValue(field,"ValueMapOtherwise")
found = False
for map in valueMaps:
mapNodeName = map.getAttributeNode("name").nodeValue
if mapNodeName == fieldMapName and not found:
found = True # it is possible for the same value map to be present in multiple gizinta project files, just use the first one.
mapValues = gzSupport.getNodeValue(map,mapName).split(",")
if otherwise != None and otherwise != '' and otherwise not in mapValues and not otherwise.count(" ") > 2:
mapValues.append(otherwise)
values = gzSupport.getFieldValues("Unique",[fieldName],[dataset])
uniqueValues = values[0]
#delta = len(uniqueValues[0]) - len(mapValues)
mismatch = []
for uVal in uniqueValues:
if uVal not in mapValues:
mismatch.append(uVal)
if len(mismatch) > 0 and not otherwise.count(" ") > 2:
gzSupport.addError(str(len(mismatch)) + " mismatches for " + fieldName + ", results located in " + gzSupport.errorTableName)
for uVal in mismatch:
gzSupport.addError("'" + str(uVal) + "' not found in value map " + str(fieldMapName))
gzSupport.logProcessError(table,gzSupport.sourceIDField,"",fieldName,"Mismatched Value Map:" + str(uVal))
success = False
elif len(mismatch) == 0:
gzSupport.addMessage("No mismatches found for ValueMaps")
return success
def checkValueMaps(dataset,table,field,fieldName,mapName):
global valueMaps
method = gzSupport.getNodeValue(field,"Method")
success = True
if method == "ValueMap":
fieldMapName = gzSupport.getNodeValue(field,"ValueMapName")
otherwise = gzSupport.getNodeValue(field,"ValueMapOtherwise")
for map in valueMaps:
mapNodeName = map.getAttributeNode("name").nodeValue
if mapNodeName == fieldMapName:
mapValues = gzSupport.getNodeValue(map,mapName).split(",")
if otherwise != None and otherwise != '' and otherwise not in mapValues and not otherwise.count(" ") > 2:
mapValues.append(otherwise)
values = gzSupport.getFieldValues("Unique",[fieldName],[dataset])
uniqueValues = values[0]
#delta = len(uniqueValues[0]) - len(mapValues)
mismatch = []
for uVal in uniqueValues:
if uVal not in mapValues:
mismatch.append(uVal)
if len(mismatch) > 0 and not otherwise.count(" ") > 2:
gzSupport.addError(str(len(mismatch)) + " mismatches for " + fieldName + ", results located in " + gzSupport.errorTableName)
for uVal in mismatch:
gzSupport.addError("'" + str(uVal) + "' not found in value map " + str(fieldMapName))
gzSupport.logProcessError(table,gzSupport.sourceIDField,"",fieldName,"Mismatched Value Map:" + str(uVal))
success = False
elif len(mismatch) == 0:
gzSupport.addMessage("No mismatches found for ValueMaps")
return success
def runDatasetChecks(dataset,table,qaRulesDataset):
qaRules = qaRulesDataset.split(",")
success = True
for rule in qaRules:
if rule == "RepairGeometry":
i = 0
count = 1
gzSupport.addMessage("Running " + rule + " for " + table)
while i < 3 and count > 0:
arcpy.RepairGeometry_management(table)
count = checkGeometry(table)
i += 1
if count > 0:
err = str(count) + " Geometry Errors found after repairing " + str(i) + " times"
gzSupport.addError(err)
gzSupport.logProcessError(table,rule,rule,str(count),err)
success = False
else:
gzSupport.addMessage("Geometry successfully repaired")
elif rule == "CheckGeometry":
gzSupport.addMessage("Running " + rule + " for " + table)
count = checkGeometry(table)
if count > 0:
success = False
gzSupport.logProcessError(table,rule,rule,str(count),"Geometry Errors Found")
return success
def exportDataset(sourceLayer, targetName, dataset):
result = True
targetTable = os.path.join(gzSupport.workspace, targetName)
gzSupport.addMessage("Exporting Layer from " + sourceLayer)
whereClause = ""
try:
try:
whereClause = gzSupport.getNodeValue(dataset, "WhereClause")
except:
whereClause = ""
gzSupport.addMessage("Where '" + whereClause + "'")
sourceName = sourceLayer[sourceLayer.rfind(os.sep) + 1 : sourceLayer.lower().rfind(".lyr")]
viewName = sourceName + "_View"
xmlFields = xmlDoc.getElementsByTagName("Field")
view = gzSupport.makeFeatureViewForLayer(gzSupport.workspace, sourceLayer, viewName, whereClause, xmlFields)
count = arcpy.GetCount_management(view).getOutput(0)
gzSupport.addMessage(str(count) + " source rows")
arcpy.FeatureClassToFeatureClass_conversion(view, gzSupport.workspace, targetName)
except:
err = "Failed to create new dataset " + targetName
gzSupport.showTraceback()
gzSupport.addError(err)
gzSupport.logProcessError(sourceLayer, gzSupport.sourceIDField, sourceLayer, targetName, err)
result = False
return result
def importLayer(cadPath, cadName, dataset):
result = False
try:
name = dataset.getAttributeNode("targetName").nodeValue
except:
name = dataset.getAttributeNode("name").nodeValue
table = os.path.join(gzSupport.workspace, name)
layerName = dataset.getAttributeNode("sourceName").nodeValue
layer = os.path.join(cadPath, cadName, layerName)
gzSupport.addMessage("Importing Layer " + layer)
try:
whereClause = gzSupport.getNodeValue(dataset, "WhereClause")
xmlFields = dataset.getElementsByTagName("Field")
gzSupport.addMessage("Where " + whereClause)
if not arcpy.Exists(table):
err = "Feature Class " + name + " does not exist"
gzSupport.addError(err)
gzSupport.logProcessError(cadName, gzSupport.sourceIDField, name,
name, err)
return False
if whereClause != '':
view = gzSupport.makeFeatureView(gzSupport.workspace, layer,
layerName + "_View", whereClause,
xmlFields)
else:
view = layer
count = arcpy.GetCount_management(view).getOutput(0)
gzSupport.addMessage(str(count) + " source Features for " + name)
if hasJoinTo(dataset) == True:
res = joinToCsv(view, dataset, cadPath, cadName)
result = res[0]
view = res[1]
else:
view = view
result = True
if result == True and count > 0:
arcpy.Append_management([view], table, "NO_TEST", "", "")
arcpy.ClearWorkspaceCache_management(gzSupport.workspace)
except:
err = "Failed to import layer " + name
gzSupport.addError(err)
gzSupport.showTraceback()
gzSupport.logProcessError(cadName, gzSupport.sourceIDField, name,
layerName, err)
gzSupport.cleanupGarbage()
try:
del view
except:
gzSupport.addMessage("")
return result
def importLayer(cadPath, cadName, dataset):
result = False
try:
name = dataset.getAttributeNode("targetName").nodeValue
except:
name = dataset.getAttributeNode("name").nodeValue
table = os.path.join(gzSupport.workspace, name)
layerName = dataset.getAttributeNode("sourceName").nodeValue
layer = os.path.join(cadPath, cadName, layerName)
gzSupport.addMessage("Importing Layer " + layer)
try:
whereClause = gzSupport.getNodeValue(dataset, "WhereClause")
xmlFields = dataset.getElementsByTagName("Field")
gzSupport.addMessage("Where " + whereClause)
if not arcpy.Exists(table):
err = "Feature Class " + name + " does not exist"
gzSupport.addError(err)
gzSupport.logProcessError(cadName, gzSupport.sourceIDField, name, name, err)
return False
if whereClause != "":
view = gzSupport.makeFeatureView(gzSupport.workspace, layer, layerName + "_View", whereClause, xmlFields)
else:
view = layer
count = arcpy.GetCount_management(view).getOutput(0)
gzSupport.addMessage(str(count) + " source Features for " + name)
if hasJoinTo(dataset) == True:
res = joinToCsv(view, dataset, cadPath, cadName)
result = res[0]
view = res[1]
else:
view = view
result = True
if result == True and count > 0:
arcpy.Append_management([view], table, "NO_TEST", "", "")
arcpy.ClearWorkspaceCache_management(gzSupport.workspace)
except:
err = "Failed to import layer " + name
gzSupport.addError(err)
gzSupport.showTraceback()
gzSupport.logProcessError(cadName, gzSupport.sourceIDField, name, layerName, err)
gzSupport.cleanupGarbage()
try:
del view
except:
gzSupport.addMessage("")
return result
def findDuplicates(dataset,table,field):
success = True
uValues = gzSupport.getFieldValues("Unique",[field],[dataset])
uniqueValues = uValues[0]
diffValues = uValues[1]
fieldValues = gzSupport.getFieldValues("All",[field],[dataset])[0]
delta = len(diffValues)
if delta > 0:
gzSupport.addMessage(str(len(fieldValues)) + " All : " + str(len(uniqueValues)) + " Unique")
gzSupport.addError(str(delta) + " Duplicates found, results located in " + gzSupport.errorTableName)
for x in diffValues:
gzSupport.logProcessError(table,field,str(x),field,"Duplicate Value:" + str(x))
success = False
elif delta == 0:
gzSupport.addMessage("No Duplicates found")
return success
def findDuplicates(dataset,table,field):
success = True
uValues = gzSupport.getFieldValues("Unique",[field],[dataset])
uniqueValues = uValues[0]
diffValues = uValues[1]
#fieldValues = gzSupport.getFieldValues("All",[field],[dataset])[0]
delta = len(diffValues)
if delta > 0:
count = int(arcpy.GetCount_management(table).getOutput(0))
gzSupport.addMessage(str(count) + " rows : " + str(len(uniqueValues)) + " Unique")
gzSupport.addError(str(delta) + " Duplicates found, results located in " + gzSupport.errorTableName)
for x in diffValues:
gzSupport.logProcessError(table,field,str(x),field,"Duplicate Value:" + str(x))
success = False
elif delta == 0:
gzSupport.addMessage("No Duplicates found")
return success
def runOneFieldCheck(dataset,table,field,fieldTag):
success = True
fieldVal = field.getElementsByTagName(fieldTag)[0]
qaRulesField = fieldVal.getAttributeNode("qa").nodeValue
fieldName = gzSupport.getNodeValue(field,fieldTag)
gzSupport.addMessage("Field QA (" + qaRulesField + ")for " + fieldName)
if fieldTag == "SourceName":
mapName = "SourceValues"
else:
mapName = "TargetValues"
if qaRulesField.find("Unique") > -1:
retVal = findDuplicates(dataset,table,fieldName)
if retVal == False:
success = False
gzSupport.logProcessError(table,"FieldName",fieldName,"","Duplicate values found")
if qaRulesField.find("Required") > -1 and qaRulesField.find("Unique") == -1:
retVal = getCountNullBlank(table,fieldName,"")
if retVal == False:
success = False
gzSupport.logProcessError(table,"FieldName",fieldName,"","Null or blank values found")
if qaRulesField.find("ValueMaps") > -1:
retVal = checkValueMaps(dataset,table,field,fieldName,mapName)
if retVal == False:
success = False
gzSupport.logProcessError(table,"FieldName",fieldName,"","Values found that do not match ValueMaps")
if qaRulesField.find("Check") > -1:
retVal = getCountNullBlank(table,fieldName,"")
#retVal = findDuplicates(dataset,table,fieldName)
retVal = checkValueMaps(dataset,table,field,fieldName,mapName)
return success
def runOneFieldCheck(dataset,table,field,fieldTag):
success = True
fieldVal = field.getElementsByTagName(fieldTag)[0]
qaRulesField = fieldVal.getAttributeNode("qa").nodeValue
fieldName = gzSupport.getNodeValue(field,fieldTag)
gzSupport.addMessage("Field QA (" + qaRulesField + ")for " + fieldName)
if fieldTag == "SourceName":
mapName = "SourceValues"
else:
mapName = "TargetValues"
if qaRulesField.find("Unique") > -1:
retVal = findDuplicates(dataset,table,fieldName)
if retVal == False:
success = False
gzSupport.logProcessError(table,"FieldName",fieldName,"","Duplicate values found")
if qaRulesField.find("Required") > -1 and qaRulesField.find("Unique") == -1:
retVal = getCountNullBlank(table,fieldName,"")
if retVal == False:
success = False
gzSupport.logProcessError(table,"FieldName",fieldName,"","Null or blank values found")
if qaRulesField.find("ValueMaps") > -1:
retVal = checkValueMaps(dataset,table,field,fieldName,mapName)
if retVal == False:
success = False
gzSupport.logProcessError(table,"FieldName",fieldName,"","Values found that do not match ValueMaps")
if qaRulesField.find("Check") > -1:
retVal = getCountNullBlank(table,fieldName,"")
#retVal = findDuplicates(dataset,table,fieldName)
retVal = checkValueMaps(dataset,table,field,fieldName,mapName)
return success
def joinToCsv(view, dataset, cadPath, cadName):
retVal = False
joinTo = ""
if hasJoinTo(dataset) == True:
try:
joinTo = dataset.getAttributeNode("joinTo").nodeValue
cadPart0 = cadName.split(".dwg")[0]
csvFile = os.path.join(cadPath, cadPart0, cadPart0 + joinTo)
if joinTo and joinTo != "":
cadKey = dataset.getAttributeNode("cadKey").nodeValue
csvKey = dataset.getAttributeNode("csvKey").nodeValue
prefix = dataset.getAttributeNode("fieldPrefix").nodeValue
tempTable = os.path.join(gzSupport.workspace, prefix)
# Create temporary table
if arcpy.Exists(tempTable):
arcpy.Delete_management(tempTable)
if os.path.isfile(csvFile) == True:
arcpy.CopyRows_management(csvFile, tempTable)
arcpy.AddJoin_management(view, cadKey, tempTable, csvKey)
retVal = True
else:
err = "Missing csv file - " + csvFile
gzSupport.addError(err)
gzSupport.logProcessError(cadName, gzSupport.sourceIDField,
name, csvFile, err)
retVal = False
except:
err = "Unable to create join for " + name + ", " + csvFile
gzSupport.logProcessError(cadName, gzSupport.sourceIDField, name,
csvFile, err)
gzSupport.addError(err)
gzSupport.showTraceback()
retVal = False
#finally:
# if arcpy.Exists(tempTable):
# arcpy.Delete_management(tempTable)
return [retVal, view]
def joinToCsv(view, dataset, cadPath, cadName):
retVal = False
joinTo = ""
if hasJoinTo(dataset) == True:
try:
joinTo = dataset.getAttributeNode("joinTo").nodeValue
cadPart0 = cadName.split(".dwg")[0]
csvFile = os.path.join(cadPath, cadPart0, cadPart0 + joinTo)
if joinTo and joinTo != "":
cadKey = dataset.getAttributeNode("cadKey").nodeValue
csvKey = dataset.getAttributeNode("csvKey").nodeValue
prefix = dataset.getAttributeNode("fieldPrefix").nodeValue
tempTable = os.path.join(gzSupport.workspace, prefix)
# Create temporary table
if arcpy.Exists(tempTable):
arcpy.Delete_management(tempTable)
if os.path.isfile(csvFile) == True:
arcpy.CopyRows_management(csvFile, tempTable)
arcpy.AddJoin_management(view, cadKey, tempTable, csvKey)
retVal = True
else:
err = "Missing csv file - " + csvFile
gzSupport.addError(err)
gzSupport.logProcessError(cadName, gzSupport.sourceIDField, name, csvFile, err)
retVal = False
except:
err = "Unable to create join for " + name + ", " + csvFile
gzSupport.logProcessError(cadName, gzSupport.sourceIDField, name, csvFile, err)
gzSupport.addError(err)
gzSupport.showTraceback()
retVal = False
# finally:
# if arcpy.Exists(tempTable):
# arcpy.Delete_management(tempTable)
return [retVal, view]
if mapExpr and mapExpr != "":
currentValue = calcValue(row,attrs,mapExpr)
if currentValue == sourceTest or currentValue == sourceValue: # this will check numeric and non-numeric equivalency for current values in value maps
found = True
try:
idx = sourceValues.index(sourceValue)
newValue = targetValues[idx]
row.setValue(targetName,newValue)
except:
errCount += 1
row.setValue(targetName,currentValue)
success = False
err = "Unable to map values for " + targetName + ", value = " + str(newValue)
gzSupport.showTraceback()
gzSupport.addError(err)
gzSupport.logProcessError(row.getValue(gzSupport.sourceNameField),gzSupport.sourceIDField,row.getValue(gzSupport.sourceIDField),targetName,err)
if not found:
if otherwise and str(otherwise) != "None":
otherwise = str(otherwise)
if otherwise.count(" ") > 2 or otherwise.count("!") > 1:
otherwise = calcValue(row,attrs,otherwise)
#gzSupport.addMessage(otherwise)
row.setValue(targetName,otherwise)
else:
errCount += 1
success = False
err = "Unable to find map value (otherwise) for " + str(targetName) + ", value = " + str(currentValue)
gzSupport.addError(err)
gzSupport.logProcessError(row.getValue(gzSupport.sourceNameField),gzSupport.sourceIDField,row.getValue(gzSupport.sourceIDField),targetName,err)
sourceValue)
newValue = targetValues[idx]
row.setValue(targetName, newValue)
except:
errCount += 1
row.setValue(
targetName, currentValue)
success = False
err = "Unable to map values for " + targetName + ", value = " + str(
newValue)
gzSupport.showTraceback()
gzSupport.addError(err)
gzSupport.logProcessError(
row.getValue(
gzSupport.sourceNameField),
gzSupport.sourceIDField,
row.getValue(
gzSupport.sourceIDField),
targetName, err)
if not found:
if otherwise and str(otherwise) != "None":
otherwise = str(otherwise)
if otherwise.count(" ") > 2 or otherwise.count(
"!") > 1:
otherwise = calcValue(row, attrs, otherwise)
#gzSupport.addMessage(otherwise)
row.setValue(targetName, otherwise)
else:
errCount += 1
success = False
