def calcValue(row,names,calcString):
# calculate a value based on source fields and/or other expressions
outVal = ""
calcList = calcString.split("|")
for strVal in calcList:
if strVal in names:
try:
fidx = names.index(strVal)
if str(row[fidx]) != row[fidx]:
outVal += str(row[fidx])
else:
outVal += '"' + str(row[fidx]) + '"'
except:
outVal += strVal
else:
outVal += strVal
if len(calcList) == 1 and outVal == '':
outVal = calcList[0]
try:
if(outVal != "" and outVal != None):
outVal = eval(outVal)
except:
dla.addMessage("Error evaluating:" + outVal)
dla.showTraceback()
dla.addError("Error calculating field values:" + outVal)
outVal = None
return outVal
def preview(xmlFileName):
global sourceLayer,targetLayer,rowLimit
dla.setWorkspace()
dla._errCount = 0
xmlDoc = dla.getXmlDoc(xmlFileName)
#arcpy.AddMessage("rowLimit = " + str(rowLimit) )
if rowLimit == "" or rowLimit == None:
rowLimit = 100
if sourceLayer == "" or sourceLayer == None:
sourceLayer = dla.getNodeValue(xmlDoc,"Source")
if targetLayer == "" or targetLayer == None:
targetLayer = dla.getNodeValue(xmlDoc,"Target")
dte = datetime.datetime.now().strftime("%Y%m%d%H%M")
targetName = dla.getTargetName(xmlDoc) + dte
targetFC = os.path.join(dla.workspace,targetName)
res = dlaExtractLayerToGDB.extract(xmlFileName,rowLimit,dla.workspace,sourceLayer,targetFC)
if res == True:
res = dlaFieldCalculator.calculate(xmlFileName,dla.workspace,targetName,False)
if res == True:
arcpy.env.addOutputsToMap = True
layer = targetName
layertmp = targetName + "tmp"
if arcpy.Exists(layertmp):
arcpy.Delete_management(layertmp)
arcpy.MakeFeatureLayer_management(targetFC,layertmp)
fieldInfo = dla.getLayerVisibility(layertmp,xmlFileName)
arcpy.MakeFeatureLayer_management(targetFC,layer,None,dla.workspace,fieldInfo)
# should make only the target fields visible
arcpy.SetParameter(_success,layer)
else:
dla.addError("Failed to Extract data")
print("Failed to Extract data")
dla.writeFinalMessage("Data Assistant - Preview")
def main(argv = None):
xmlDoc = dla.getXmlDoc(xmlFileName)
targetName = dla.getTargetName(xmlDoc)
success = calculate(xmlFileName,dla.workspace,targetName,False)
if success == False:
dla.addError("Errors occurred during field calculation")
arcpy.SetParameter(SUCCESS, success)
def calcValue(row, names, calcString):
# calculate a value based on source fields and/or other expressions
outVal = ""
calcList = calcString.split("|")
for strVal in calcList:
if strVal in names:
try:
fidx = names.index(strVal)
if str(row[fidx]) != row[fidx]:
outVal += str(row[fidx])
else:
outVal += '"' + str(row[fidx]) + '"'
except:
outVal += strVal
else:
outVal += strVal
if len(calcList) == 1 and outVal == '':
outVal = calcList[0]
try:
if (outVal != "" and outVal != None):
outVal = eval(outVal)
except:
dla.addMessage("Error evaluating:" + outVal)
dla.showTraceback()
dla.addError("Error calculating field values:" + outVal)
outVal = None
return outVal
def doPublish(xmlDoc,dlaTable,targetLayer):
# either truncate and replace or replace by field value
# run locally or update agol
success = False
expr = ''
dlaTable = handleGeometryChanges(dlaTable,targetLayer)
if useReplaceSettings == True:
expr = getWhereClause(xmlDoc)
if useReplaceSettings == True and (expr == '' or expr == None):
dla.addError("There must be an expression for replacing by field value, current value = '" + str(expr) + "'")
return False
if targetLayer.startswith("GIS Servers\\") == True or targetLayer.startswith("http") == True:
targetLayer = dla.getLayerSourceUrl(targetLayer)
success = doPublishPro(dlaTable,targetLayer,expr)
else:
# logic change - if not replace field settings then only append
if expr != '' and useReplaceSettings == True:
if dla.deleteRows(targetLayer,expr) == True:
success = dla.appendRows(dlaTable,targetLayer,expr)
else:
success = False
else:
success = dla.doInlineAppend(dlaTable,targetLayer)
return success
def doPublish(xmlDoc, dlaTable, targetLayer):
# either truncate and replace or replace by field value
# run locally or update agol
success = False
expr = ''
dlaTable = handleGeometryChanges(dlaTable, targetLayer)
if useReplaceSettings == True:
expr = getWhereClause(xmlDoc)
if useReplaceSettings == True and (expr == '' or expr == None):
dla.addError(
"There must be an expression for replacing by field value, current value = '"
+ str(expr) + "'")
return False
if targetLayer.startswith(
"GIS Servers\\") == True or targetLayer.startswith("http") == True:
targetLayer = dla.getLayerSourceUrl(targetLayer)
success = doPublishPro(dlaTable, targetLayer, expr)
else:
# logic change - if not replace field settings then only append
if expr != '' and useReplaceSettings == True:
if dla.deleteRows(targetLayer, expr) == True:
success = dla.appendRows(dlaTable, targetLayer, expr)
else:
success = False
else:
success = dla.doInlineAppend(dlaTable, targetLayer)
return success
def exportDataset(xmlDoc, sourceLayer, workspace, targetName, rowLimit):
result = True
xmlFields = xmlDoc.getElementsByTagName("Field")
dla.addMessage("Exporting Layer from " + sourceLayer)
whereClause = ""
if rowLimit != None:
# try:
whereClause = getObjectIdWhereClause(sourceLayer, rowLimit)
# except:
# dla.addMessage("Unable to obtain where clause to Preview " + sourceLayer + ", continuing with all records")
if whereClause != "" and whereClause != " ":
# dla.addMessage("rowLimit " + str(rowLimit))
dla.addMessage("Where " + str(whereClause))
sourceName = dla.getSourceName(xmlDoc)
viewName = sourceName + "_View"
dla.addMessage(viewName)
targetRef = getSpatialReference(xmlDoc, "Target")
# sourceRef = getSpatialReference(xmlDoc,"Source")
if targetRef != "":
arcpy.env.workspace = workspace
view = dla.makeFeatureView(dla.workspace, sourceLayer, viewName, whereClause, xmlFields)
dla.addMessage("View Created")
srcCount = arcpy.GetCount_management(view).getOutput(0)
dla.addMessage(str(srcCount) + " source rows")
if srcCount == 0:
result = False
dla.addError("Failed to extract " + sourceName + ", Nothing to export")
else:
arcpy.env.preserveGlobalIds = False # need to run this way until support added for GlobalIDs
# dla.addMessage("names: " + sourceName + "|" + targetName)
arcpy.env.overwriteOutput = True
try:
arcpy.CreateFeatureclass_management(
workspace, targetName, template=sourceLayer, spatial_reference=targetRef
)
except:
arcpy.AddError(
"Unable to create intermediate feature class, exiting: " + workspace + os.sep + targetName
)
return False
fc = workspace + os.sep + targetName
arcpy.Append_management(view, fc, schema_type="NO_TEST")
dla.addMessage(arcpy.GetMessages(2)) # only serious errors
count = arcpy.GetCount_management(fc).getOutput(0)
dla.addMessage(str(count) + " source rows exported to " + targetName)
if str(count) == "0":
result = False
dla.addError(
"Failed to load to "
+ targetName
+ ", it is likely that your data falls outside of the target Spatial Reference Extent"
)
dla.addMessage("To verify please use the Append tool to load some data to the target dataset")
return result
def deleteFeatures(sourceLayer,targelUrl,expr):
# delete features using chunks of _chunkSize
retval = False
error = False
# delete section
ids = getOIDs(targelUrl,expr)
try:
lenDeleted = 100
#Chunk deletes using chunk size at a time
featuresProcessed = 0
numFeat = len(ids)
if numFeat == 0:
dla.addMessage("0 Features to Delete, exiting")
return True # nothing to delete is OK
if numFeat > _chunkSize:
chunk = _chunkSize
else:
chunk = numFeat
arcpy.SetProgressor("default","Deleting Features")
while featuresProcessed < numFeat and error == False:
#Chunk deletes using chunk size at a time
next = featuresProcessed + chunk
msg = "Deleting features " + str(featuresProcessed) + ":" + str(next)
dla.addMessage(msg)
arcpy.SetProgressorLabel(msg)
oids = ",".join(str(e) for e in ids[featuresProcessed:next])
url = targelUrl + '/deleteFeatures'
token = dla.getSigninToken()
params = {'f': 'pjson', 'objectIds': oids,'token':token}
result = dla.sendRequest(url,params)
try:
if result['error'] != None:
retval = False
dla.addMessage("Delete features from Feature Service failed")
dla.addMessage(json.dumps(result))
error = True
except:
try:
lenDeleted = len(result['deleteResults'])
msg = str(lenDeleted) + " features deleted, " + str(featuresProcessed + chunk) + "/" + str(numFeat)
print(msg)
dla.addMessage(msg)
retval = True
except:
retval = False
error = True
dla.showTraceback()
dla.addMessage("Delete features from Feature Service failed")
dla.addError(json.dumps(result))
featuresProcessed += chunk
except:
retval = False
error = True
dla.showTraceback()
dla.addMessage("Delete features from Feature Service failed")
pass
return retval
def addFeatures(sourceLayer,targelUrl,expr):
# add features using _chunkSize
retval = False
error = False
# add section
try:
arcpy.SetProgressor("default","Adding Features")
arcpy.SetProgressorLabel("Adding Features")
featurejs = featureclass_to_json(sourceLayer)
url = targelUrl + '/addFeatures'
numFeat = len(featurejs['features'])
if numFeat == 0:
dla.addMessage("0 Features to Add, exiting")
return True # nothing to add is OK
if numFeat > _chunkSize:
chunk = _chunkSize
else:
chunk = numFeat
featuresProcessed = 0
while featuresProcessed < numFeat and error == False:
next = featuresProcessed + chunk
features = featurejs['features'][featuresProcessed:next]
msg = "Adding features " + str(featuresProcessed) + ":" + str(next)
dla.addMessage(msg)
arcpy.SetProgressorLabel(msg)
token = dla.getSigninToken()
params = {'rollbackonfailure': 'true','f':'json', 'token':token, 'features': json.dumps(features)}
result = dla.sendRequest(url,params)
try:
if result['error'] != None:
retval = False
dla.addMessage("Add features to Feature Service failed")
dla.addMessage(json.dumps(result))
error = True
except:
try:
lenAdded = len(result['addResults'])
msg = str(lenAdded) + " features added, " + str(featuresProcessed + chunk) + "/" + str(numFeat)
print(msg)
dla.addMessage(msg)
retval = True
except:
retval = False
dla.addMessage("Add features to Feature Service failed")
dla.showTraceback()
dla.addError(json.dumps(result))
error = True
featuresProcessed += chunk
except:
retval = False
dla.addMessage("Add features to Feature Service failed")
dla.showTraceback()
error = True
pass
return retval
def isFeatureLayerUrl(url):
# assume layer string has already had \ and GIS Servers or other characters switched to be a url
parts = url.split('/')
lngth = len(parts)
try:
# check for number at end
# last = int(parts[lngth-1])
if parts[lngth - 2] == 'FeatureServer':
return True
except:
dla.addError("2nd last part of url != 'FeatureServer'")
return False
def createDlaFile(source,target,xmlFileName):
# entry point for calling this tool from another python script
if str(source) == '' or str(target) == '':
dla.addError("This tool requires both a source and target dataset, exiting.")
elif source == target:
dla.addError("2 layers with the same name is not supported by this tool, please rename one of the layers, exiting.")
else:
sourcePath = dla.getLayerPath(source)
targetPath = dla.getLayerPath(target)
writeDocument(sourcePath,targetPath,xmlFileName)
return True
def getExpression(row,names,expression):
calcNew = None
try:
calcNew = calcValue(row,names,expression)
#setValue(row,targetName,sourceValue,calcNew,idx)
except:
err = "Exception caught: unable to set value for expression=" + expression
dla.showTraceback()
dla.addError(err)
print(err)
dla._errCount += 1
return calcNew
def getExpression(row, names, expression):
calcNew = None
try:
calcNew = calcValue(row, names, expression)
#setValue(row,targetName,sourceValue,calcNew,idx)
except:
err = "Exception caught: unable to set value for expression=" + expression
dla.showTraceback()
dla.addError(err)
print(err)
dla._errCount += 1
return calcNew
def getLayerPath(desc): # requires arcpy.Describe object
# altered May31 2016 to handle no .path for layer...
pth = None
try:
pth = desc.catalogPath
except:
try:
pth = desc.path
except:
dla.addError('Unable to obtain a source path for this layer. Please select a feature layer and re-run this tool')
if pth != None:
dla.addMessage(pth)
return pth
def getValueMap(row, names, sourceValue, field):
# run value map function for a row
valueMaps = field.getElementsByTagName("ValueMap")
newValue = None
found = False
otherwise = None
for valueMap in valueMaps:
try:
otherwise = valueMap.getElementsByTagName("Otherwise")[0]
otherwise = dla.getTextValue(otherwise)
except:
otherwise = None
sourceValues = []
sourceValues = valueMap.getElementsByTagName("sValue")
targetValues = []
targetValues = valueMap.getElementsByTagName("tValue")
i = 0
for val in sourceValues:
sValue = dla.getTextValue(val)
try:
sourceTest = float(sValue)
except ValueError:
sourceTest = str(sValue)
if sourceTest == '':
sourceTest = None
#if mapExpr and mapExpr != "":
# sourceValue = calcValue(row,names,mapExpr)
if sourceValue == sourceTest or sourceValue == sValue: # this will check numeric and non-numeric equivalency for current values in value maps
found = True
try:
newValue = dla.getTextValue(targetValues[i])
except:
dla._errCount += 1
success = False
err = "Unable to map values for " + sourceValue + ", value = " + str(
newValue)
dla.showTraceback()
dla.addError(err)
print(err)
i = i + 1
if not found:
if str(otherwise) != "None":
newValue = otherwise
else:
dla._errCount += 1
success = False
err = "Unable to find map value (otherwise) for " + str(
targetName) + ", value = " + str(sourceValue)
dla.addError(err)
return newValue
def exportDataset(xmlDoc, sourceLayer, workspace, targetName, rowLimit):
result = True
xmlFields = xmlDoc.getElementsByTagName("Field")
dla.addMessage("Exporting Layer from " + sourceLayer)
whereClause = ""
if rowLimit != None:
try:
whereClause = getObjectIdWhereClause(sourceLayer, rowLimit)
except:
dla.addMessage("Unable to obtain where clause to Preview " +
sourceLayer + ", continuing with all records")
if whereClause != '' and whereClause != ' ':
dla.addMessage("Where " + str(whereClause))
sourceName = dla.getSourceName(xmlDoc)
viewName = sourceName + "_View"
dla.addMessage(viewName)
targetRef = getSpatialReference(xmlDoc, "Target")
#sourceRef = getSpatialReference(xmlDoc,"Source")
if targetRef != '':
arcpy.env.workspace = workspace
view = dla.makeFeatureView(dla.workspace, sourceLayer, viewName,
whereClause, xmlFields)
dla.addMessage("View Created")
count = arcpy.GetCount_management(view).getOutput(0)
dla.addMessage(str(count) + " source rows")
arcpy.env.overwriteOutput = True
arcpy.CreateFeatureclass_management(workspace,
targetName,
template=sourceLayer,
spatial_reference=targetRef)
fc = workspace + os.sep + targetName
arcpy.Append_management(view, fc, schema_type="NO_TEST")
dla.addMessage(arcpy.GetMessages(2)) # only serious errors
count = arcpy.GetCount_management(fc).getOutput(0)
dla.addMessage(str(count) + " source rows exported to " + targetName)
if str(count) == '0':
result = False
dla.addError(
"Failed to load to " + targetName +
", it is likely that your data falls outside of the target Spatial Reference Extent"
)
dla.addMessage(
"To verify please use the Append tool to load some data to the target dataset"
)
return result
def extract(xmlFileName, rowLimit, workspace, sourceLayer, targetFC):
xmlDoc = dla.getXmlDoc(xmlFileName)
if workspace == "" or workspace == "#" or workspace == None:
dla.workspace = arcpy.env.scratchGDB
else:
dla.workspace = workspace
fields = dla.getFields(xmlFileName)
success = True
name = ''
try:
if not arcpy.Exists(dla.workspace):
dla.addMessage(dla.workspace +
" does not exist, attempting to create")
dla.createGeodatabase()
if len(fields) > 0:
arcpy.SetProgressor("step", "Importing Layer...", 0, 1, 1)
if sourceLayer == '' or sourceLayer == '#':
source = dla.getNodeValue(xmlDoc, "Datasets/Source")
else:
source = sourceLayer
if targetFC == '' or targetFC == '#':
targetName = dla.getTargetName(xmlDoc)
else:
targetName = targetFC[targetFC.rfind(os.sep) + 1:]
sourceName = dla.getSourceName(xmlDoc)
arcpy.SetProgressorLabel("Loading " + sourceName + " to " +
targetName + "...")
#if not arcpy.Exists(sourceLayer):
# dla.addError("Layer " + sourceLayer + " does not exist, exiting")
# return
retVal = exportDataset(xmlDoc, sourceLayer, dla.workspace,
targetName, rowLimit)
if retVal == False:
success = False
arcpy.SetProgressorPosition()
except:
dla.addError("A Fatal Error occurred")
dla.showTraceback()
success = False
finally:
arcpy.ResetProgressor()
#arcpy.RefreshCatalog(dla.workspace)
arcpy.ClearWorkspaceCache_management(dla.workspace)
return success
def doPublishPro(sourceLayer, targelUrl, expr):
# logic for publishing to service registered on Portal or ArcGIS Online
retval = True
token = dla.getSigninToken()
if token == None:
dla.addError("Unable to retrieve token, exiting")
return False
dla.setupProxy()
if expr != '' and useReplaceSettings == True:
retval = deleteFeatures(sourceLayer, targelUrl, expr)
if retval == True:
retval = addFeatures(sourceLayer, targelUrl, expr)
return retval
def doPublishPro(sourceLayer,targelUrl,expr):
# logic for publishing to service registered on Portal or ArcGIS Online
retval = True
token = dla.getSigninToken()
if token == None:
dla.addError("Unable to retrieve token, exiting")
return False
dla.setupProxy()
if expr != '' and useReplaceSettings == True:
retval = deleteFeatures(sourceLayer,targelUrl,expr)
if retval == True:
retval = addFeatures(sourceLayer,targelUrl,expr)
return retval
def getValueMap(row,names,sourceValue,field):
# run value map function for a row
valueMaps = field.getElementsByTagName("ValueMap")
newValue = None
found = False
otherwise = None
for valueMap in valueMaps:
try:
otherwise = valueMap.getElementsByTagName("Otherwise")[0]
otherwise = dla.getTextValue(otherwise)
except:
otherwise = None
sourceValues = []
sourceValues = valueMap.getElementsByTagName("sValue")
targetValues = []
targetValues = valueMap.getElementsByTagName("tValue")
i = 0
for val in sourceValues:
sValue = dla.getTextValue(val)
try:
sourceTest = float(sValue)
except ValueError:
sourceTest = str(sValue)
if sourceTest == '':
sourceTest = None
#if mapExpr and mapExpr != "":
# sourceValue = calcValue(row,names,mapExpr)
if sourceValue == sourceTest or sourceValue == sValue: # this will check numeric and non-numeric equivalency for current values in value maps
found = True
try:
newValue = dla.getTextValue(targetValues[i])
except:
dla._errCount += 1
success = False
err = "Unable to map values for " + sourceValue + ", value = " + str(newValue)
dla.showTraceback()
dla.addError(err)
print(err)
i = i + 1
if not found:
if str(otherwise) != "None":
newValue = otherwise
else:
dla._errCount += 1
success = False
err = "Unable to find map value (otherwise) for " + str(targetName) + ", value = " + str(sourceValue)
dla.addError(err)
return newValue
def main(argv = None):
global sourceLayer,targetLayer
xmlDoc = dla.getXmlDoc(xmlFileName)
if dla.workspace == "" or dla.workspace == "#" or dla.workspace == None:
dla.workspace = arcpy.env.scratchGDB
if sourceLayer == "" or sourceLayer == None:
sourceLayer = dla.getNodeValue(xmlDoc,"Source")
if targetLayer == "" or targetLayer == None:
targetLayer = dla.getNodeValue(xmlDoc,"Target")
if success == False:
dla.addError("Errors occurred during process")
success = extract(xmlFileName,rowLimit,dla.workspace,sourceLayer,targetLayer)
arcpy.SetParameter(SUCCESS, success)
def main(argv = None):
global sourceLayer,targetLayer
xmlDoc = dla.getXmlDoc(xmlFileName)
if dla.workspace == "" or dla.workspace == "#" or dla.workspace == None:
dla.workspace = arcpy.env.scratchGDB
if sourceLayer == "" or sourceLayer == None:
sourceLayer = dla.getNodeValue(xmlDoc,"Source")
if targetLayer == "" or targetLayer == None:
targetLayer = dla.getNodeValue(xmlDoc,"Target")
if success == False:
dla.addError("Errors occurred during process")
success = extract(xmlFileName,rowLimit,dla.workspace,sourceLayer,targetLayer)
arcpy.SetParameter(SUCCESS, success)
def getLayerPath(desc): # requires arcpy.Describe object
# altered May31 2016 to handle no .path for layer...
pth = None
try:
pth = desc.catalogPath
except:
try:
pth = desc.path
except:
dla.addError(
'Unable to obtain a source path for this layer. Please select a feature layer and re-run this tool'
)
if pth != None:
dla.addMessage(pth)
return pth
def extract(xmlFileName,rowLimit,workspace,sourceLayer,targetFC):
xmlDoc = dla.getXmlDoc(xmlFileName)
if workspace == "" or workspace == "#" or workspace == None:
dla.workspace = arcpy.env.scratchGDB
else:
dla.workspace = workspace
fields = dla.getFields(xmlFileName)
success = True
name = ''
try:
if not arcpy.Exists(dla.workspace):
dla.addMessage(dla.workspace + " does not exist, attempting to create")
dla.createGeodatabase()
if len(fields) > 0:
arcpy.SetProgressor("step", "Importing Layer...",0,1,1)
if sourceLayer == '' or sourceLayer == '#':
source = dla.getNodeValue(xmlDoc,"Datasets/Source")
else:
source = sourceLayer
if targetFC == '' or targetFC == '#':
targetName = dla.getTargetName(xmlDoc)
else:
targetName = targetFC[targetFC.rfind(os.sep)+1:]
sourceName = dla.getSourceName(xmlDoc)
arcpy.SetProgressorLabel("Loading " + sourceName + " to " + targetName +"...")
#if not arcpy.Exists(sourceLayer):
# dla.addError("Layer " + sourceLayer + " does not exist, exiting")
# return
retVal = exportDataset(xmlDoc,sourceLayer,dla.workspace,targetName,rowLimit)
if retVal == False:
success = False
arcpy.SetProgressorPosition()
except:
dla.addError("A Fatal Error occurred")
dla.showTraceback()
success = False
finally:
arcpy.ResetProgressor()
#arcpy.RefreshCatalog(dla.workspace)
arcpy.ClearWorkspaceCache_management(dla.workspace)
return success
def createDataset(dsType, workspace, targetName, geomType, xmlDoc, source,
targetRef):
if source.lower().endswith('.lyrx') and dla.hasJoin(source):
if dsType == 'Table':
arcpy.CreateTable_management(workspace, targetName)
else:
arcpy.CreateFeatureclass_management(workspace,
targetName,
geometry_type=geomType,
spatial_reference=targetRef)
sourceFields = xmlDoc.getElementsByTagName("SourceField")
for sfield in sourceFields:
# <SourceField AliasName="FIPS_CNTRY" Length="2" Name="SampleData.FIPS_CNTRY" Type="String" />
fname = sfield.getAttributeNode('Name').nodeValue
if fname.count('.') > 0:
fname = fname.replace('.', '_')
ftype = sfield.getAttributeNode('Type').nodeValue
flength = sfield.getAttributeNode('Length').nodeValue
dla.addDlaField(os.path.join(workspace,
targetName), fname, sfield, [],
ftype, flength) # attrs is empty list
else:
if dsType == 'Table':
try:
arcpy.CreateTable_management(workspace,
targetName,
template=source)
except:
dla.addError("Unable to Create intermediate table")
return False
else:
try:
arcpy.CreateFeatureclass_management(
workspace,
targetName,
template=source,
spatial_reference=targetRef)
except:
dla.addError("Unable to Create intermediate feature class")
return False
return True
def extract(xmlFileName, rowLimit, workspace, source, target, datasetType):
xmlDoc = dla.getXmlDoc(xmlFileName)
if workspace == "" or workspace == "#" or workspace == None:
dla.workspace = dla.setWorkspace()
else:
dla.workspace = workspace
fields = dla.getFields(xmlFileName)
success = True
name = ''
try:
if not arcpy.Exists(dla.workspace):
dla.addMessage(dla.workspace +
" does not exist, attempting to create")
dla.createGeodatabase()
if len(fields) > 0:
arcpy.SetProgressor("step", "Importing Layer...", 0, 1, 1)
targetName = dla.getDatasetName(target)
sourceName = dla.getDatasetName(source)
arcpy.SetProgressorLabel("Loading " + sourceName + " to " +
targetName + "...")
if not arcpy.Exists(source):
dla.addError("Layer " + source + " does not exist, exiting")
return
retVal = exportDataset(xmlDoc, source, dla.workspace, targetName,
rowLimit, datasetType)
if retVal == False:
success = False
arcpy.SetProgressorPosition()
except:
dla.addError("A Fatal Error occurred")
dla.showTraceback()
success = False
finally:
arcpy.ResetProgressor()
#arcpy.RefreshCatalog(dla.workspace)
arcpy.ClearWorkspaceCache_management(dla.workspace)
return success
def theProjectWay():
"""
This function is currently not used. It is an alternative to the create feature class/append approach
currently being used. It is slower because the entire dataset is projected first, and it is less
straightforward because it adds the transform method that append seems to know how to handle already.
It is better though because it will actually raise trappable errors while Append fails silently...
The solution in the other function is to count the resulting records and report issues.
"""
if targetRef != '':
if arcpy.Exists(targetName):
arcpy.Delete_management(targetName)
inttable = workspace + os.sep + targetName + "_prj"
arcpy.env.workspace = workspace
xform = None
desc = arcpy.Describe(sourceLayer)
xforms = arcpy.ListTransformations(desc.spatialReference, targetRef,
desc.extent)
#if sourceRef.exportToString().find("NAD_1983") > -1 and targetRef.exportToString().find("WGS_1984") > -1:
xform = xforms[0]
#for xform in xforms:
dla.addMessage("Transform: " + xform)
try:
res = arcpy.Project_management(sourceLayer,
inttable,
out_coor_system=targetRef,
transform_method=xform)
except:
dla.showTraceback()
err = "Unable to project the data to the target spatial reference, please check settings and try projecting manually in ArcGIS"
dla.addError(err)
return False
dla.addMessage("Features projected")
view = dla.makeFeatureViewForLayer(dla.workspace, inttable, viewName,
whereClause, xmlFields)
dla.addMessage("View Created")
#except:
# arcpy.AddError("Unabled to create feature View " + viewName)
count = arcpy.GetCount_management(view).getOutput(0)
dla.addMessage(str(count) + " source rows")
#sourceRef = getSpatialReference(xmlDoc,"Source")
#res = arcpy.CreateFeatureclass_management(workspace,targetName,template=sourceLayer,spatial_reference=targetRef)
res = arcpy.CopyFeatures_management(view, targetName)
dla.addMessage("Features copied")
def doPublishHttp(source, targetUrl, expr, useReplaceSettings):
# logic for publishing to service registered on Portal or ArcGIS Online
retval = True
token = getSigninToken()
if token == None:
dla.addError("Unable to retrieve token, exiting")
return False
dla.setupProxy()
if expr != '' and useReplaceSettings == True:
arcpy.SetProgressor("default", "Deleting Existing Rows")
arcpy.SetProgressorLabel("Deleting Existing Rows")
retval = deleteRows(source, targetUrl, expr)
if retval == True:
arcpy.SetProgressor("default", "Appending Rows")
arcpy.SetProgressorLabel("Appending Rows")
#retval = dla.appendRows(source,targetUrl,expr) does this work from a local file gdb? I can't make it...
retval = addRows(source, targetUrl, expr)
return retval
def main(argv=None):
global source, target
xmlDoc = dla.getXmlDoc(xmlFileName)
if dla.workspace == "" or dla.workspace == "#" or dla.workspace == None:
dla.workspace = arcpy.env.scratchGDB
if source == "" or source == None:
source = dla.getDatasetPath(xmlDoc, "Source")
if target == "" or target == None:
target = dla.getDatasetPath(xmlDoc, "Target")
if success == False:
dla.addError("Errors occurred during process")
if dla.isTable(source) or dla.isTable(target):
datasetType = 'Table'
else:
datasetType = 'FeatureClass'
success = extract(xmlFileName, rowLimit, dla.workspace, source, target,
datasetType)
arcpy.SetParameter(SUCCESS, success)
def getLayerPath(pth): # requires string for layer argument
# altered May31 2016 to handle no .path for layer...
# altered August 2016 - Pro 1.3.1 changes in urls for feature service layers
if pth != None:
pthurl = dla.getLayerSourceUrl(pth)
desc = arcpy.Describe(pthurl)
try:
pth = desc.catalogPath
#dla.addMessage("catalogPath:" + pth)
except:
try:
pth = desc.path
#dla.addMessage("path:" + pth)
except:
dla.addError('Unable to obtain a source path for this layer. Please select a feature layer and re-run this tool')
pth = None
if pth != None:
pth = dla.getLayerServiceUrl(pth)
dla.addMessage("Output path:" + pth)
return pth
def exportDataset(xmlDoc,sourceLayer,workspace,targetName,rowLimit):
result = True
xmlFields = xmlDoc.getElementsByTagName("Field")
dla.addMessage("Exporting Layer from " + sourceLayer)
whereClause = ""
if rowLimit != None:
try:
whereClause = getObjectIdWhereClause(sourceLayer,rowLimit)
except:
dla.addMessage("Unable to obtain where clause to Preview " + sourceLayer + ", continuing with all records")
if whereClause != '' and whereClause != ' ':
dla.addMessage("Where " + str(whereClause))
sourceName = dla.getSourceName(xmlDoc)
viewName = sourceName + "_View"
dla.addMessage(viewName)
targetRef = getSpatialReference(xmlDoc,"Target")
#sourceRef = getSpatialReference(xmlDoc,"Source")
if targetRef != '':
arcpy.env.workspace = workspace
view = dla.makeFeatureView(dla.workspace,sourceLayer,viewName,whereClause,xmlFields)
dla.addMessage("View Created")
count = arcpy.GetCount_management(view).getOutput(0)
dla.addMessage(str(count) + " source rows")
arcpy.env.overwriteOutput = True
arcpy.CreateFeatureclass_management(workspace,targetName,template=sourceLayer,spatial_reference=targetRef)
fc = workspace + os.sep + targetName
arcpy.Append_management(view,fc,schema_type="NO_TEST")
dla.addMessage(arcpy.GetMessages(2)) # only serious errors
count = arcpy.GetCount_management(fc).getOutput(0)
dla.addMessage(str(count) + " source rows exported to " + targetName)
if str(count) == '0':
result = False
dla.addError("Failed to load to " + targetName + ", it is likely that your data falls outside of the target Spatial Reference Extent")
dla.addMessage("To verify please use the Append tool to load some data to the target dataset")
return result
def hasCapabilities(url, token, checkList):
hasit = False
if token != None and isFeatureLayerUrl(url):
params = {'f': 'pjson', 'token': token}
response = sendRequest(url, params)
if response != None:
try:
error = json.dumps(response['error'])
dla.addError('Unable to access service properties ' + error)
return False
except:
hasit = True
try:
capabilities = json.dumps(response['capabilities'])
dla.addMessage('Service REST capabilities: ' + capabilities)
for item in checkList:
if capabilities.find(item) == -1:
#dla.addMessage('Service does not support: ' + item)
hasit = False
else:
dla.addMessage('Service supports: ' + item)
except:
dla.addError('Unable to access service capabilities')
hasit = False
else:
dla.addError('Unable to access service')
hasit = False
return hasit
def theProjectWay():
"""
This function is currently not used. It is an alternative to the create feature class/append approach
currently being used. It is slower because the entire dataset is projected first, and it is less
straightforward because it adds the transform method that append seems to know how to handle already.
It is better though because it will actually raise trappable errors while Append fails silently...
The solution in the other function is to count the resulting records and report issues.
"""
if targetRef != '':
if arcpy.Exists(targetName):
arcpy.Delete_management(targetName)
inttable = workspace+os.sep+targetName+"_prj"
arcpy.env.workspace = workspace
xform = None
desc = arcpy.Describe(sourceLayer)
xforms = arcpy.ListTransformations(desc.spatialReference, targetRef, desc.extent)
#if sourceRef.exportToString().find("NAD_1983") > -1 and targetRef.exportToString().find("WGS_1984") > -1:
xform = xforms[0]
#for xform in xforms:
dla.addMessage("Transform: " + xform)
try:
res = arcpy.Project_management(sourceLayer,inttable,out_coor_system=targetRef,transform_method=xform)
except:
dla.showTraceback()
err = "Unable to project the data to the target spatial reference, please check settings and try projecting manually in ArcGIS"
dla.addError(err)
return False
dla.addMessage("Features projected")
view = dla.makeFeatureViewForLayer(dla.workspace,inttable,viewName,whereClause,xmlFields)
dla.addMessage("View Created")
#except:
# arcpy.AddError("Unabled to create feature View " + viewName)
count = arcpy.GetCount_management(view).getOutput(0)
dla.addMessage(str(count) + " source rows")
#sourceRef = getSpatialReference(xmlDoc,"Source")
#res = arcpy.CreateFeatureclass_management(workspace,targetName,template=sourceLayer,spatial_reference=targetRef)
res = arcpy.CopyFeatures_management(view,targetName)
dla.addMessage("Features copied")
def preview(xmlFileName):
global sourceLayer, targetLayer, rowLimit
dla.setWorkspace()
dla._errorCount = 0
xmlDoc = dla.getXmlDoc(xmlFileName)
if rowLimit == "" or rowLimit == None:
rowLimit = 100
if sourceLayer == "" or sourceLayer == None:
sourceLayer = dla.getNodeValue(xmlDoc, "Source")
if targetLayer == "" or targetLayer == None:
targetLayer = dla.getNodeValue(xmlDoc, "Target")
dte = datetime.datetime.now().strftime("%Y%m%d%H%M")
targetName = dla.getTargetName(xmlDoc) + dte
targetFC = os.path.join(dla.workspace, targetName)
res = dlaExtractLayerToGDB.extract(xmlFileName, rowLimit, dla.workspace,
sourceLayer, targetFC)
if res == True:
res = dlaFieldCalculator.calculate(xmlFileName, dla.workspace,
targetName, False)
if res == True:
arcpy.env.addOutputsToMap = True
layer = targetName
layertmp = targetName + "tmp"
if arcpy.Exists(layertmp):
arcpy.Delete_management(layertmp)
arcpy.MakeFeatureLayer_management(targetFC, layertmp)
fieldInfo = dla.getLayerVisibility(layertmp, xmlFileName)
arcpy.MakeFeatureLayer_management(targetFC, layer, None,
dla.workspace, fieldInfo)
# should make only the target fields visible
arcpy.SetParameter(_success, layer)
else:
dla.addError("Failed to Extract data")
print("Failed to Extract data")
dla.writeFinalMessage("Data Assistant - Preview")
def getDomainMap(row, sourceValue, field):
# run domain map function for a row
valueMaps = field.getElementsByTagName("DomainMap")
newValue = sourceValue
found = False
otherwise = None
for valueMap in valueMaps:
sourceValues = []
sourceValues = valueMap.getElementsByTagName("sValue")
targetValues = []
targetValues = valueMap.getElementsByTagName("tValue")
i = 0
for val in sourceValues:
sValue = dla.getTextValue(val)
try:
sourceTest = float(sValue)
except ValueError:
sourceTest = str(sValue)
if sourceTest == '' or sourceTest == 'None':
sourceTest = None
if sourceValue == sourceTest or sourceValue == sValue: # this will check numeric and non-numeric equivalency for current values in maps
found = True
try:
newValue = dla.getTextValue(targetValues[i])
except:
dla._errCount += 1
success = False
err = "Unable to map values for " + sourceValue + ", value = " + str(
newValue)
dla.showTraceback()
dla.addError(err)
print(err)
i = i + 1
return newValue
def createDlaFile(source,target,xmlFileName):
# entry point for calling this tool from another python script
res = False
if str(source) == '' or str(target) == '':
dla.addError("This tool requires both a source and target dataset, exiting.")
elif str(source) == str(target):
dla.addError("2 string layers with the same value is not supported by this tool, please rename one of the layers, exiting.")
else:
prj = dla.getProject()
sourcePath = dla.getLayerPath(source)
targetPath = dla.getLayerPath(target)
if sourcePath == '' or targetPath == '':
if sourcePath == '':
dla.addError("Invalid Path/Type for Source layer , exiting: " + str(source) )
if targetPath == '':
dla.addError("Invalid Path/Type for Target layer, exiting: " + str(target) )
return res
else:
res = writeDocument(sourcePath,targetPath,xmlFileName)
return res
def getValue(ftype, flength, targetName, targetValue, val):
retVal = val # init to the value calculated so far. This function will alter as needed for field type
try:
if retVal == 'None':
retVal = None
if retVal != targetValue:
if ftype == 'Integer' or ftype == 'Double':
# if the type is numeric then try to cast to float
if str(val) == 'None' or str(val) == dla._noneFieldName:
retVal = None
else:
try:
valTest = float(val)
retVal = val
except:
err = "Exception caught: unable to cast " + targetName + " to " + ftype + " : '" + str(
val) + "'"
dla.addError(err)
dla._errCount += 1
elif ftype == 'String':
# if a string then cast to string or encode utf-8
if type(val) == 'str':
retVal = val.encode('utf-8', errors='replace').decode(
'utf-8', errors='backslashreplace') # handle unicode
else:
retVal = str(val)
# check length to make sure it is not too long.
if len(retVal) > int(flength):
err = "Exception caught: value length > field length for " + targetName + "(Length " + str(
flength) + ") : '" + str(retVal) + "'"
dla.addError(err)
dla._errCount += 1
else:
retVal = val
except:
err = "Exception caught: unable to get value for value=" + str(
val) + " fieldname=" + targetName
dla.showTraceback()
dla.addError(err)
dla._errCount += 1
return retVal
def setValue(row, names, types, lengths, targetName, targetValue, val):
try:
if val == 'None':
val = None
if val != targetValue:
idx = names.index(targetName)
if types[idx] == 'Integer' or types[idx] == 'Double':
# if the type is numeric then try to cast to float
try:
valTest = float(val)
row[idx] = val
except:
err = "Exception caught: unable to cast " + targetName + " to " + types[
idx] + " : '" + str(val) + "'"
dla.addError(err)
print(err)
dla._errCount += 1
elif types[idx] == 'String':
# if a string then cast to string and check length
if type(val) != 'str':
val = str(val)
if len(val) > int(lengths[idx]):
err = "Exception caught: value length > field length for " + targetName + "(Length " + str(
lengths[idx]) + ") : '" + str(val) + "'"
dla.addError(err)
print(err)
dla._errCount += 1
else:
row[idx] = val
else:
row[idx] = val
except:
success = False
err = "Exception caught: unable to set value for value=" + str(
val) + " fieldname=" + targetName
dla.showTraceback()
dla.addError(err)
print(err)
dla._errCount += 1
def getValue(names,ftypes,lengths,targetName,targetValue,val):
retVal = val # init to the value calculated so far. This function will alter as needed for field type
try:
idx = names.index(targetName)
if str(retVal) == 'None':
retVal = None
if str(retVal) != str(targetValue):
if ftypes[idx] == 'Integer' or ftypes[idx] == 'Double' or ftypes[idx] == 'Float':
# if the type is numeric then try to cast to float
try:
valTest = float(val)
retVal = val
except:
err = "Exception caught: unable to cast " + targetName + " to " + ftypes[idx] + " : '" + str(val) + "'"
dla.addError(err)
dla._errCount += 1
elif ftypes[idx] == 'String':
# if a string then cast to string or encode utf-8
if type(val) == 'str':
retVal = val.encode('utf-8', errors='replace').decode('utf-8',errors='backslashreplace') # handle unicode
else:
retVal = str(val)
# check length to make sure it is not too long.
if len(retVal) > int(lengths[idx]):
err = "Exception caught: value length > field length for " + targetName + "(Length " + str(lengths[idx]) + ") : '" + str(retVal) + "'"
dla.addError(err)
dla._errCount += 1
else:
retVal = val
except:
err = "Exception caught: unable to get value for value=" + str(val) + " fieldname=" + targetName
dla.showTraceback()
dla.addError(err)
dla._errCount += 1
return retVal
def setValue(row,names,types,lengths,targetName,targetValue,val):
try:
if val == 'None':
val = None
if val != targetValue:
idx = names.index(targetName)
if types[idx] == 'Integer' or types[idx] == 'Double':
# if the type is numeric then try to cast to float
try:
valTest = float(val)
row[idx] = val
except:
err = "Exception caught: unable to cast " + targetName + " to " + types[idx] + " : '" + str(val) + "'"
dla.addError(err)
print(err)
dla._errCount += 1
elif types[idx] == 'String':
# if a string then cast to string and check length
if type(val) != 'str':
val = str(val)
if len(val) > int(lengths[idx]):
err = "Exception caught: value length > field length for " + targetName + "(Length " + str(lengths[idx]) + ") : '" + str(val) + "'"
dla.addError(err)
print(err)
dla._errCount += 1
else:
row[idx] = val
else:
row[idx] = val
except:
success = False
err = "Exception caught: unable to set value for value=" + str(val) + " fieldname=" + targetName
dla.showTraceback()
dla.addError(err)
print(err)
dla._errCount += 1
def addFeatures(sourceLayer, targelUrl, expr):
# add features using _chunkSize
retval = False
error = False
# add section
try:
arcpy.SetProgressor("default", "Adding Features")
arcpy.SetProgressorLabel("Adding Features")
featurejs = featureclass_to_json(sourceLayer)
url = targelUrl + '/addFeatures'
numFeat = len(featurejs['features'])
if numFeat == 0:
dla.addMessage("0 Features to Add, exiting")
return True # nothing to add is OK
if numFeat > _chunkSize:
chunk = _chunkSize
else:
chunk = numFeat
featuresProcessed = 0
while featuresProcessed < numFeat and error == False:
next = featuresProcessed + chunk
features = featurejs['features'][featuresProcessed:next]
msg = "Adding features " + str(featuresProcessed) + ":" + str(next)
dla.addMessage(msg)
arcpy.SetProgressorLabel(msg)
token = dla.getSigninToken()
params = {
'rollbackonfailure': 'true',
'f': 'json',
'token': token,
'features': json.dumps(features)
}
result = dla.sendRequest(url, params)
try:
if result['error'] != None:
retval = False
dla.addMessage("Add features to Feature Service failed")
dla.addMessage(json.dumps(result))
error = True
except:
try:
lenAdded = len(result['addResults'])
msg = str(lenAdded) + " features added, " + str(
featuresProcessed + chunk) + "/" + str(numFeat)
print(msg)
dla.addMessage(msg)
retval = True
except:
retval = False
dla.addMessage("Add features to Feature Service failed")
dla.showTraceback()
dla.addError(json.dumps(result))
error = True
featuresProcessed += chunk
except:
retval = False
dla.addMessage("Add features to Feature Service failed")
dla.showTraceback()
error = True
pass
return retval
def deleteFeatures(sourceLayer, targelUrl, expr):
# delete features using chunks of _chunkSize
retval = False
error = False
# delete section
ids = getOIDs(targelUrl, expr)
try:
lenDeleted = 100
#Chunk deletes using chunk size at a time
featuresProcessed = 0
numFeat = len(ids)
if numFeat == 0:
dla.addMessage("0 Features to Delete, exiting")
return True # nothing to delete is OK
if numFeat > _chunkSize:
chunk = _chunkSize
else:
chunk = numFeat
arcpy.SetProgressor("default", "Deleting Features")
while featuresProcessed < numFeat and error == False:
#Chunk deletes using chunk size at a time
next = featuresProcessed + chunk
msg = "Deleting features " + str(featuresProcessed) + ":" + str(
next)
dla.addMessage(msg)
arcpy.SetProgressorLabel(msg)
oids = ",".join(str(e) for e in ids[featuresProcessed:next])
url = targelUrl + '/deleteFeatures'
token = dla.getSigninToken()
params = {'f': 'pjson', 'objectIds': oids, 'token': token}
result = dla.sendRequest(url, params)
try:
if result['error'] != None:
retval = False
dla.addMessage(
"Delete features from Feature Service failed")
dla.addMessage(json.dumps(result))
error = True
except:
try:
lenDeleted = len(result['deleteResults'])
msg = str(lenDeleted) + " features deleted, " + str(
featuresProcessed + chunk) + "/" + str(numFeat)
print(msg)
dla.addMessage(msg)
retval = True
except:
retval = False
error = True
dla.showTraceback()
dla.addMessage(
"Delete features from Feature Service failed")
dla.addError(json.dumps(result))
featuresProcessed += chunk
except:
retval = False
error = True
dla.showTraceback()
dla.addMessage("Delete features from Feature Service failed")
pass
return retval
def calculate(xmlFileName,workspace,name,ignore):
dla.workspace = workspace
success = True
arcpy.ClearWorkspaceCache_management(dla.workspace)
xmlDoc = dla.getXmlDoc(xmlFileName)
dla.addMessage("Field Calculator: " + xmlFileName)
arcpy.env.Workspace = dla.workspace
table = dla.getTempTable(name)
if not arcpy.Exists(table):
dla.addError("Feature Class " + table + " does not exist, exiting")
arcpy.SetParameter(SUCCESS, False)
return
if not arcpy.TestSchemaLock(table):
dla.addError("Unable to obtain a schema lock for " + table + ", exiting")
arcpy.SetParameter(SUCCESS, False)
return -1
desc = arcpy.Describe(table)
fields = dla.getXmlElements(xmlFileName,"Field")
sourceFields = dla.getXmlElements(xmlFileName,"SourceField")
targetFields = dla.getXmlElements(xmlFileName,"TargetField")
attrs = [f.name for f in arcpy.ListFields(table)]
for field in fields:
arcpy.env.Workspace = dla.workspace
targetName = dla.getNodeValue(field,"TargetName")
sourceName = dla.getNodeValue(field,"SourceName")
ftype = "String"
length = "50"
for target in targetFields:
nm = target.getAttributeNode("Name").nodeValue
if nm == targetName:
ftype = target.getAttributeNode("Type").nodeValue
length = target.getAttributeNode("Length").nodeValue
# uppercase compare, later need to check for orig/upper name for calc
#ups = [nm.upper() for nm in attrs]
dla.addDlaField(table,targetName,field,attrs,ftype,length)
allFields = sourceFields + targetFields
desc = arcpy.Describe(table)
layerNames = []
names = []
ftypes = []
lengths = []
ignore = ['FID','OBJECTID','GLOBALID','SHAPE','SHAPE_AREA','SHAPE_LENGTH','SHAPE_LEN','STLENGTH()','STAREA()','raster']
for name in ['OIDFieldName','ShapeFieldName','LengthFieldName','AreaFieldName','GlobalIDFieldName','RasterFieldName']:
try:
val = eval("desc." + name)
val = val[val.rfind('.')+1:]
ignore.append(val).upper()
except:
pass
for field in desc.fields:
if field.name.upper() not in ignore:
layerNames.append(field.name.upper())
for field in allFields:
nm = field.getAttributeNode("Name").nodeValue
if nm != dla.noneName and nm.upper() not in ignore and nm.upper() in layerNames:
try:
names.index(nm)
except:
names.append(nm)
typ = field.getAttributeNode("Type").nodeValue
leng = field.getAttributeNode("Length").nodeValue
ftypes.append(typ)
lengths.append(leng)
retVal = setFieldValues(table,fields,names,ftypes,lengths)
if retVal == False:
success = False
arcpy.ClearWorkspaceCache_management(dla.workspace)
dla.cleanupGarbage()
arcpy.ResetProgressor()
if ignore == True:
success = True
return success
for name in names:
expression = expression.replace(name, "|" + name + "|")
val = getExpression(row, names, expression)
# set field value
if method != "None":
setValue(row, names, types, lengths, targetName,
targetValue, val)
try:
updateCursor.updateRow(row)
except:
dla._errCount += 1
success = False
err = "Exception caught: unable to update row"
printRow(row, names)
dla.showTraceback()
dla.addError(err)
except:
dla._errCount += 1
success = False
err = "Exception caught: unable to update dataset"
if row != None:
printRow(row, names)
dla.showTraceback()
dla.addError(err)
finally:
del updateCursor
dla.cleanupGarbage()
arcpy.ResetProgressor()
return success
def exportDataset(xmlDoc, source, workspace, targetName, rowLimit,
datasetType):
result = True
xmlFields = xmlDoc.getElementsByTagName("Field")
dla.addMessage("Exporting Data from " + source)
whereClause = ""
if rowLimit != None:
whereClause = getObjectIdWhereClause(source, rowLimit)
if whereClause != '' and whereClause != ' ':
dla.addMessage("Where " + str(whereClause))
sourceName = dla.getDatasetName(source)
viewName = sourceName + "_View"
dla.addMessage(viewName)
targetRef = getSpatialReference(xmlDoc, "Target")
sourceRef = getSpatialReference(xmlDoc, "Source")
if datasetType == 'Table':
isTable = True
elif targetRef != '':
isTable = False
arcpy.env.workspace = workspace
if source.lower().endswith('.lyrx') and not dla.hasJoin(source):
view = dla.getLayerFromString(source)
elif isTable:
view = dla.makeTableView(dla.workspace, source, viewName, whereClause,
xmlFields)
elif not isTable:
view = dla.makeFeatureView(dla.workspace, source, viewName,
whereClause, xmlFields)
dla.addMessage("View Created")
srcCount = arcpy.GetCount_management(view).getOutput(0)
dla.addMessage(str(srcCount) + " source rows")
if str(srcCount) == '0':
result = False
dla.addError("Failed to extract " + sourceName + ", Nothing to export")
else:
arcpy.env.overwriteOutput = True
ds = workspace + os.sep + targetName
currentPreserveGlobalIDs = arcpy.env.preserveGlobalIds
if dla.processGlobalIds(
xmlDoc
): # both datasets have globalids in the correct workspace types
arcpy.env.preserveGlobalIds = True # try to preserve
dla.addMessage("Attempting to preserve GlobalIDs")
else:
arcpy.env.preserveGlobalIds = False # don't try to preserve
dla.addMessage("Unable to preserve GlobalIDs")
if isTable:
arcpy.TableToTable_conversion(in_rows=view,
out_path=workspace,
out_name=targetName)
else:
spRefMatch = dla.compareSpatialRef(xmlDoc)
currentRef = arcpy.env.outputCoordinateSystem # grab currrent env settings
currentTrans = arcpy.env.geographicTransformations
if not spRefMatch:
arcpy.env.outputCoordinateSystem = targetRef
transformations = arcpy.ListTransformations(
sourceRef, targetRef)
transformations = ";".join(
transformations
) # concat the values - format change for setting the values.
arcpy.env.geographicTransformations = transformations
arcpy.FeatureClassToFeatureClass_conversion(in_features=view,
out_path=workspace,
out_name=targetName)
if not spRefMatch: # set the spatial reference back
arcpy.env.outputCoordinateSystem = currentRef
arcpy.env.geographicTransformations = currentTrans
arcpy.env.preserveGlobalIds = currentPreserveGlobalIDs
removeDefaultValues(
ds
) # don't want to turn nulls into defaultValues in the intermediate data
# not needed if doing the transformations approach above...
# if isTable:
# if not createDataset('Table',workspace,targetName,None,xmlDoc,source,None):
# arcpy.AddError("Unable to create intermediate table, exiting: " + workspace + os.sep + targetName)
# return False
# elif not isTable:
# geomType = arcpy.Describe(source).shapeType
# if not createDataset('FeatureClass',workspace,targetName,geomType,xmlDoc,source,targetRef):
# arcpy.AddError("Unable to create intermediate feature class, exiting: " + workspace + os.sep + targetName)
# return False
# fieldMap = getFieldMap(view,ds)
# arcpy.Append_management(view,ds,schema_type="NO_TEST",field_mapping=fieldMap)
dla.addMessage(arcpy.GetMessages(2)) # only serious errors
count = arcpy.GetCount_management(ds).getOutput(0)
dla.addMessage(str(count) + " source rows exported to " + targetName)
if str(count) == '0':
result = False
dla.addError(
"Failed to load to " + targetName +
", it is likely that your data falls outside of the target Spatial Reference Extent or there is another basic issue"
)
dla.addError(
"To verify please use the Append and/or Copy Features tool to load some data to an intermediate dataset:"
)
dla.addError(ds)
dla.showTraceback()
return result
def setFieldValues(table, fields, names, types, lengths):
# from source xml file match old values to new values to prepare for append to target geodatabase
success = False
row = None
try:
updateCursor = arcpy.da.UpdateCursor(table, names)
result = arcpy.GetCount_management(table)
numFeat = int(result.getOutput(0))
dla.addMessage(table + ", " + str(numFeat) + " features")
i = 0
arcpy.SetProgressor("Step", "Calculating " + table + "...", 0, numFeat,
getProgressUpdate(numFeat))
for row in updateCursor:
success = True
if dla._errCount > dla.maxErrorCount:
#dla.addMessage("Exceeded max number of errors in dla.maxErrorCount: " + str(dla.maxErrorCount))
dla.addError(
"Exceeded max number of errors in dla.maxErrorCount: " +
str(dla.maxErrorCount))
return False
if i > dla.maxrows:
#dla.addMessage("Exceeded max number of rows supported in dla.maxrows: " + str(dla.maxrows))
dla.addError(
"Exceeded max number of rows supported in dla.maxrows: " +
str(dla.maxrows))
return True
i = i + 1
setProgressor(i, numFeat)
for field in fields:
method = "None"
sourceName = dla.getNodeValue(field, "SourceName")
targetName = dla.getNodeValue(field, "TargetName")
targetValue = getTargetValue(row, field, names, sourceName,
targetName)
sourceValue = getSourceValue(row, names, sourceName,
targetName)
method = dla.getNodeValue(field, "Method").replace(" ", "")
if method == "None" or (method == "Copy"
and sourceName == '(None)'):
method = "None"
val = None
elif method == "Copy":
val = sourceValue
elif method == "DefaultValue":
val = dla.getNodeValue(field, "DefaultValue")
elif method == "SetValue":
val = dla.getNodeValue(field, "SetValue")
elif method == "ValueMap":
val = getValueMap(row, names, sourceValue, field)
elif method == "ChangeCase":
case = dla.getNodeValue(field, method)
expression = getChangeCase(sourceValue, case)
val = getExpression(row, names, expression)
elif method == "Concatenate":
val = getConcatenate(row, names, field)
elif method == "Left":
chars = dla.getNodeValue(field, "Left")
val = getSubstring(sourceValue, "0", chars)
elif method == "Right":
chars = dla.getNodeValue(field, "Right")
val = getSubstring(sourceValue,
len(str(sourceValue)) - int(chars),
len(str(sourceValue)))
def publish(xmlFileNames):
# function called from main or from another script, performs the data update processing
global sourceLayer,targetLayer,_success
dla._errorCount = 0
arcpy.SetProgressor("default","Data Assistant")
arcpy.SetProgressorLabel("Data Assistant")
xmlFiles = xmlFileNames.split(";")
for xmlFile in xmlFiles: # multi value parameter, loop for each file
dla.addMessage("Configuration file: " + xmlFile)
xmlDoc = dla.getXmlDoc(xmlFile) # parse the xml document
if xmlDoc == None:
return
svceS = False
svceT = False
if sourceLayer == "" or sourceLayer == None:
sourceLayer = dla.getNodeValue(xmlDoc,"Source")
svceS = dla.checkLayerIsService(sourceLayer)
if targetLayer == "" or targetLayer == None:
targetLayer = dla.getNodeValue(xmlDoc,"Target")
svceT = dla.checkLayerIsService(targetLayer)
dla.addMessage(targetLayer)
## Added May2016. warn user if capabilities are not correct, exit if not a valid layer
if not dla.checkServiceCapabilities(sourceLayer,True):
return False
if not dla.checkServiceCapabilities(targetLayer,True):
return False
if svceS == True or svceT == True:
token = dla.getSigninToken() # when signed in get the token and use this. Will be requested many times during the publish
if token == None:
dla.addError("User must be signed in for this tool to work with services")
return
expr = getWhereClause(xmlDoc)
if useReplaceSettings == True and (expr == '' or expr == None):
dla.addError("There must be an expression for replacing by field value, current value = " + str(expr))
return False
dla.setWorkspace()
targetName = dla.getTargetName(xmlDoc)
res = dlaExtractLayerToGDB.extract(xmlFile,None,dla.workspace,sourceLayer,targetName)
if res != True:
table = dla.getTempTable(targetName)
msg = "Unable to export data, there is a lock on existing datasets or another unknown error"
if arcpy.TestSchemaLock(table) != True:
msg = "Unable to export data, there is a lock on the intermediate feature class: " + table
dla.addError(msg)
print(msg)
return
else:
res = dlaFieldCalculator.calculate(xmlFile,dla.workspace,targetName,False)
if res == True:
dlaTable = dla.getTempTable(targetName)
res = doPublish(xmlDoc,dlaTable,targetLayer)
arcpy.ResetProgressor()
sourceLayer = None # set source and target back to None for multiple file processing
targetLayer = None
if res == False:
err = "Data Assistant Update Failed, see messages for details"
dla.addError(err)
print(err)
def calculate(xmlFileName, workspace, name, ignore):
dla.workspace = workspace
success = True
arcpy.ClearWorkspaceCache_management(dla.workspace)
xmlDoc = dla.getXmlDoc(xmlFileName)
arcpy.env.Workspace = dla.workspace
table = dla.getTempTable(name)
if not arcpy.Exists(table):
dla.addError("Feature Class " + table + " does not exist, exiting")
arcpy.SetParameter(SUCCESS, False)
return
if not arcpy.TestSchemaLock(table):
dla.addError("Unable to obtain a schema lock for " + table +
", exiting")
arcpy.SetParameter(SUCCESS, False)
return -1
desc = arcpy.Describe(table)
fields = dla.getXmlElements(xmlFileName, "Field")
sourceFields = dla.getXmlElements(xmlFileName, "SourceField")
targetFields = dla.getXmlElements(xmlFileName, "TargetField")
attrs = [f.name for f in arcpy.ListFields(table)]
for field in fields:
arcpy.env.Workspace = dla.workspace
targetName = dla.getNodeValue(field, "TargetName")
sourceName = dla.getNodeValue(field, "SourceName")
type = "String"
length = "50"
for target in targetFields:
nm = target.getAttributeNode("Name").nodeValue
if nm == targetName:
type = target.getAttributeNode("Type").nodeValue
length = target.getAttributeNode("Length").nodeValue
# uppercase compare, later need to check for orig/upper name for calc
#ups = [nm.upper() for nm in attrs]
dla.addDlaField(table, targetName, field, attrs, type, length)
allFields = sourceFields + targetFields
names = []
types = []
lengths = []
for field in allFields:
nm = field.getAttributeNode("Name").nodeValue
if nm != dla.noneName:
names.append(nm)
typ = field.getAttributeNode("Type").nodeValue
leng = field.getAttributeNode("Length").nodeValue
types.append(typ)
lengths.append(leng)
retVal = setFieldValues(table, fields, names, types, lengths)
if retVal == False:
success = False
arcpy.ClearWorkspaceCache_management(dla.workspace)
dla.cleanupGarbage()
arcpy.ResetProgressor()
if ignore == True:
success = True
return success
def calculate(xmlFileName,workspace,name,ignore):
dla.workspace = workspace
success = True
arcpy.ClearWorkspaceCache_management(dla.workspace)
xmlDoc = dla.getXmlDoc(xmlFileName)
arcpy.env.Workspace = dla.workspace
table = dla.getTempTable(name)
if not arcpy.Exists(table):
dla.addError("Feature Class " + table + " does not exist, exiting")
arcpy.SetParameter(SUCCESS, False)
return
if not arcpy.TestSchemaLock(table):
dla.addError("Unable to obtain a schema lock for " + table + ", exiting")
arcpy.SetParameter(SUCCESS, False)
return -1
desc = arcpy.Describe(table)
fields = dla.getXmlElements(xmlFileName,"Field")
sourceFields = dla.getXmlElements(xmlFileName,"SourceField")
targetFields = dla.getXmlElements(xmlFileName,"TargetField")
attrs = [f.name for f in arcpy.ListFields(table)]
for field in fields:
arcpy.env.Workspace = dla.workspace
targetName = dla.getNodeValue(field,"TargetName")
sourceName = dla.getNodeValue(field,"SourceName")
type = "String"
length = "50"
for target in targetFields:
nm = target.getAttributeNode("Name").nodeValue
if nm == targetName:
type = target.getAttributeNode("Type").nodeValue
length = target.getAttributeNode("Length").nodeValue
# uppercase compare, later need to check for orig/upper name for calc
#ups = [nm.upper() for nm in attrs]
dla.addDlaField(table,targetName,field,attrs,type,length)
allFields = sourceFields + targetFields
names = []
types = []
lengths = []
for field in allFields:
nm = field.getAttributeNode("Name").nodeValue
if nm != dla.noneName:
names.append(nm)
typ = field.getAttributeNode("Type").nodeValue
leng = field.getAttributeNode("Length").nodeValue
types.append(typ)
lengths.append(leng)
retVal = setFieldValues(table,fields,names,types,lengths)
if retVal == False:
success = False
arcpy.ClearWorkspaceCache_management(dla.workspace)
dla.cleanupGarbage()
arcpy.ResetProgressor()
if ignore == True:
success = True
return success
def setFieldValues(table,fields,names,ftypes,lengths):
# from source xml file match old values to new values to prepare for append to target geodatabase
success = False
row = None
try:
updateCursor = arcpy.da.UpdateCursor(table,names)
result = arcpy.GetCount_management(table)
numFeat = int(result.getOutput(0))
dla.addMessage(table + ", " + str(numFeat) + " features")
i = 0
arcpy.SetProgressor("Step","Calculating " + table + "...",0,numFeat,getProgressUpdate(numFeat))
for row in updateCursor:
success = True
if dla._errCount > dla.maxErrorCount:
#dla.addMessage("Exceeded max number of errors in dla.maxErrorCount: " + str(dla.maxErrorCount))
dla.addError("Exceeded max number of errors in dla.maxErrorCount: " + str(dla.maxErrorCount))
return False
if i > dla.maxrows:
#dla.addMessage("Exceeded max number of rows supported in dla.maxrows: " + str(dla.maxrows))
dla.addError("Exceeded max number of rows supported in dla.maxrows: " + str(dla.maxrows))
return True
i = i + 1
setProgressor(i,numFeat)
for field in fields:
method = "None"
sourceName = dla.getNodeValue(field,"SourceName")
targetName = dla.getNodeValue(field,"TargetName")
targetValue = getTargetValue(row,field,names,sourceName,targetName)
sourceValue = getSourceValue(row,names,sourceName,targetName)
method = dla.getNodeValue(field,"Method").replace(" ","")
fnum = names.index(targetName)
if method == "None" or (method == "Copy" and sourceName == dla._noneFieldName):
val = None
method = "None"
elif method == "Copy":
val = sourceValue
elif method == "DefaultValue":
val = dla.getNodeValue(field,"DefaultValue")
elif method == "SetValue":
val = dla.getNodeValue(field,"SetValue")
elif method == "ValueMap":
val = getValueMap(row,names,sourceValue,field)
elif method == "ChangeCase":
case = dla.getNodeValue(field,method)
expression = getChangeCase(sourceValue,case)
val = getExpression(row,names,expression)
elif method == "Concatenate":
val = getConcatenate(row,names,field)
elif method == "Left":
chars = dla.getNodeValue(field,"Left")
val = getSubstring(sourceValue,"0",chars)
elif method == "Right":
chars = dla.getNodeValue(field,"Right")
val = getSubstring(sourceValue,len(str(sourceValue))-int(chars),len(str(sourceValue)))
elif method == "Substring":
start = int(dla.getNodeValue(field,"Start"))
lngth = int(dla.getNodeValue(field,"Length"))
if sourceValue != None:
lngth = start + lngth
val = getSubstring(sourceValue,start,lngth)
elif method == "Split":
splitter = dla.getNodeValue(field,"SplitAt")
splitter = splitter.replace("(space)"," ")
part = dla.getNodeValue(field,"Part")
val = getSplit(sourceValue,splitter,part)
elif method == "ConditionalValue":
sname = dla.getNodeValue(field,"SourceName")
oper = dla.getNodeValue(field,"Oper")
iif = dla.getNodeValue(field,"If")
if iif != " " and type(iif) == 'str':
for name in names:
if name in iif:
iif = iif.replace(name,"|"+name+"|")
tthen = dla.getNodeValue(field,"Then")
eelse = dla.getNodeValue(field,"Else")
for name in names:
if name in eelse:
eelse = eelse.replace(name,"|"+name+"|")
expression = "|" + tthen + "| " + " if |" + sname + "| " + oper + " |" + iif + "| else " + eelse
val = getExpression(row,names,expression)
elif method == "Expression":
expression = dla.getNodeValue(field,method)
for name in names:
expression = expression.replace(name,"|" + name + "|")
val = getExpression(row,names,expression)
# set field value
if method != "None" and val != None:
newVal = getValue(names,ftypes,lengths,targetName,targetValue,val)
row[fnum] = newVal
else:
row[fnum] = val
try:
updateCursor.updateRow(row)
except:
dla._errCount += 1
success = False
err = "Exception caught: unable to update row"
printRow(row,names)
dla.showTraceback()
dla.addError(err)
except:
dla._errCount += 1
success = False
err = "Exception caught: unable to update dataset"
if row != None:
printRow(row,names)
dla.showTraceback()
dla.addError(err)
finally:
del updateCursor
dla.cleanupGarbage()
arcpy.ResetProgressor()
return success
def publish(xmlFileNames):
# function called from main or from another script, performs the data update processing
global sourceLayer, targetLayer, _success
dla._errorCount = 0
arcpy.SetProgressor("default", "Data Assistant")
arcpy.SetProgressorLabel("Data Assistant")
xmlFiles = xmlFileNames.split(";")
for xmlFile in xmlFiles: # multi value parameter, loop for each file
dla.addMessage("Configuration file: " + xmlFile)
xmlDoc = dla.getXmlDoc(xmlFile) # parse the xml document
if xmlDoc == None:
return
svceS = False
svceT = False
if sourceLayer == "" or sourceLayer == None:
sourceLayer = dla.getNodeValue(xmlDoc, "Source")
svceS = dla.checkLayerIsService(sourceLayer)
if targetLayer == "" or targetLayer == None:
targetLayer = dla.getNodeValue(xmlDoc, "Target")
svceT = dla.checkLayerIsService(targetLayer)
dla.addMessage(targetLayer)
## Added May2016. warn user if capabilities are not correct, exit if not a valid layer
if not dla.checkServiceCapabilities(sourceLayer, True):
return False
if not dla.checkServiceCapabilities(targetLayer, True):
return False
if svceS == True or svceT == True:
token = dla.getSigninToken(
) # when signed in get the token and use this. Will be requested many times during the publish
if token == None:
dla.addError(
"User must be signed in for this tool to work with services"
)
return
expr = getWhereClause(xmlDoc)
if useReplaceSettings == True and (expr == '' or expr == None):
dla.addError(
"There must be an expression for replacing by field value, current value = "
+ str(expr))
return False
dla.setWorkspace()
targetName = dla.getTargetName(xmlDoc)
res = dlaExtractLayerToGDB.extract(xmlFile, None, dla.workspace,
sourceLayer, targetName)
if res != True:
table = dla.getTempTable(targetName)
msg = "Unable to export data, there is a lock on existing datasets or another unknown error"
if arcpy.TestSchemaLock(table) != True:
msg = "Unable to export data, there is a lock on the intermediate feature class: " + table
dla.addError(msg)
print(msg)
return
else:
res = dlaFieldCalculator.calculate(xmlFile, dla.workspace,
targetName, False)
if res == True:
dlaTable = dla.getTempTable(targetName)
res = doPublish(xmlDoc, dlaTable, targetLayer)
arcpy.ResetProgressor()
sourceLayer = None # set source and target back to None for multiple file processing
targetLayer = None
if res == False:
err = "Data Assistant Update Failed, see messages for details"
dla.addError(err)
print(err)
def writeDocument(sourcePath,targetPath,xmlFileName):
if sourcePath == None or targetPath == None:
return False
## Warn user if capabilities are not correct, exit if not valid layers
errs = False
if dlaService.validateSourceUrl(sourcePath) == False:
dla.addError("Errors in Source Service Capabilities, exiting without writing the output file")
errs = True
if dlaService.validateTargetUrl(targetPath) == False:
dla.addError("Errors in Target Service Capabilities, exiting without writing the output file")
errs = True
try:
desc = arcpy.Describe(sourcePath)
except:
dla.addError("Unable to Describe the source dataset, exiting")
errs = True
try:
descT = arcpy.Describe(targetPath)
except:
dla.addError("Unable to Describe the target dataset, exiting")
errs = True
if errs:
return False
xmlDoc = Document()
root = xmlDoc.createElement('SourceTargetMatrix')
xmlDoc.appendChild(root)
root.setAttribute("version",'1.1')
root.setAttribute("xmlns:esri",'http://www.esri.com')
dataset = xmlDoc.createElement("Datasets")
root.appendChild(dataset)
prj = dla.getProject()
if prj == None:
prj = ''
else:
prj = prj.filePath
setSourceTarget(dataset,xmlDoc,"Project",dla.dropXmlFolder(xmlFileName,prj))
setSourceTarget(dataset,xmlDoc,"Source",dla.dropXmlFolder(xmlFileName,sourcePath))
setSourceTarget(dataset,xmlDoc,"Target",dla.dropXmlFolder(xmlFileName,targetPath))
setSpatialReference(dataset,xmlDoc,desc,"Source")
setSpatialReference(dataset,xmlDoc,descT,"Target")
setSourceTarget(dataset,xmlDoc,"ReplaceBy","")
fieldroot = xmlDoc.createElement("Fields")
root.appendChild(fieldroot)
fields = getFields(descT)
sourceFields = getFields(desc)
#sourceNames = [field.name[field.name.rfind(".")+1:] for field in sourceFields] ***
sourceNames = [field.name for field in sourceFields]
upperNames = [nm.upper() for nm in sourceNames]
#try:
for field in fields:
fNode = xmlDoc.createElement("Field")
fieldroot.appendChild(fNode)
fieldName = field.name #[field.name.rfind(".")+1:] ***
matchSourceFields(xmlDoc,fNode,field,fieldName,sourceNames,upperNames)
# write the source field values
setSourceFields(root,xmlDoc,sourceFields)
setTargetFields(root,xmlDoc,fields)
# add data to the document
if len(sourceNames) > 0:
writeDataSample(xmlDoc,root,sourceNames,sourcePath,10)
# write it out
xmlDoc.writexml( open(xmlFileName, 'wt', encoding='utf-8'),indent=" ",addindent=" ",newl='\n')
xmlDoc.unlink()
return True
