def moveLocalErrorsToSQL(prefixKeyName):
print("Moving errors from the Local gdb to SQL for the prefix key name of: " + str(prefixKeyName) + ".")
for errorItemFC in prefixesToMoveErrorsFor:
errorItemFCBaseName = returnFeatureClass(errorItemFC)
sqlPath = returnGDBOrSDEPath(dailyProcessSDESourceCenterlinesLocation)
errorItemCopyName = prefixKeyName + '_' + errorItemFCBaseName
errorItemSQLLocation = os.path.join(sqlPath, errorItemCopyName)
lowerstrErrorItemFCBaseName = str(errorItemFCBaseName).lower()
if lowerstrErrorItemFCBaseName.find('selfintclassification') >= 0:
# Already includes the prefix name in the basename, so just use the full BaseName for the class.
errorItemSQLLocation = os.path.join(sqlPath, errorItemFCBaseName)
else:
pass
if Exists(errorItemSQLLocation):
try:
Delete_management(errorItemSQLLocation)
except:
print("Could not delete the FC at: " + str(errorItemSQLLocation) + ".")
print("Please make sure that the FC does not have any locks on it and try again.")
try:
CopyFeatures_management(errorItemFC, errorItemSQLLocation)
except:
print("Could not copy from the FC at: " + str(errorItemFC))
print("to the FC at: " + str(errorItemSQLLocation) + ".")
print("Please make sure that the FC to copy from exists")
print("and that the FC to copy to is not locked.")
def routesSourceCreation():
env.workspace = returnGDBOrSDEPath(routesSourceCenterlines)
env.overwriteOutput = 1
# Checking to see if the output already exists.
# If so, remove it.
if Exists(routesSourceCenterlines):
Delete_management(routesSourceCenterlines)
else:
pass
# Create a new file for the output.
print("Making a copy of " + returnFeatureClass(inputCenterlines) + " called " + returnFeatureClass(routesSourceCenterlines) + ".")
CopyFeatures_management(inputCenterlines, routesSourceCenterlines)
print("Adding fields to " + returnFeatureClass(routesSourceCenterlines) + ".")
#Addfields:
# SourceRouteId (Text, 50)
AddField_management(routesSourceCenterlines, "SourceRouteId", "TEXT", "", "", 50, "SourceRouteId", nullable)
# SourceFromMeasure (Double)
AddField_management(routesSourceCenterlines, "SourceFromMeasure", "DOUBLE", "", "", "", "SourceFromMeasure", nullable)
# SourceToMeasure (Double)
AddField_management(routesSourceCenterlines, "SourceToMeasure", "DOUBLE", "", "", "", "SourceToMeasure", nullable)
if useNewFieldLogic == True:
KDOTKeyCalculation_NewFieldLogic()
else:
TranscendFieldCalculation()
TranscendRampReplacement()
if useNewFieldLogic == True:
KDOTKeyCalculation_NewFieldLogic()
else:
TranscendFieldCalculation()
LocalRouteReduction()
def routeCreation():
env.workspace = returnGDBOrSDEPath(createRoutesOutputFC)
env.overwriteOutput = 1
'''
# Need to match what Transcend used. -- Done.
routeId = 'SourceRouteId'
measureSource = 'TWO_FIELDS'
fMeasureField = 'SourceFromMeasure'
tMeasureField = 'SourceToMeasure'
coordinatePriority = 'UPPER_LEFT'
measureFactor = 1
measureOffset = 0
ignoreGaps = True
buildIndex = True
'''
# Modifications for County Target Network.
routeId = 'TargetCountyLRSKey'
measureSource = 'Shapelength'
#fMeasureField = 'SourceFromMeasure'
#tMeasureField = 'SourceToMeasure'
coordinatePriority = 'LOWER_LEFT'
measureFactor = 1
measureOffset = 0
ignoreGaps = True
buildIndex = True
routesOutputGDB = returnGDBOrSDEPath(createRoutesOutputFC)
routesOutputGDBName = returnGDBOrSDEName(routesOutputGDB)
# Need to implement a new path function to get the GDB's folder.
routesOutputGDBFolder = mainFolder
if Exists(routesOutputGDB):
Delete_management(routesOutputGDB)
else:
pass
CreateFileGDB_management(routesOutputGDBFolder, routesOutputGDBName)
# Checking to see if the copy for routes output exists.
# If so, remove it.
#if Exists(createRoutesOutputFC):
# Delete_management(createRoutesOutputFC)
#else:
# pass
print("Creating the lrs routes.")
# CreateRoutes_lr GP Tool
CreateRoutes_lr(createRoutesInputFC, routeId, createRoutesOutputFC,
measureSource, fMeasureField, tMeasureField,
coordinatePriority, measureFactor, measureOffset,
ignoreGaps, buildIndex)
print("Adding date fields to " + returnFeatureClass(createRoutesOutputFC) +
".")
#Addfields:
AddField_management(createRoutesOutputFC, "F_Date", "DATE", "", "", "",
"F_Date", nullable)
pyDateExpression = '''def pyFindTheDate():
import time
return time.strftime("%Y/%m/%d")'''
CalculateField_management(createRoutesOutputFC, "F_Date",
"pyFindTheDate()", "PYTHON_9.3",
pyDateExpression)
# T_Date (Date)
AddField_management(createRoutesOutputFC, "T_Date", "DATE", "", "", "",
"T_Date", nullable)
# ---- Add route calibration point creation steps for Start & End points. ----
MakeFeatureLayer_management(createRoutesInputFC, 'tempFeatureLayer')
# Checking to see if the output already exists.
# If so, remove it so that it can be recreated.
if Exists(startCalibrationPoints):
Delete_management(startCalibrationPoints)
else:
pass
if Exists(endCalibrationPoints):
Delete_management(endCalibrationPoints)
else:
pass
# Create 2 fieldInfo objects. Turn off all the fields in each one.
featureDesc = Describe('tempFeatureLayer')
if featureDesc.dataType == "FeatureLayer":
fieldInfo_For_Start_CP_Fields = featureDesc.fieldInfo
fieldInfo_For_End_CP_Fields = featureDesc.fieldInfo
# Use the count property to iterate through all the fields
for index in range(0, fieldInfo_For_Start_CP_Fields.count):
fieldInfo_For_Start_CP_Fields.setVisible(index, 'HIDDEN')
fieldInfo_For_End_CP_Fields.setVisible(index, 'HIDDEN')
# Turn on the needed fields.
visibile_Fields_For_Start_CP_Layer = [routeId, 'SourceFromMeasure']
for visibile_Field in visibile_Fields_For_Start_CP_Layer:
tempIndex = fieldInfo_For_Start_CP_Fields.findFieldByName(
visibile_Field)
fieldInfo_For_Start_CP_Fields.setVisible(tempIndex, 'VISIBLE')
# Create a feature layer that only shows the needed fields.
MakeFeatureLayer_management(createRoutesInputFC,
featureLayerCL_For_Start_CP, "", "",
fieldInfo_For_Start_CP_Fields)
# Use that feature layer to create the 1st calibration point set.
FeatureVerticesToPoints_management(featureLayerCL_For_Start_CP,
startCalibrationPoints, "START")
# Turn on the needed fields.
visibile_Fields_For_End_CP_Layer = [routeId, 'SourceToMeasure']
for visibile_Field in visibile_Fields_For_End_CP_Layer:
tempIndex = fieldInfo_For_End_CP_Fields.findFieldByName(visibile_Field)
fieldInfo_For_End_CP_Fields.setVisible(tempIndex, 'VISIBLE')
# Create a feature layer that only shows the needed fields.
MakeFeatureLayer_management(createRoutesInputFC, featureLayerCL_For_End_CP,
"", "", fieldInfo_For_End_CP_Fields)
# Use that feature layer to create the 2nd calibration point set.
FeatureVerticesToPoints_management(featureLayerCL_For_End_CP,
endCalibrationPoints, "END")
# ---- Merge the Start & End calibration points. ----
# Checking to see if the output already exists.
# If so, remove it so that it can be recreated.
if Exists(mergedCalibrationPoints):
Delete_management(mergedCalibrationPoints)
else:
pass
# RoutesSource_Start_CP.SourceRouteId to CalPts_Merge.RouteId
# RoutesSource_End_CP.SourceRouteId to CalPts_Merge.RouteId
mcp_Field1 = FieldMap()
mcp_Field1.addInputField(startCalibrationPoints, routeId)
mcp_Field1.addInputField(endCalibrationPoints, routeId)
mcp_Field1_OutField = mcp_Field1.outputField
mcp_Field1_OutField.name = 'RouteId'
mcp_Field1_OutField.aliasName = 'RouteId'
mcp_Field1_OutField.type = 'String'
mcp_Field1_OutField.length = 50
mcp_Field1.outputField = mcp_Field1_OutField
# RoutesSource_Start_CP.SourceFromMeasure to CalPts_Merge.Measure
mcp_Field2 = FieldMap()
mcp_Field2.addInputField(startCalibrationPoints, 'SourceFromMeasure')
mcp_Field2.addInputField(endCalibrationPoints, 'SourceToMeasure')
mcp_Field2_OutField = mcp_Field2.outputField
mcp_Field2_OutField.name = 'Measure'
mcp_Field2_OutField.aliasName = 'Measure'
mcp_Field2_OutField.type = 'Double'
mcp_Field2.outputField = mcp_Field2_OutField
# Create a fieldMappings object for the layer merge.
calibrationPointsMappings = FieldMappings()
calibrationPointsMappings.addFieldMap(mcp_Field1)
calibrationPointsMappings.addFieldMap(mcp_Field2)
#Merge the points together into a single feature class.
inputMergeLayers = [startCalibrationPoints, endCalibrationPoints]
Merge_management(inputMergeLayers, mergedCalibrationPoints,
calibrationPointsMappings)
MakeFeatureLayer_management(mergedCalibrationPoints, 'tempMergedPoints')
dissolveFields = ["RouteId", "Measure"]
print('Dissolving points.')
Dissolve_management('tempMergedPoints', dissolvedCalibrationPoints,
dissolveFields, "#", "SINGLE_PART")
def main():
if usePrefixSetTestingAndReporting == True:
for prefixKey in prefixSetErrorReportingDict.keys():
prefixAttributeQuery = prefixSetErrorReportingDict[prefixKey]
prefixKeyItemDict = outerTestDict[prefixKey]
prefixSetGdbBaseName = prefixKeyItemDict["prefixSetGdbBaseName"]
prefixSetSourceGDBName = prefixSetGdbBaseName + '_Source.gdb'
prefixSetSourceGDBLocation = os.path.join(mainFolder,
prefixSetSourceGDBName)
routesSourceOutputLocation = os.path.join(
prefixSetSourceGDBLocation, 'RoutesSource')
try:
Delete_management(
routesSourceFCAsALayer
) #pre-emptive layer delete prior to rebuilding it
time.sleep(3)
except:
pass
MakeFeatureLayer_management(routesSourceFC, routesSourceFCAsALayer)
routesSourceSelectionClause = """ """ + str(
prefixAttributeQuery) + """ """
SelectLayerByAttribute_management(routesSourceFCAsALayer,
"NEW_SELECTION",
routesSourceSelectionClause)
if Exists(returnGDBOrSDEPath(routesSourceOutputLocation)):
pass
else:
CreateFileGDB_management(
mainFolder,
returnGDBOrSDEName(
returnGDBOrSDEPath(routesSourceOutputLocation)))
# Checking to see if the output already exists.
# If so, remove it.
if Exists(routesSourceOutputLocation):
print(
"Deleting the previous routesSourceOutputLocation at: \n" +
str(routesSourceOutputLocation) + ".")
Delete_management(routesSourceOutputLocation)
time.sleep(7)
else:
pass
# Create a new file for the output.
print(
"Making a copy of the selection in the routesSourceFCAsALayer at: \n"
+ routesSourceOutputLocation + ".")
CopyFeatures_management(routesSourceFCAsALayer,
routesSourceOutputLocation)
#Repeat for each of the other layers to be copied into the new *_Source.gdb.
for itemToCopy in otherFCsToCopyList:
itemToCopyInputLocation = os.path.join(
returnGDBOrSDEPath(routesSourceFC), itemToCopy)
itemToCopyOutputLocation = os.path.join(
prefixSetSourceGDBLocation, itemToCopy)
if Exists(itemToCopyOutputLocation):
print(
"Deleting the previous itemToCopyOutputLocation at: \n"
+ str(itemToCopyOutputLocation) + ".")
Delete_management(itemToCopyOutputLocation)
time.sleep(7)
else:
pass
print("Making a copy of the itemToCopy at: \n" +
str(itemToCopyOutputLocation) + ".")
CopyFeatures_management(itemToCopyInputLocation,
itemToCopyOutputLocation)
else:
print(
"The usePrefixSetTestingAndReporting value is not True. Will not create separate prefix set gdbs."
)
def mainProcessFeatureSimplification(inputFeatures, maxCount, outputFeatures):
# Split the input features into intermediary features:
# Add each intermediary feature class to a list and
# pass one feature class of the intermediary features
# to each subprocess.
# When all of the subprocesses complete, use the
# list of the intermediary feature classes to append
# the data into the output features.
countResult = GetCount_management(inputFeatures)
intCount = int(countResult.getOutput(0))
# debug print
print("Counted " + str(intCount) + " features in the " + inputFeatures + " feature class.")
if maxCount > 15000:
maxCount = 15000
elif maxCount < 2000:
maxCount = 7000
else:
pass
neededMirrors = intCount / maxCount + 1
# debug print
print("Will create " + str(neededMirrors) + " reflection gdbs.")
infoForSubprocess = list()
gdbToCreateList = list()
for countItem in xrange(0, neededMirrors):
gdbMirrorName = mirrorBaseName + '_' + '0' + str(countItem) + '.gdb'
gdbMirrorFullPath = os.path.join(mainFolder, gdbMirrorName)
gdbToCreateList.append(gdbMirrorFullPath)
try:
if Exists(gdbMirrorFullPath):
try:
Delete_management(gdbMirrorFullPath)
except:
pass
else:
pass
except:
pass
CreateFileGDB_management(mainFolder, gdbMirrorName)
# do a selection on the input features here
# then copyfeatures to get the selected features
# output to the target gdb.
if Exists(simplifyTempLayer):
try:
Delete_management(simplifyTempLayer)
except:
pass
else:
pass
MakeFeatureLayer_management(inputFeatures, simplifyTempLayer)
currentSelectMin = int(countItem * maxCount)
currentSelectMax = int((countItem + 1) * maxCount)
dynSelectClause = """"OBJECTID" >= """ + str(currentSelectMin) + """ AND "OBJECTID" < """ + str(currentSelectMax) + """"""
SelectLayerByAttribute_management(simplifyTempLayer, "NEW_SELECTION", dynSelectClause)
selectedSimplifyFeatures = os.path.join(gdbMirrorFullPath, simplifyInputName)
CopyFeatures_management(simplifyTempLayer, selectedSimplifyFeatures)
subprocessInfoItem = [mainFolder, gdbMirrorFullPath, simplifyAlgorithm, simplifyDistance]
infoForSubprocess.append(subprocessInfoItem)
# Predivide the list of data driven pages that each process needs to run
# and pass it as a list of exportItems.
coreCount = mp.cpu_count()
# To support running this on the slow AR60, reduce the coreCount used to try to keep
# this script from crashing there.
if coreCount >= 3 and useMultithreading == True:
coreCount = coreCount - 1
print("Starting a multi-threaded job which will use (up to) " + str(coreCount) + " cores at once.")
workPool = mp.Pool(processes=coreCount)
# Note: This is a different usage of the word map than the one generally used in GIS.
workPool.map(subProcessFeatureSimplification, infoForSubprocess)
print("Multi-threaded job's done!")
print("Waiting a few moments before closing down the worker processes...")
time.sleep(20)
workPool.close()
time.sleep(20)
workPool.join()
print("Worker processes closed.")
else:
# Don't use multithreading here.
print("Using the single threaded process for feature simplification.")
print("This will be slower than the multi-threaded version,")
print("but it should also be less likely to crash on slower machines")
print("or those with low core counts.")
for singleThreadedProcessInfoListItem in infoForSubprocess:
singleThreadedProcessForSlowMachines(singleThreadedProcessInfoListItem)
print("Waiting a few moments before continuing to the next part of the script...")
time.sleep(20)
# Delete the output target prior to recreating it and appending data into it.
if Exists(outputFeatures):
try:
Delete_management(outputFeatures)
except:
pass
else:
pass
# Need the gdb and fc name here from outputFeatures.
outGDB = returnGDBOrSDEPath(outputFeatures)
outGDBName = returnGDBOrSDEName(outGDB)
outGDBFolder = returnGDBOrSDEFolder(outGDB)
outFCName = returnFeatureClass(outputFeatures)
if not Exists(outGDB):
CreateFileGDB_management(outGDBFolder, outGDBName)
# Use the inputFeatures as a template.
CreateFeatureclass_management(outGDB, outFCName, "", inputFeatures)
appendOutputFCList = list()
for gdbToCreate in gdbToCreateList:
appendOutputFC = os.path.join(gdbToCreate, 'simplificationOutput')
appendOutputFCList.append(appendOutputFC)
# Do appends here, then sleep again for a bit.
# Shouldn't need a field mapping since they should all be the same.
Append_management(appendOutputFCList, outputFeatures, "NO_TEST")
print "Waiting a few moments to be sure that all of the locks have been removed prior to deleting the reflection gdbs..."
time.sleep(20)
# Then remove the mirror gdbs.
for gdbToCreate in gdbToCreateList:
try:
if Exists(gdbToCreate):
try:
Delete_management(gdbToCreate)
except:
pass
else:
pass
except:
pass