def initProgress(folder, rerun):
try:
xmlFile = getProgressFilenames(folder).xmlFile
if not rerun:
removeFile(xmlFile)
# Create file if it does not exist
if not os.path.exists(xmlFile):
root = ET.Element("data")
tree = ET.ElementTree(root)
tree.write(xmlFile, encoding="utf-8", xml_declaration=True)
else:
# Open file for reading
tree = ET.parse(xmlFile)
root = tree.getroot()
# Write scratch GDB to XML file if not already present
scratchGDBNode = root.find('ScratchGDB')
if scratchGDBNode is None:
scratchGDBNode = createXMLNode(root, 'ScratchGDB')
scratchGDBNode.text = str(arcpy.env.scratchGDB)
try:
# Save the XML file
tree.write(xmlFile, encoding='utf-8', xml_declaration=True)
except Exception:
log.warning("Problem saving XML file " + str(xmlFile))
raise
except Exception:
log.warning('Could not initialise progress.xml')
def codeSuccessfullyRun(codeBlockName, folder, rerun):
try:
success = False
xmlFile = getProgressFilenames(folder).xmlFile
if rerun:
try:
# Open file for reading
tree = ET.parse(xmlFile)
root = tree.getroot()
codeBlockNodes = root.findall('CodeBlock')
except Exception:
removeFile(xmlFile)
else:
codeBlockNames = []
for codeBlockNode in codeBlockNodes:
names = codeBlockNode.findall('Name')
for name in names:
codeBlockNames.append(name.text)
if codeBlockName in codeBlockNames:
success = True
if success:
log.info('Skipping: ' + str(codeBlockName))
return success
except Exception:
log.warning('Could not check if code block was previously run')
log.warning(traceback.format_exc())
def checkInputRaster(raster, baseFolder):
noDataInRaster = int(arcpy.GetRasterProperties_management(raster, "ALLNODATA").getOutput(0)) # returns 0 (false) or 1 (true)
if noDataInRaster:
warning = 'Input raster ' + raster + ' only contains NoData values. Please check file.'
log.warning(warning)
common.logWarnings(baseFolder, warning)
def checkInputFC(featureClass, baseFolder):
# Find number of rows in feature class. arcpy.GetCount_management(featureClass).getOutput(0) should work but doesn't in some cases. KM
rows = [row for row in arcpy.da.SearchCursor(featureClass, "*")]
numFeatures = len(rows)
if numFeatures == 0:
warning = 'Input feature class ' + featureClass + ' contains no data. Please check file'
log.warning(warning)
common.logWarnings(baseFolder, warning)
def createXMLNode(parent, name):
try:
newNode = ET.Element(name)
parent.append(newNode)
return newNode
except Exception:
log.warning("Could not create node " + name)
raise
def getProgressFilenames(folder):
try:
class Files:
''' Declare filenames here '''
def __init__(self):
self.xmlFile = "progress.xml"
return common.addPath(Files(), folder)
except Exception:
log.warning("Error occurred while generating filenames")
raise
def clipLargeDEM(DEM, StudyAreaMask):
try:
# Work out filesize of DEM
cols = arcpy.GetRasterProperties_management(DEM, "COLUMNCOUNT").getOutput(0)
rows = arcpy.GetRasterProperties_management(DEM, "ROWCOUNT").getOutput(0)
bitType = int(arcpy.GetRasterProperties_management(DEM, "VALUETYPE").getOutput(0))
if bitType <= 4: # 8 bit
bytes = 1
elif bitType <= 6: # 16 bit
bytes = 2
elif bitType <= 9: # 32 bit
bytes = 4
elif bitType <= 14: # 64 bit
bytes = 8
else:
bytes = 4
sizeInGb = int(cols) * int(rows) * bytes / (1024 * 1024 * 1024)
if sizeInGb > 1: # 1Gb
log.info('Clipping DEM as original DEM is too large (approximately ' + str(sizeInGb) + 'Gb)')
# Buffer study area mask by 5km
bufferSAM = os.path.join(arcpy.env.scratchGDB, "bufferSAM")
arcpy.Buffer_analysis(StudyAreaMask, bufferSAM, "5000 meters", "FULL", "ROUND", "ALL")
# Clip DEM to this buffered area
bufferedDEM = os.path.join(arcpy.env.scratchWorkspace, "bufferedDEM")
extent = arcpy.Describe(bufferSAM).extent
arcpy.Clip_management(DEM, str(extent), bufferedDEM, bufferSAM, nodata_value="-3.402823e+038", clipping_geometry="ClippingGeometry", maintain_clipping_extent="NO_MAINTAIN_EXTENT")
log.warning('Since the DEM is large, reconditioning and preprocessing operations may take a long time')
return bufferedDEM
else:
return DEM
except Exception:
log.error("Error occurred when determining if DEM needs to be clipped or not")
raise
def checkCoverage(maskA, maskB, inputFile=None):
# Checks the coverage between polygons mask A and mask B
# Set temporary variables
prefix = os.path.join(arcpy.env.scratchGDB, "cover_")
maskACopy = prefix + "maskACopy"
maskBCopy = prefix + "maskBCopy"
maskACoverage = prefix + "maskACoverage"
# Copy maskA and maskBCopy to temporary variables
arcpy.CopyFeatures_management(maskA, maskACopy)
arcpy.CopyFeatures_management(maskB, maskBCopy)
percOut = 0
if checkLicenceLevel('Advanced') or arcpy.ProductInfo() == "ArcServer":
# Check if the fields Area_ha exist in maskACopy and maskBCopy
# If they do, delete them
maskAFields = arcpy.ListFields(maskACopy)
for field in maskAFields:
if field.name == "Area_ha":
arcpy.DeleteField_management(maskACopy, "Area_ha")
maskBFields = arcpy.ListFields(maskBCopy)
for field in maskBFields:
if field.name == "Area_ha":
arcpy.DeleteField_management(maskBCopy, "Area_ha")
# Delete fields area_ha in in maskACopy and maskBCopy
# Calculate total area of the coverage (study area or contributing area)
arcpy.AddField_management(maskBCopy, "Area_ha", "DOUBLE")
exp = "!SHAPE.AREA@HECTARES!"
arcpy.CalculateField_management(maskBCopy, "Area_ha", exp,
"PYTHON_9.3")
maskBCopyVal = 0
for row in arcpy.da.SearchCursor(maskBCopy, ["Area_ha"]):
maskBCopyVal += float(row[0])
# Use symmetrical difference to find which areas in maskA and maskBCopyCopy do not overlap
arcpy.SymDiff_analysis(maskACopy, maskBCopy, maskACoverage)
# Check if there are non-overlapping areas in the land use and coverage
rows = [row for row in arcpy.da.SearchCursor(maskACoverage, "*")]
numFeatures = len(rows)
if numFeatures != 0: # There is a discrepancy between mask A and mask B
# Calculate the area of the polygons in the discrepancy
arcpy.AddField_management(maskACoverage, "Sliver", "DOUBLE")
exp = "!SHAPE.AREA@HECTARES!"
arcpy.CalculateField_management(maskACoverage, "Sliver", exp,
"PYTHON_9.3")
area = 0
for row in arcpy.da.SearchCursor(maskACoverage, ["Sliver"]):
area += float(row[0])
# Calculate percentage of the coverage area that the discrepancy covers
percOut = float(area) / float(maskBCopyVal) * 100.0
if percOut > 2.5:
warning = 'Input data coverage is less tha 97.5 percent of the study area'
log.warning(warning)
warning = 'This may cause discrepancies in later calculations'
log.warning(warning)
warning = 'Please check this input: ' + str(inputFile)
log.warning(warning)
# Delete temporary masks
arcpy.Delete_management(maskACopy)
arcpy.Delete_management(maskBCopy)
arcpy.Delete_management(maskACoverage)
else:
log.warning(
'Coverage discrepancies between soil, land use, and coverage extent not checked as advanced license not present.'
)
log.warning(
'Please ensure the soil and land use shapefile cover at least 97.5 percent of the coverage extent'
)
def runSystemChecks(folder=None, rerun=False):
import LUCI_SEEA.lib.progress as progress
# Set overwrite output
arcpy.env.overwriteOutput = True
# Check spatial analyst licence is available
if arcpy.CheckExtension("Spatial") == "Available":
arcpy.CheckOutExtension("Spatial")
else:
raise RuntimeError(
"Spatial Analyst license not present or could not be checked out")
### Set workspaces so that temporary files are written to the LUCI scratch geodatabase ###
if arcpy.ProductInfo() == "ArcServer":
log.info('arcpy.env.scratchWorkspace on server: ' +
str(arcpy.env.scratchWorkspace))
# Set current workspace
arcpy.env.workspace = arcpy.env.scratchGDB
else:
# If rerunning a tool, check if scratch workspace has been set. If it has, use it as it is (with temporary rasters and feature classes from the previous run).
scratchGDB = None
if rerun:
xmlFile = progress.getProgressFilenames(folder).xmlFile
if os.path.exists(xmlFile):
scratchGDB = readXML(xmlFile, 'ScratchGDB')
if not arcpy.Exists(scratchGDB):
log.error('Previous scratch GDB ' + str(scratchGDB) +
' does not exist. Tool cannot be rerun.')
log.error('Exiting tool')
sys.exit()
if scratchGDB is None:
# Set scratch path from values in user settings file if values present
scratchPath = None
try:
if os.path.exists(configuration.userSettingsFile):
tree = ET.parse(configuration.userSettingsFile)
root = tree.getroot()
scratchPath = root.find("scratchPath").text
except Exception:
pass # If any errors occur, ignore them. Just use the default scratch path.
# Set scratch path if needed
if scratchPath is None:
scratchPath = configuration.scratchPath
# Create scratch path folder
if not os.path.exists(scratchPath):
os.makedirs(scratchPath)
# Remove old date/time stamped scratch folders if they exist and if they do not contain lock ArcGIS lock files.
for root, dirs, files in os.walk(scratchPath):
for dir in dirs:
# Try to rename folder. If this is possible then no locks are held on it and it can then be removed.
try:
fullDirPath = os.path.join(scratchPath, dir)
renamedDir = os.path.join(scratchPath,
'ready_for_deletion')
os.rename(fullDirPath, renamedDir)
except Exception:
# import traceback
# log.warning(traceback.format_exc())
pass
else:
try:
shutil.rmtree(renamedDir)
except Exception:
# import traceback
# log.warning(traceback.format_exc())
pass
# Create new date/time stamped scratch folder for the scratch GDB to live in
dateTimeStamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
scratchGDBFolder = os.path.join(scratchPath,
'scratch_' + dateTimeStamp)
if not os.path.exists(scratchGDBFolder):
os.mkdir(scratchGDBFolder)
# Create scratch GDB
scratchGDB = os.path.join(scratchGDBFolder, 'scratch.gdb')
if not os.path.exists(scratchGDB):
arcpy.CreateFileGDB_management(os.path.dirname(scratchGDB),
os.path.basename(scratchGDB))
# Try to remove old scratch path if still exists
try:
shutil.rmtree(configuration.oldScratchPath, ignore_errors=True)
except Exception:
pass
# Set scratch and current workspaces
arcpy.env.scratchWorkspace = scratchGDB
arcpy.env.workspace = scratchGDB
# Scratch folder
scratchFolder = arcpy.env.scratchFolder
if not os.path.exists(scratchFolder):
os.mkdir(scratchFolder)
# Remove all in_memory data sets
arcpy.Delete_management("in_memory")
# Check disk space for disk with scratch workspace
freeSpaceGb = 3
if getFreeDiskSpaceGb(arcpy.env.scratchWorkspace) < freeSpaceGb:
log.warning("Disk containing scratch workspace has less than " +
str(freeSpaceGb) +
"Gb free space. This may cause this tool to fail.")
def function(outputFolder,
DEM,
studyAreaMask,
streamInput,
minAccThresh,
majAccThresh,
smoothDropBuffer,
smoothDrop,
streamDrop,
reconDEM,
rerun=False):
try:
# Set environment variables
arcpy.env.compression = "None"
arcpy.env.snapRaster = DEM
arcpy.env.extent = DEM
arcpy.env.cellSize = arcpy.Describe(DEM).meanCellWidth
########################
### Define filenames ###
########################
files = common.getFilenames('preprocess', outputFolder)
rawDEM = files.rawDEM
hydDEM = files.hydDEM
hydFDR = files.hydFDR
hydFDRDegrees = files.hydFDRDegrees
hydFAC = files.hydFAC
streamInvRas = files.streamInvRas # Inverse stream raster - 0 for stream, 1 for no stream
streams = files.streams
streamDisplay = files.streamDisplay
multRaster = files.multRaster
hydFACInt = files.hydFACInt
slopeRawDeg = files.slopeRawDeg
slopeRawPer = files.slopeRawPer
slopeHydDeg = files.slopeHydDeg
slopeHydPer = files.slopeHydPer
###############################
### Set temporary variables ###
###############################
prefix = os.path.join(arcpy.env.scratchGDB, "base_")
cellSizeDEM = float(arcpy.env.cellSize)
burnedDEM = prefix + "burnedDEM"
streamAccHaFile = prefix + "streamAccHa"
rawFDR = prefix + "rawFDR"
allPolygonSinks = prefix + "allPolygonSinks"
DEMTemp = prefix + "DEMTemp"
hydFACTemp = prefix + "hydFACTemp"
# Saved as .tif as did not save as ESRI grid on server
streamsRasterFile = os.path.join(arcpy.env.scratchFolder,
"base_") + "StreamsRaster.tif"
###############################
### Save DEM to base folder ###
###############################
codeBlock = 'Save DEM'
if not progress.codeSuccessfullyRun(codeBlock, outputFolder, rerun):
# Save DEM to base folder as raw DEM with no compression
pixelType = int(
arcpy.GetRasterProperties_management(DEM,
"VALUETYPE").getOutput(0))
if pixelType == 9: # 32 bit float
arcpy.CopyRaster_management(DEM,
rawDEM,
pixel_type="32_BIT_FLOAT")
else:
log.info("Converting DEM to 32 bit floating type")
arcpy.CopyRaster_management(DEM, DEMTemp)
arcpy.CopyRaster_management(Float(DEMTemp),
rawDEM,
pixel_type="32_BIT_FLOAT")
# Delete temporary DEM
arcpy.Delete_management(DEMTemp)
# Calculate statistics for raw DEM
arcpy.CalculateStatistics_management(rawDEM)
progress.logProgress(codeBlock, outputFolder)
################################
### Create multiplier raster ###
################################
codeBlock = 'Create multiplier raster'
if not progress.codeSuccessfullyRun(codeBlock, outputFolder, rerun):
Reclassify(rawDEM, "Value", RemapRange([[-999999.9, 999999.9, 1]]),
"NODATA").save(multRaster)
progress.logProgress(codeBlock, outputFolder)
codeBlock = 'Calculate slope in percent'
if not progress.codeSuccessfullyRun(codeBlock, outputFolder, rerun):
intSlopeRawPer = Slope(rawDEM, "PERCENT_RISE")
intSlopeRawPer.save(slopeRawPer)
del intSlopeRawPer
log.info('Slope calculated in percent')
progress.logProgress(codeBlock, outputFolder)
if reconDEM is True:
#######################
### Burn in streams ###
#######################
codeBlock = 'Burn in streams'
if not progress.codeSuccessfullyRun(codeBlock, outputFolder,
rerun):
# Recondition DEM (burning stream network in using AGREE method)
log.info("Burning streams into DEM.")
reconditionDEM.function(rawDEM, streamInput, smoothDropBuffer,
smoothDrop, streamDrop, burnedDEM)
log.info("Completed stream network burn in to DEM")
progress.logProgress(codeBlock, outputFolder)
##################
### Fill sinks ###
##################
codeBlock = 'Fill sinks'
if not progress.codeSuccessfullyRun(codeBlock, outputFolder,
rerun):
Fill(burnedDEM).save(hydDEM)
log.info("Sinks in DEM filled")
progress.logProgress(codeBlock, outputFolder)
######################
### Flow direction ###
######################
codeBlock = 'Flow direction'
if not progress.codeSuccessfullyRun(codeBlock, outputFolder,
rerun):
FlowDirection(hydDEM, "NORMAL").save(hydFDR)
log.info("Flow Direction calculated")
progress.logProgress(codeBlock, outputFolder)
#################################
### Flow direction in degrees ###
#################################
codeBlock = 'Flow direction in degrees'
if not progress.codeSuccessfullyRun(codeBlock, outputFolder,
rerun):
# Save flow direction raster in degrees (for display purposes)
degreeValues = RemapValue([[1, 90], [2, 135], [4, 180],
[8, 225], [16, 270], [32, 315],
[64, 0], [128, 45]])
Reclassify(hydFDR, "Value", degreeValues,
"NODATA").save(hydFDRDegrees)
progress.logProgress(codeBlock, outputFolder)
#########################
### Flow accumulation ###
#########################
codeBlock = 'Flow accumulation'
if not progress.codeSuccessfullyRun(codeBlock, outputFolder,
rerun):
hydFACTemp = FlowAccumulation(hydFDR, "", "FLOAT")
hydFACTemp.save(hydFAC)
arcpy.sa.Int(Raster(hydFAC)).save(hydFACInt) # integer version
log.info("Flow Accumulation calculated")
progress.logProgress(codeBlock, outputFolder)
#######################
### Calculate slope ###
#######################
codeBlock = 'Calculate slope on burned DEM'
if not progress.codeSuccessfullyRun(codeBlock, outputFolder,
rerun):
intSlopeHydDeg = Slope(hydDEM, "DEGREE")
intSlopeHydDeg.save(slopeHydDeg)
del intSlopeHydDeg
intSlopeHydPer = Slope(hydDEM, "PERCENT_RISE")
intSlopeHydPer.save(slopeHydPer)
del intSlopeHydPer
log.info('Slope calculated')
progress.logProgress(codeBlock, outputFolder)
##########################
### Create stream file ###
##########################
codeBlock = 'Create stream file'
if not progress.codeSuccessfullyRun(codeBlock, outputFolder,
rerun):
# Create accumulation in metres
streamAccHaFileInt = hydFACTemp * cellSizeDEM * cellSizeDEM / 10000.0
streamAccHaFileInt.save(streamAccHaFile)
del streamAccHaFileInt
# Check stream initiation threshold reached
streamYes = float(
arcpy.GetRasterProperties_management(
streamAccHaFile, "MAXIMUM").getOutput(0))
if streamYes > float(minAccThresh):
reclassifyRanges = RemapRange(
[[-1000000, float(minAccThresh), 1],
[float(minAccThresh), 9999999999, 0]])
outLUCIstream = Reclassify(streamAccHaFile, "VALUE",
reclassifyRanges)
outLUCIstream.save(streamInvRas)
del outLUCIstream
log.info("Stream raster for input to LUCI created")
# Create stream file for display
reclassifyRanges = RemapRange(
[[0, float(minAccThresh), "NODATA"],
[float(minAccThresh),
float(majAccThresh), 1],
[float(majAccThresh), 99999999999999, 2]])
streamsRaster = Reclassify(streamAccHaFile, "Value",
reclassifyRanges, "NODATA")
streamOrderRaster = StreamOrder(streamsRaster, hydFDR,
"STRAHLER")
streamsRaster.save(streamsRasterFile)
# Create two streams feature classes - one for analysis and one for display
arcpy.sa.StreamToFeature(streamOrderRaster, hydFDR,
streams, 'NO_SIMPLIFY')
arcpy.sa.StreamToFeature(streamOrderRaster, hydFDR,
streamDisplay, 'SIMPLIFY')
# Rename grid_code column to 'Strahler'
for streamFC in [streams, streamDisplay]:
arcpy.AddField_management(streamFC, "Strahler", "LONG")
arcpy.CalculateField_management(
streamFC, "Strahler", "!GRID_CODE!", "PYTHON_9.3")
arcpy.DeleteField_management(streamFC, "GRID_CODE")
del streamsRaster
del streamOrderRaster
log.info("Stream files created")
else:
warning = 'No streams initiated'
log.warning(warning)
common.logWarnings(outputFolder, warning)
# Create LUCIStream file from multiplier raster (i.e. all cells have value of 1 = no stream)
arcpy.CopyRaster_management(multRaster, streamInvRas)
progress.logProgress(codeBlock, outputFolder)
codeBlock = 'Clip data, build pyramids and generate statistics'
if not progress.codeSuccessfullyRun(codeBlock, outputFolder, rerun):
try:
# Generate pyramids and stats
arcpy.BuildPyramidsandStatistics_management(
outputFolder, "", "", "", "")
log.info(
"Pyramids and Statistics calculated for all LUCI topographical information rasters"
)
except Exception:
log.info("Warning - could not generate all raster statistics")
progress.logProgress(codeBlock, outputFolder)
# Reset snap raster
arcpy.env.snapRaster = None
except Exception:
log.error("Error in preprocessing operations")
raise
def function(params):
try:
###################
### Read inputs ###
###################
pText = common.paramsAsText(params)
outputFolder = pText[1]
inputDEM = common.fullPath(pText[2])
inputStudyAreaMask = pText[3]
inputLC = pText[4]
lcCode = pText[5]
inputSoil = pText[6]
soilCode = pText[7]
reconDEM = common.strToBool(pText[8])
inputStreamNetwork = pText[9]
streamAccThresh = pText[10]
riverAccThresh = pText[11]
smoothDropBuffer = pText[12]
smoothDrop = pText[13]
streamDrop = pText[14]
rerun = common.strToBool(pText[15])
log.info('Inputs read in')
###########################
### Tool initialisation ###
###########################
# Create Baseline folder
if not os.path.exists(outputFolder):
os.mkdir(outputFolder)
# Set up logging output to file
log.setupLogging(outputFolder)
# Run system checks
common.runSystemChecks(outputFolder, rerun)
# Set up progress log file
progress.initProgress(outputFolder, rerun)
# Write input params to XML
common.writeParamsToXML(params, outputFolder, 'PreprocessDEM')
log.info('Tool initialised')
########################
### Define filenames ###
########################
files = common.getFilenames('preprocess', outputFolder)
studyAreaMask = files.studyareamask
outputLCras = files.lc_ras
outputLCvec = files.lc_vec
outputSoilras = files.soil_ras
outputSoilvec = files.soil_vec
###############################
### Set temporary variables ###
###############################
prefix = os.path.join(arcpy.env.scratchGDB, 'base_')
DEMTemp = prefix + 'DEMTemp'
clippedDEM = prefix + 'clippedDEM'
clippedLC = prefix + 'clippedLC'
clippedSoil = prefix + 'clippedSoil'
clippedStreamNetwork = prefix + 'clippedStreamNetwork'
studyAreaMaskTemp = prefix + "studyAreaMaskTemp"
studyAreaMaskBuff = prefix + "studyAreaMaskBuff"
studyAreaMaskDiss = prefix + "studyAreaMaskDiss"
log.info('Temporary variables set')
# Check formats of inputs
lcFormat = arcpy.Describe(inputLC).dataType
soilFormat = arcpy.Describe(inputSoil).dataType
###################
### Data checks ###
###################
codeBlock = 'Data checks 1'
if not progress.codeSuccessfullyRun(codeBlock, outputFolder, rerun):
inputFiles = [inputDEM, inputStudyAreaMask, inputLC, inputSoil]
if inputStreamNetwork is not None:
inputFiles.append(inputStreamNetwork)
for file in inputFiles:
common.checkSpatialRef(file)
# Set environment variables
arcpy.env.snapRaster = inputDEM
arcpy.env.cellSize = inputDEM
arcpy.env.compression = "None"
cellsizedem = float(
arcpy.GetRasterProperties_management(inputDEM,
"CELLSIZEX").getOutput(0))
# Get spatial references of DEM and study area mask
DEMSpatRef = arcpy.Describe(inputDEM).SpatialReference
maskSpatRef = arcpy.Describe(inputStudyAreaMask).SpatialReference
# Reproject study area mask if it does not have the same coordinate system as the DEM
if not common.equalProjections(DEMSpatRef, maskSpatRef):
warning = "Study area mask does not have the same coordinate system as the DEM"
log.warning(warning)
common.logWarnings(outputFolder, warning)
warning = "Mask coordinate system is " + maskSpatRef.Name + " while DEM coordinate system is " + DEMSpatRef.Name
log.warning(warning)
common.logWarnings(outputFolder, warning)
warning = "Reprojecting study area mask to DEM coordinate system"
log.warning(warning)
common.logWarnings(outputFolder, warning)
arcpy.Project_management(inputStudyAreaMask, studyAreaMaskTemp,
DEMSpatRef)
arcpy.CopyFeatures_management(studyAreaMaskTemp, studyAreaMask)
else:
arcpy.CopyFeatures_management(inputStudyAreaMask,
studyAreaMask)
# If DEM is large, clip it to a large buffer around the study area mask (~5km)
inputDEM = baseline.clipLargeDEM(inputDEM, studyAreaMask)
rasterInputFiles = []
fcInputFiles = []
# Sort land cover and soil into appropriate arrays based on data type
if lcFormat in ['RasterDataset', 'RasterLayer']:
rasterInputFiles.append(inputLC)
outputLC = os.path.join(outputFolder, 'landcover')
elif lcFormat in ['ShapeFile', 'FeatureClass']:
fcInputFiles.append(inputLC)
outputLC = os.path.join(outputFolder, 'landcover.shp')
if soilFormat in ['RasterDataset', 'RasterLayer']:
rasterInputFiles.append(inputSoil)
outputSoil = os.path.join(outputFolder, 'soil')
elif soilFormat in ['ShapeFile', 'FeatureClass']:
fcInputFiles.append(inputSoil)
outputSoil = os.path.join(outputFolder, 'soil.shp')
if reconDEM is True and inputStreamNetwork is None:
log.error(
'Cannot recondition the DEM without an input stream network'
)
log.error('Please provide an input stream network')
sys.exit()
# If the user has provided a stream network, add it to the list of inputs to check
if inputStreamNetwork is not None:
fcInputFiles.append(inputStreamNetwork)
# Check that the inputs contain data
for ras in rasterInputFiles:
if ras is not None:
# Check file size
fileSizeGB = baseline.checkRasterSizeGB(ras)
if fileSizeGB < 1.0:
baseline.checkInputRaster(ras, outputFolder)
else:
log.warning(
"Cannot check if raster is empty or all NoData because it is too large"
)
log.warning(
"Please ensure this raster is not empty or all NoData: "
+ str(ras))
for fc in fcInputFiles:
if fc is not None:
baseline.checkInputFC(fc, outputFolder)
# Check that the land cover and soil FCs have the linking codes specified by the user
if lcFormat in ['ShapeFile', 'FeatureClass']:
if len(arcpy.ListFields(inputLC, lcCode)) != 1:
log.error('Field ' + lcCode +
'does not exist in feature class ' + inputLC)
sys.exit()
if soilFormat in ['ShapeFile', 'FeatureClass']:
if len(arcpy.ListFields(inputSoil, soilCode)) != 1:
log.error('Field ' + soilCode +
'does not exist in feature class ' + inputSoil)
sys.exit()
progress.logProgress(codeBlock, outputFolder)
###############################
### Tidy up study area mask ###
###############################
codeBlock = 'Tidy up study area mask'
if not progress.codeSuccessfullyRun(codeBlock, outputFolder, rerun):
# Check how many polygons are in the mask shapefile
numPolysInMask = int(
arcpy.GetCount_management(studyAreaMask).getOutput(0))
if numPolysInMask > 1:
# Reduce multiple features where possible
arcpy.Union_analysis(studyAreaMask, studyAreaMaskDiss,
"ONLY_FID", "", "NO_GAPS")
arcpy.Dissolve_management(studyAreaMaskDiss, studyAreaMask, "",
"", "SINGLE_PART", "DISSOLVE_LINES")
# Buffer study area mask
baseline.bufferMask(inputDEM,
studyAreaMask,
outputStudyAreaMaskBuff=studyAreaMaskBuff)
log.info('Study area mask buffered')
progress.logProgress(codeBlock, outputFolder)
#######################
### Clip input data ###
#######################
codeBlock = 'Clip inputs'
if not progress.codeSuccessfullyRun(codeBlock, outputFolder, rerun):
baseline.clipInputs(outputFolder,
studyAreaMaskBuff,
inputDEM,
inputLC,
inputSoil,
inputStreamNetwork,
outputDEM=clippedDEM,
outputLC=clippedLC,
outputSoil=clippedSoil,
outputStream=clippedStreamNetwork)
progress.logProgress(codeBlock, outputFolder)
##############################################
### Coverage checks on soil and land cover ###
##############################################
codeBlock = 'Do coverage checks on clipped land cover and soil'
if not progress.codeSuccessfullyRun(codeBlock, outputFolder, rerun):
# Do coverage checks on land cover and soil and copy to outputFolder
if lcFormat in ['RasterDataset', 'RasterLayer']:
lcMask = common.extractRasterMask(clippedLC)
common.checkCoverage(lcMask, studyAreaMaskBuff, inputLC)
arcpy.CopyRaster_management(clippedLC, outputLCras)
elif lcFormat in ['ShapeFile', 'FeatureClass']:
lcMask = common.dissolvePolygon(clippedLC)
common.checkCoverage(lcMask, studyAreaMaskBuff, inputLC)
arcpy.CopyFeatures_management(clippedLC, outputLCvec)
if soilFormat in ['RasterDataset', 'RasterLayer']:
soilMask = common.extractRasterMask(clippedSoil)
common.checkCoverage(soilMask, studyAreaMaskBuff, inputLC)
arcpy.CopyRaster_management(clippedSoil, outputSoilras)
elif soilFormat in ['ShapeFile', 'FeatureClass']:
soilMask = common.dissolvePolygon(clippedSoil)
common.checkCoverage(soilMask, studyAreaMaskBuff, inputSoil)
arcpy.CopyFeatures_management(clippedSoil, outputSoilvec)
progress.logProgress(codeBlock, outputFolder)
######################################
### Convert LC and soil to rasters ###
######################################
# For the RUSLE tool, the LC and soil must be in raster format
codeBlock = 'Convert land cover and soil to rasters'
if not progress.codeSuccessfullyRun(codeBlock, outputFolder, rerun):
if lcFormat in ['ShapeFile', 'FeatureClass']:
arcpy.PolygonToRaster_conversion(clippedLC, lcCode,
outputLCras, "CELL_CENTER",
"", cellsizedem)
log.info('Land cover raster produced')
if soilFormat in ['ShapeFile', 'FeatureClass']:
arcpy.PolygonToRaster_conversion(clippedSoil, soilCode,
outputSoilras, "CELL_CENTER",
"", cellsizedem)
log.info('Soil raster produced')
# Delete intermediate files
arcpy.Delete_management(clippedLC)
arcpy.Delete_management(clippedSoil)
progress.logProgress(codeBlock, outputFolder)
###########################
### Run HydTopo process ###
###########################
log.info("*** Preprocessing DEM ***")
preprocess_dem.function(outputFolder, clippedDEM, studyAreaMask,
clippedStreamNetwork, streamAccThresh,
riverAccThresh, smoothDropBuffer, smoothDrop,
streamDrop, reconDEM, rerun)
except Exception:
arcpy.SetParameter(0, False)
log.exception("Preprocessing DEM functions did not complete")
raise