def test_real(self):
from lib.raster import Raster, PrintArr
from lib.shapefileloader import Shapefile
r = Raster("D:/CHaMP/Harold/2014/Entiat/ENT00001-1BC1/VISIT_2447/Topo/GISLayers/DEM.tif")
theArr = r.rasterMaskLayer("D:/CHaMP/Harold/2014/Entiat/ENT00001-1BC1/VISIT_2447/Topo/GISLayers/Channel_Units.shp", 'Unit_Numbe')
PrintArr(theArr)
def rasterToCSV(rasterPath, outputCSVPath):
print "Writing raster {0} to {1}".format(os.path.basename(rasterPath),
outputCSVPath)
raster = Raster(rasterPath)
npArr = raster.array
# Retrieve and return the extent
extent = {}
extent['Top'] = raster.top
extent['Left'] = raster.left
extent['Right'] = raster.getRight()
extent['Bottom'] = raster.getBottom()
writer = csv.writer(open(outputCSVPath, 'wb'))
header = ['x', 'y', 'value']
writer.writerow(header)
for i, iVal in enumerate(npArr):
for j, jVal in enumerate(iVal):
if not np.ma.is_masked(jVal):
x = raster.left + (j * raster.cellWidth) + (raster.cellWidth /
2)
y = raster.top + (i * raster.cellHeight) + (raster.cellHeight /
2)
writer.writerow([x, y, raster.array[i][j]])
return extent
def get_z_on_dem(self, filterlist=None):
feats = self.features
if feats and self.dem:
r = Raster(self.dem)
if filterlist:
feats = [feat for feat in feats if feat['fields'][self.fieldName_Description] in filterlist]
feats = [feat for feat in feats for poly in self.demDataExtent if poly.contains(feat['geometry'])]
list_z = [r.getPixelVal([feat['geometry'].x, feat['geometry'].y]) for feat in feats]
return list_z
else:
return []
def RasterMetrics(rpath):
result = {'StDev': None}
theRaster = Raster(rpath)
result['StDev'] = np.std(theRaster.array)
return result
def calc(self, channelName, shpWaterExtent, shpCenterline, rasDepth):
if not os.path.isfile(shpCenterline):
raise MissingException("Missing centerline shapefile")
if not os.path.isfile(shpWaterExtent):
raise MissingException("Missing water extent shapefile")
if not os.path.isfile(rasDepth):
raise MissingException("Missing depth raster")
# Load the depth raster
depthRaster = Raster(rasDepth)
# Load the water extent ShapeFile and sum the area of all features
shpExtent = Shapefile(shpWaterExtent)
feats = shpExtent.featuresToShapely()
arrMask = depthRaster.rasterMaskLayer(shpWaterExtent)
self.metrics['Area'] = 0
# TODO: See issue #79 and #15. We need to use a different strategy if we want volume for bankfull https://github.com/SouthForkResearch/CHaMP_Metrics/issues/79
if channelName == "Bankfull":
self.metrics['Volume'] = None
for aFeat in feats:
self.metrics['Area'] += aFeat['geometry'].area
else:
self.metrics['Volume'] = np.sum(
depthRaster.array[arrMask > 0]) * (depthRaster.cellWidth**2)
for aFeat in feats:
self.metrics['Area'] += aFeat['geometry'].area
dDepth = {}
dDepth['Max'] = float(np.amax(depthRaster.array[arrMask > 0]))
dDepth['Mean'] = np.mean(depthRaster.array[arrMask > 0])
self.metrics['Depth'] = dDepth
# Retrieve the centerline mainstem. If the ShapeFile possesses a 'Channel field then
# find the 1 feature with a value of 'Main', otherwise this is simply the only ShapeFile feature
mainstem = self._getMainstemGeometry(shpCenterline)
self.metrics['IntegratedWidth'] = None
if mainstem and mainstem.length > 0:
self.metrics[
'IntegratedWidth'] = self.metrics['Area'] / mainstem.length
def getDEMExtents(projects):
outerExtent = {'Top': None, 'Left': None, 'Right': None, 'Bottom': None}
extents = {}
for proj in projects:
dempath = proj['project'].getpath('DEM')
vid = proj['visit']
dem = Raster(dempath)
extents[vid] = {
'Top': dem.top,
'Left': dem.left,
'Right': dem.getRight(),
'Bottom': dem.getBottom()
}
if outerExtent['Top']:
outerExtent['Top'] = max(dem.top, outerExtent['Top'])
else:
outerExtent['Top'] = dem.top
if outerExtent['Left']:
outerExtent['Left'] = min(dem.left, outerExtent['Left'])
else:
outerExtent['Left'] = dem.left
if outerExtent['Right']:
outerExtent['Right'] = max(dem.getRight(), outerExtent['Right'])
else:
outerExtent['Right'] = dem.getRight()
if outerExtent['Bottom']:
outerExtent['Bottom'] = min(dem.getBottom(), outerExtent['Bottom'])
else:
outerExtent['Bottom'] = dem.getBottom()
extents['OuterExtent'] = copy.deepcopy(outerExtent)
return extents
def calc(self, sThalwegshp, sDepthRaster, sWaterSurfaceRaster, fDist,
visitMetrics):
if not path.isfile(sThalwegshp):
raise MissingException("Thalweg shapefile missing")
if not path.isfile(sDepthRaster):
raise MissingException("Depth raster missing")
if not path.isfile(sWaterSurfaceRaster):
raise MissingException("Surface raster missing")
wettedMainstemLength = visitMetrics['Wetted']['Centerline'][
'MainstemLength']
if wettedMainstemLength is None:
raise MissingException(
"No wetted mainstem length found in visit metrics")
sfile = Shapefile(sThalwegshp).featuresToShapely()
if len(sfile) < 1:
raise DataException("Thalweg shapefile has no features")
thalweg = sfile[0]['geometry']
depthRaster = Raster(sDepthRaster)
waterSurfaceRaster = Raster(sWaterSurfaceRaster)
samplepts = ThalwegMetrics.interpolateRasterAlongLine(thalweg, fDist)
results = ThalwegMetrics.lookupRasterValues(samplepts,
depthRaster)['values']
# Get the elevation at the first (downstream) point on the Thalweg
dsElev = waterSurfaceRaster.getPixelVal(thalweg.coords[0])
usElev = waterSurfaceRaster.getPixelVal(thalweg.coords[-1])
if (np.isnan(dsElev)):
raise DataException(
'nodata detected in the raster for downstream point on the thalweg'
)
elif np.isnan(usElev):
raise DataException(
'nodata detected in the raster for upstream point on the thalweg'
)
waterSurfaceGradientRatio = (usElev - dsElev) / thalweg.length
waterSurfaceGradientPC = waterSurfaceGradientRatio * 100.0
# Thalweg straight length and sinuosity
firstPoint = Point(thalweg.coords[0])
lastPoint = Point(thalweg.coords[-1])
straightLength = firstPoint.distance(lastPoint)
sinuosity = thalweg.length / straightLength
self.metrics = {
'Min': np.nanmin(results),
'Max': np.nanmax(results),
'Mean': np.mean(results),
'StDev': np.std(results),
'Count': np.count_nonzero(results),
'Length': thalweg.length,
'WSGradientRatio': waterSurfaceGradientRatio,
'WSGradientPC': waterSurfaceGradientPC,
'Sinuosity': sinuosity,
'CV': 0.0,
'ThalwegToCenterlineRatio': thalweg.length / wettedMainstemLength
#, 'Values': results.data
}
if self.metrics['StDev'] != 0 and self.metrics['Mean'] != 0:
self.metrics['CV'] = self.metrics['StDev'] / self.metrics['Mean']
def get_raster(self):
return Raster(self.filename)
def calc(self, shpCUPath, shpThalweg, rasDepth, visitMetrics, dUnits,
unitDefs):
if not os.path.isfile(shpCUPath):
raise MissingException("Channel units file not found")
if not os.path.isfile(shpThalweg):
raise MissingException("Thalweg shape file not found")
if not os.path.isfile(rasDepth):
raise MissingException("Depth raster file not found")
siteLength = visitMetrics['Wetted']['Centerline']['MainstemLength']
if siteLength is None:
raise DataException("No valid site length found in visit metrics")
# Give us a fresh template with 0's in the value positions
self.metrics = self._templateMaker(0, unitDefs)
dResultsChannelSummary = self.metrics['ResultsChannelSummary']
dResultsTier1 = self.metrics['ResultsTier1']
dResultsTier2 = self.metrics['ResultsTier2']
resultsCU = self.metrics['resultsCU']
#Load the Thalweg feature
thalweg = Shapefile(shpThalweg).featuresToShapely()
thalwegLine = thalweg[0]['geometry']
# Load the depth raster
depthRaster = Raster(rasDepth)
# Load the channel unit polygons and calculate the total area
# The channel units should be clipped to the wetted extent and so this
# can be used as the site area wetted
shpCU = Shapefile(shpCUPath)
arrCU = depthRaster.rasterMaskLayer(shpCUPath, "UnitNumber")
feats = shpCU.featuresToShapely()
for aFeat in feats:
dResultsChannelSummary['Main']['Area'] += aFeat['geometry'].area
# Loop over each channel unit and calculate topometrics
for aFeat in feats:
nCUNumber = int(aFeat['fields']['UnitNumber'])
if nCUNumber not in dUnits:
self.log.error(
"Channel Unit: '{0}' not present in the aux data.".format(
nCUNumber))
# Keep it general for the exception so we can aggregate them
raise DataException(
"The Channel Unit ShapeFile contains a unit number that is not present in the aux data."
)
tier1Name = dUnits[nCUNumber][0]
tier2Name = dUnits[nCUNumber][1]
nSegment = dUnits[nCUNumber][2]
#print "Channel Unit Number {0}, Segment {1}, Tier 1 - {2}, Tier 2 - {3}".format(nCUNumber, nSegment, tier1Name, tier2Name)
unitMetrics = {}
resultsCU.append(unitMetrics)
unitMetrics['ChannelUnitNumber'] = nCUNumber
unitMetrics['Area'] = aFeat['geometry'].area
unitMetrics['Tier1'] = tier1Name
unitMetrics['Tier2'] = tier2Name
unitMetrics['Length'] = None
unitMetrics['ResidualDepth'] = None
unitMetrics['DepthAtThalwegExit'] = None
unitMetrics['ThalwegIntersect'] = 0
# Get the depth raster for this unit as variable so we can check
# whether it is entirely masked below.
depArr = depthRaster.array[arrCU == nCUNumber]
if depArr.count() == 0:
unitMetrics['MaxDepth'] = 0
unitMetrics['Volume'] = 0
else:
unitMetrics['MaxDepth'] = np.max(depArr)
unitMetrics['Volume'] = np.sum(depthRaster.array[
arrCU == nCUNumber]) * (depthRaster.cellWidth**2)
if nSegment != 1:
dSideChannelSummary = dResultsChannelSummary[
'SideChannelSummary']
dMain = dResultsChannelSummary['Main']
# Side channel summary captures both small and large side channels
dSideChannelSummary['Area'] += aFeat['geometry'].area
dSideChannelSummary['Count'] += 1
dSideChannelSummary['Percent'] = 100 * dSideChannelSummary[
'Area'] / dMain['Area']
dSideChannelSummary['Volume'] += unitMetrics['Volume']
if 'side' in tier1Name.lower():
dSmallSideChannel = dResultsChannelSummary[
'SmallSideChannel']
dSmallSideChannel['Area'] += aFeat['geometry'].area
dSmallSideChannel['Count'] += 1
dSmallSideChannel['Percent'] = 100 * dSmallSideChannel[
'Area'] / dMain['Area']
dSmallSideChannel['Volume'] += unitMetrics['Volume']
else:
dLargeSideChannel = dResultsChannelSummary[
'LargeSideChannel']
dLargeSideChannel['Area'] += aFeat['geometry'].area
dLargeSideChannel['Count'] += 1
dLargeSideChannel['Percent'] = 100 * dLargeSideChannel[
'Area'] / dMain['Area']
dLargeSideChannel['Volume'] += unitMetrics['Volume']
if tier1Name is None:
raise DataException("tier1Name cannot be 'None'")
if 'side' in tier1Name.lower():
dResultsChannelSummary['ChannelUnitBreakdown'][
'SmallSideChannel'] += 1
else:
dResultsChannelSummary['ChannelUnitBreakdown']['Other'] += 1
if (thalwegLine.intersects(aFeat['geometry'])):
cuThalwegLine = thalwegLine.intersection(aFeat['geometry'])
exitPoint = None
if cuThalwegLine.type == 'LineString':
exitPoint = cuThalwegLine.coords[0]
else:
exitPoint = cuThalwegLine[0].coords[0]
# Retrieve a list of points along the Thalweg in the channel unit
thalwegPoints = ChannelUnitMetrics.interpolatePointsAlongLine(
cuThalwegLine, 0.13)
thalwegDepths = ChannelUnitMetrics.lookupRasterValuesAtPoints(
thalwegPoints, depthRaster)
unitMetrics['MaxDepth'] = np.nanmax(thalwegDepths['values'])
unitMetrics['DepthAtThalwegExit'] = depthRaster.getPixelVal(
exitPoint)
unitMetrics['ResidualDepth'] = unitMetrics[
'MaxDepth'] - unitMetrics['DepthAtThalwegExit']
unitMetrics['Length'] = cuThalwegLine.length
unitMetrics['ThalwegIntersect'] = 1
# Tier 1 and tier 2 topometrics. Note that metric dictionary keys are used for XML tags & require cleaning
tier1NameClean = getCleanTierName(tier1Name)
self._calcTierLevelMetrics(dResultsTier1[tier1NameClean],
tier1Name, unitMetrics, siteLength,
dResultsChannelSummary['Main']['Area'])
tier2NameClean = getCleanTierName(tier2Name)
self._calcTierLevelMetrics(dResultsTier2[tier2NameClean],
tier2Name, unitMetrics, siteLength,
dResultsChannelSummary['Main']['Area'])
# Calculate the average of the channel unit max depths for each tier 1 and tier 2 type
for tierKey, tierMetrics in {
'Tier1': dResultsTier1,
'Tier2': dResultsTier2
}.iteritems():
for tierName, metricDict in tierMetrics.iteritems():
maxDepthList = [
aResult['MaxDepth'] for aResult in resultsCU
if getCleanTierName(aResult[tierKey]) == tierName
]
if len(maxDepthList) > 0:
metricDict['AvgMaxDepth'] = np.average(maxDepthList)
# Convert the sum of residual depth and depth at thalweg exist
# to average residual depth for each tier 1 and tier 2 type
for tierMetricDict in [dResultsTier1, dResultsTier2]:
for tierName, tierMetrics in tierMetricDict.iteritems():
# channel unit types that don't occur should retain the value None for Residual Depth and Depth at Thalweg exit
if tierMetrics['Count'] > 0 and tierMetrics[
'ThalwegIntersectCount'] > 0:
for metricName in ['ResidualDepth', 'DepthAtThalwegExit']:
if tierMetrics[metricName] is not None and tierMetrics[
metricName] != 0:
tierMetrics[metricName] = tierMetrics[
metricName] / tierMetrics[
'ThalwegIntersectCount']
else:
tierMetrics[metricName] = 0
def validate(self):
results = super(CHaMP_Thalweg, self).validate()
validate_maxfeaturecount = ValidationResult(self.__class__.__name__, "MaxFeatureCount")
validate_wsedemExtent = ValidationResult(self.__class__.__name__, "WithinWSEDEMExtent")
validate_in_end_dist = ValidationResult(self.__class__.__name__, "InPointNearEnd")
validate_out_start_dist = ValidationResult(self.__class__.__name__, "OutPointNearStart")
validate_out_higher_inflow = ValidationResult(self.__class__.__name__, "StartPointLowerEndPoint")
validate_thalwegstartstopraster = ValidationResult(self.__class__.__name__, "ThalwegStartStopOnDEM")
if self.exists():
if len(self.features) > self.maxFeatureCount:
validate_maxfeaturecount.error("Number of features (" + str(len(self.features)) +
") exceeds the maximum number allowed ("
+ str(self.maxFeatureCount) + ")")
else:
validate_maxfeaturecount.pass_validation()
if self.wsedemExtent:
if self.start_stop_on_raster(raster_extent=self.wsedemExtent):
validate_wsedemExtent.pass_validation()
else:
validate_wsedemExtent.error("Thalweg not entirely contained within WSEDEM.")
if self.topo_in_point:
inbuffer = self.topo_in_point.buffer(15)
if inbuffer.contains(self.thalweg_end_pt()):
validate_in_end_dist.pass_validation()
else:
validate_in_end_dist.warning("End point is greater than 15m from Topo 'in' point")
if self.topo_out_point:
if self.topo_out_point.buffer(15).contains(self.thalweg_start_pnt()):
validate_out_start_dist.pass_validation()
else:
validate_out_start_dist.warning("Start point is greater than 15m from Topo 'out' point")
if self.dem:
if self.demDataExtent and self.features:
if self.start_stop_on_raster():
validate_thalwegstartstopraster.pass_validation()
else:
validate_thalwegstartstopraster.error("One or more line features does not start or stop on the DEM")
r = Raster(self.dem)
tStart = self.thalweg_start_pnt()
tEnd = self.thalweg_end_pt()
if tStart is None or tEnd is None:
validate_out_higher_inflow.error("Could not determine thalweg start and finish")
else:
z_start = r.getPixelVal([tStart.x, tStart.y])
z_end = r.getPixelVal([tEnd.x, tEnd.y])
if z_start > z_end + 0.1:
validate_out_higher_inflow.error("Thalweg Start (outflow) more than 10cm higher than end (inflow)")
else:
validate_out_higher_inflow.pass_validation()
results.append(validate_maxfeaturecount.get_dict())
results.append(validate_wsedemExtent.get_dict())
results.append(validate_in_end_dist.get_dict())
results.append(validate_out_start_dist.get_dict())
results.append(validate_out_higher_inflow.get_dict())
results.append(validate_thalwegstartstopraster.get_dict())
return results
def calc(self, crosssections, waterExtent, demPath, stationInterval):
# Save space by only loading the desired fields from the ShapeFile.
# We also need the 'Channel' and 'IsValid' fields if they exist.
desiredFields = CrossSectionMetrics.dMetricTypes.keys()
desiredFields.append('IsValid')
# Open the cross section ShapeFile & build a list of all features with a dictionary of the desired fields
clShp = Shapefile(crosssections)
lChannels = ['Main']
if not clShp.loaded:
return
if "Channel" in clShp.fields:
desiredFields.append('Channel')
lChannels.append('Side')
# Older Cross Section Layers don't have the width and depth fields calculated.
# So if all the necessary metric fields are present then continue to load
# ShapeFile features. Otherwise we need to calculate the topometrics from scratch
bMetricCalculationNeeded = False
for aMetric in desiredFields:
if not aMetric in clShp.fields:
bMetricCalculationNeeded = True
break
allFeatures = []
if bMetricCalculationNeeded:
# Retrieve the water extent polygon exterior
rivershp = Shapefile(waterExtent)
polyRiverShapeFeats = rivershp.featuresToShapely()
# Try and find a channel shape. Thers's a lot of variance here.
if len(polyRiverShapeFeats) == 0 or 'geometry' not in polyRiverShapeFeats[0]:
raise DataException("No features in crosssection shape file")
# If there's only one shape then just use it
elif len(polyRiverShapeFeats) == 1:
polyRiverShape = polyRiverShapeFeats[0]['geometry']
# If there's more than one shape then see if
else:
channelShapes = [feat['geometry'] for feat in polyRiverShapeFeats if feat['fields']['ExtentType'] == 'Channel']
if len(channelShapes) == 0:
raise DataException("No features in crosssection shape file")
polyRiverShape = channelShapes[0]
# Calculate the topometrics from scratch for a single cross section
shpXS = Shapefile(crosssections)
demRaster = Raster(demPath)
if shpXS.loaded:
for aFeat in shpXS.featuresToShapely():
# Calculate the topometrics for this cross section. They will be stored on the aFeat dict under key 'topometrics'
calcXSMetrics(aFeat, polyRiverShape , demRaster, stationInterval)
# Build the all features dictionary that would be expect had the topometrics already
# existed in the XS shapefile and simply got loaded. This is a combination of the new topometrics
# and also the existing fields on the XS ShapeFile.
singleXSMetrics = copy.deepcopy(aFeat['topometrics'])
singleXSMetrics.update(aFeat['fields'])
allFeatures.append(singleXSMetrics)
# Destroying the raster object appears to prevent warning messages on Windows
demRaster = None
else:
allFeatures = clShp.attributesToList(desiredFields)
# For simple ShapeFiles, make every feature part of the main channel, and
# set every feature as valid. This helps keep code below generic
for x in allFeatures:
if 'Channel' not in x:
x['Channel'] = 'Main'
if 'IsValid' not in x:
x['IsValid'] = 1
for channelName in lChannels:
# Filter the list of features to just those in this channel
# PGB - 24 Apr 2017 - observed NULL values in 'Channel' ShapeFile field in Harold results.
# Cast field contents to string to avoid crash here.
channelFeatures = [x for x in allFeatures if str(x['Channel']).lower() == channelName.lower()]
# Filter the list of features to just those that the crew considered valid
validFeatures = [x for x in channelFeatures if x['IsValid'] <> 0]
# Filter the features to just those with a length that is within 4 standard deviations of mean wetted width
channelStatistics = getStatistics(channelFeatures, 'WetWidth')
autoFeatures = None
if channelStatistics['StdDev'] is not None:
wetWidthThreshold = channelStatistics['StdDev'] * 4
autoFeatures = [x for x in channelFeatures if abs(x['WetWidth'] - channelStatistics['Mean']) < wetWidthThreshold]
# Loop over each desired metric and calculate the statistics for each filtering type
for metricName, bestFiltering in CrossSectionMetrics.dMetricTypes.iteritems():
populateChannelStatistics(self.metrics[channelName], 'None', metricName, channelFeatures)
populateChannelStatistics(self.metrics[channelName], 'Crew', metricName, validFeatures)
if channelStatistics['StdDev'] is not None:
populateChannelStatistics(self.metrics[channelName], 'Auto', metricName, autoFeatures)
self.metrics[channelName]['Best'] = self.metrics[channelName][bestFiltering]
# The topometrics for the whole channel are always the results for 'Main'.
# For complex ShapeFiles this will be just the results for the main channel.
# For simple, single threaded, ShapeFiles this will all cross sections.
self.metrics['Channel'] = self.metrics['Main']