def integrateMetricDictionaryWithTopLevelType(topo_metrics, prefix,
newCollection):
"""
Integrate a dictionary of metrics into an existing dictionary.
Note that this differs from the dictionary merging for other metric engines.
This version looks for
:param topo_metrics:
:param prefix:
:param newCollection:
:return:
"""
if 'VisitMetrics' in newCollection:
if not 'VisitMetrics' in topo_metrics:
topo_metrics['VisitMetrics'] = {}
integrateMetricDictionary(topo_metrics['VisitMetrics'], prefix,
newCollection['VisitMetrics'])
if 'Tier1Metrics' in newCollection:
if not 'Tier1Metrics' in topo_metrics:
topo_metrics['Tier1Metrics'] = {}
for t1Type in dUnitDefs:
safet1Type = getCleanTierName(t1Type)
if not safet1Type in topo_metrics['Tier1Metrics']:
topo_metrics['Tier1Metrics'][safet1Type] = {'Name': t1Type}
integrateMetricDictionary(
topo_metrics['Tier1Metrics'][safet1Type], prefix,
newCollection['Tier1Metrics'][safet1Type])
def calculateMetricsForTier1Summary(visitobj):
visitid = visitobj["visit_id"]
channelUnits = visitobj['channelUnits']
tier1Metrics = []
# create the tier 1 summary metric rows with the correct tier 1
if channelUnits is not None:
tier1s = [t["value"]["Tier1"] for t in channelUnits["values"]]
tier1s = list(set(tier1s)) # this is a quick distinct
for c in tier1s:
t = dict()
t["Tier1"] = c
populateDefaultColumns(t, visitid)
tier1Metrics.append(t)
tier1FishCountMetrics(tier1Metrics, visitobj)
tier1LWDMetrics(tier1Metrics, visitobj)
results = {}
for t1Metrics in tier1Metrics:
rawt1Name = t1Metrics['Tier1']
safet1Name = getCleanTierName(rawt1Name)
results[safet1Name] = t1Metrics
return results
def _templateMaker(initValue, unitDefinitions):
"""
This template object is super complex so we need a whole method just to build it
:param initValue:
:return:
"""
base = {
"resultsCU": [],
"ResultsTier1": {},
"ResultsTier2": {},
"ResultsChannelSummary": {}
}
vallist = ['Area', 'Volume', 'Count', 'Spacing', 'Percent', 'Frequency', 'ThalwegIntersectCount']
nonelist = ['ResidualDepth', 'DepthAtThalwegExit', 'MaxMaxDepth', 'AvgMaxDepth']
if not unitDefinitions:
unitDefinitions = dUnitDefs
# Dictionaries for tier 1 and tier 2 results
# These require area and volume and by creating them here there will always be values for all types.
for tier1Name in unitDefinitions:
base['ResultsTier1'][getCleanTierName(tier1Name)] = {}
base['ResultsTier1'][getCleanTierName(tier1Name)]['Name'] = tier1Name
for val in vallist:
base['ResultsTier1'][getCleanTierName(tier1Name)][val] = initValue
for val in nonelist:
base['ResultsTier1'][getCleanTierName(tier1Name)][val] = None
for tier2Name in unitDefinitions[tier1Name]:
base['ResultsTier2'][getCleanTierName(tier2Name)] = {}
base['ResultsTier2'][getCleanTierName(tier2Name)]['Name'] = tier2Name
for val in vallist:
base['ResultsTier2'][getCleanTierName(tier2Name)][val] = initValue
for val in nonelist:
base['ResultsTier2'][getCleanTierName(tier2Name)][val] = None
# Dictionary for the side channel summary topometrics
base['ResultsChannelSummary'] = {
'Main': {'Area': initValue },
'SmallSideChannel': {'Area': initValue, 'Count': initValue, 'Percent': initValue, 'Volume': initValue},
'LargeSideChannel': {'Area': initValue, 'Count': initValue, 'Percent': initValue, 'Volume': initValue},
'SideChannelSummary': {'Area': initValue, 'Count': initValue, 'Percent': initValue, 'Volume': initValue},
'ChannelUnitBreakdown': {'SmallSideChannel': initValue, 'Other': initValue}
}
return base
def emptiesByChannelUnit(metricDict):
"""
Instantiate an "Empty" object full of null values in case our calculation fails
:param metricDict:
:return:
"""
cleantypes = [getCleanTierName(t1Type) for t1Type in dUnitDefs]
cuMetrics = {t1type: {} for t1type in cleantypes}
for t1Name, t1Obj in cuMetrics.items():
for metricName in metricDict.iterkeys():
# The Tier 1 types from the API need to be sanitized for the metric XML file
t1Obj[metricName] = None
t1Obj['Total'] = None
visitMetrics = {metricName: None for metricName in metricDict.iterkeys()}
visitMetrics['Total'] = None
dResults = {'VisitMetrics': visitMetrics, 'Tier1Metrics': cuMetrics}
return dResults
def metricsByChannelUnit(metricDict, channelUnitMetrics, apiValues,
channelUnitMeasurements):
# Retrieve channel unit areas and tier 1 type from channel unit metrics
dChannelUnits = {}
for unit in channelUnitMetrics:
unitNumber = int(unit['ChUnitNumber'])
unitID = next(u['value']['ChannelUnitID']
for u in channelUnitMeasurements['values']
if u['value']['ChannelUnitNumber'] == unitNumber)
dChannelUnits[unitNumber] = {}
dChannelUnits[unitNumber]['Area'] = unit['AreaTotal']
dChannelUnits[unitNumber]['Tier1'] = unit['Tier1']
# Loop over each metric.
# See the metric definitions dict in each calling function.
# Some metrics group several measurement classes into one metric
for metricName, subClasses in metricDict.items():
dChannelUnits[unitNumber][metricName] = 0.0
# Loop over the API measurements that are used by this metric
for subClass in subClasses[0]:
# Sum the proportions of the measurement for this class
vals = [
val[subClass] for val in apiValues
if val['ChannelUnitID'] == unitID and val[subClass] != None
]
if len(vals) == 0:
dChannelUnits[unitNumber][metricName] = None
else:
if dChannelUnits[unitNumber][metricName] is None:
dChannelUnits[unitNumber][metricName] = 0
dChannelUnits[unitNumber][metricName] += np.sum(vals)
cuMetrics = {}
for t1Type in dUnitDefs:
# The Tier 1 types from the API need to be sanitized for the metric XML file
safet1Type = getCleanTierName(t1Type)
cuMetrics[safet1Type] = {}
tier1Total = 0.0
# Set a template where eveyrthing is zero
cuMetrics[safet1Type]['Total'] = None
for metricName, subClasses in metricDict.items():
cuMetrics[safet1Type][metricName] = None
t1AreaTot = np.sum([
val['Area'] for val in dChannelUnits.values()
if val['Tier1'] == t1Type
])
cuMetrics[safet1Type]['Area'] = t1AreaTot
# Only do this if we have some area to work with. Otherwise we'll be dividing by zero
if t1AreaTot > 0:
for metricName, subClasses in metricDict.items():
vals = [
val[metricName] * val['Area']
for val in dChannelUnits.values()
if val[metricName] is not None and val['Tier1'] == t1Type
]
if len(vals) > 0:
# Sum product of substrate class proportions for a specific tier 1 type
t1SumProd = np.sum(vals)
cuMetrics[safet1Type][metricName] = t1SumProd / t1AreaTot
if subClasses[1] and cuMetrics[safet1Type][metricName]:
tier1Total += cuMetrics[safet1Type][metricName]
cuMetrics[safet1Type]['Total'] = tier1Total
visitMetrics = {}
for metricName, subClasses in metricDict.items():
visitMetrics[metricName] = None
visitMetrics['Total'] = None
# Throw areas into XML
areaTot = 0
for val in dChannelUnits.values():
if val['Tier1'] != "Small Side Channel":
areaTot += val['Area']
visitMetrics['TotalArea'] = areaTot
metricTotal = 0.0
for metricName, subClasses in metricDict.items():
# Sum product of metric value proportions and areas, divided by sum area for all channel units
vals = [
val[metricName] * val['Area'] for val in dChannelUnits.values()
if val[metricName] is not None
and val['Tier1'] != "Small Side Channel"
]
if len(vals) > 0:
visitMetrics[metricName] = np.sum(vals) / areaTot
# Only include the metric in the overall total if the argument tuple indicates that the metric should be included
if subClasses[1]:
metricTotal += visitMetrics[metricName]
visitMetrics['Total'] = metricTotal
dResults = {'VisitMetrics': visitMetrics, 'Tier1Metrics': cuMetrics}
return dResults
def calc(self, shpCUPath, shpThalweg, rasDepth, visitMetrics, dUnits, unitDefs):
if not path.isfile(shpCUPath):
raise MissingException("Channel units file not found")
if not path.isfile(shpThalweg):
raise MissingException("Thalweg shape file not found")
if not path.isfile(rasDepth):
raise MissingException("Depth raster file not found")
siteLength = visitMetrics['Wetted']['Centerline']['MainstemLength']
if siteLength is None:
raise DataException("No valid site length found in visit metrics")
# Give us a fresh template with 0's in the value positions
self.metrics = self._templateMaker(0, unitDefs)
dResultsChannelSummary = self.metrics['ResultsChannelSummary']
dResultsTier1 = self.metrics['ResultsTier1']
dResultsTier2 = self.metrics['ResultsTier2']
resultsCU = self.metrics['resultsCU']
#Load the Thalweg feature
thalweg = Shapefile(shpThalweg).featuresToShapely()
thalwegLine = thalweg[0]['geometry']
# Load the depth raster
depthRaster = Raster(rasDepth)
# Load the channel unit polygons and calculate the total area
# The channel units should be clipped to the wetted extent and so this
# can be used as the site area wetted
shpCU = Shapefile(shpCUPath)
arrCU = depthRaster.rasterMaskLayer(shpCUPath, "UnitNumber")
feats = shpCU.featuresToShapely()
for aFeat in feats:
dResultsChannelSummary['Main']['Area'] += aFeat['geometry'].area
# Loop over each channel unit and calculate topometrics
for aFeat in feats:
nCUNumber = int(aFeat['fields']['UnitNumber'])
if nCUNumber not in dUnits:
self.log.error("Channel Unit: '{0}' not present in the aux data.".format(nCUNumber))
# Keep it general for the exception so we can aggregate them
raise DataException("The Channel Unit ShapeFile contains a unit number that is not present in the aux data.")
tier1Name = dUnits[nCUNumber][0]
tier2Name = dUnits[nCUNumber][1]
nSegment = dUnits[nCUNumber][2]
#print("Channel Unit Number {0}, Segment {1}, Tier 1 - {2}, Tier 2 - {3}").format(nCUNumber, nSegment, tier1Name, tier2Name)
unitMetrics = {}
resultsCU.append(unitMetrics)
unitMetrics['ChannelUnitNumber'] = nCUNumber
unitMetrics['Area'] = aFeat['geometry'].area
unitMetrics['Tier1'] = tier1Name
unitMetrics['Tier2'] = tier2Name
unitMetrics['Length'] = None
unitMetrics['ResidualDepth'] = None
unitMetrics['DepthAtThalwegExit'] = None
unitMetrics['ThalwegIntersect'] = 0
# Get the depth raster for this unit as variable so we can check
# whether it is entirely masked below.
depArr = depthRaster.array[arrCU == nCUNumber]
if depArr.count() == 0:
unitMetrics['MaxDepth'] = 0
unitMetrics['Volume'] = 0
else:
unitMetrics['MaxDepth'] = np.max(depArr)
unitMetrics['Volume'] = np.sum(depthRaster.array[arrCU == nCUNumber]) * (depthRaster.cellWidth**2)
if nSegment != 1:
dSideChannelSummary = dResultsChannelSummary['SideChannelSummary']
dMain = dResultsChannelSummary['Main']
# Side channel summary captures both small and large side channels
dSideChannelSummary['Area'] += aFeat['geometry'].area
dSideChannelSummary['Count'] += 1
dSideChannelSummary['Percent'] = 100 * dSideChannelSummary['Area'] / dMain['Area']
dSideChannelSummary['Volume'] += unitMetrics['Volume']
if 'side' in tier1Name.lower():
dSmallSideChannel = dResultsChannelSummary['SmallSideChannel']
dSmallSideChannel['Area'] += aFeat['geometry'].area
dSmallSideChannel['Count'] += 1
dSmallSideChannel['Percent'] = 100 * dSmallSideChannel['Area'] / dMain['Area']
dSmallSideChannel['Volume'] += unitMetrics['Volume']
else:
dLargeSideChannel = dResultsChannelSummary['LargeSideChannel']
dLargeSideChannel['Area'] += aFeat['geometry'].area
dLargeSideChannel['Count'] += 1
dLargeSideChannel['Percent'] = 100 * dLargeSideChannel['Area'] / dMain['Area']
dLargeSideChannel['Volume'] += unitMetrics['Volume']
if tier1Name is None:
raise DataException("tier1Name cannot be 'None'")
if 'side' in tier1Name.lower():
dResultsChannelSummary['ChannelUnitBreakdown']['SmallSideChannel'] += 1
else:
dResultsChannelSummary['ChannelUnitBreakdown']['Other'] += 1
if (thalwegLine.intersects(aFeat['geometry'])):
cuThalwegLine = thalwegLine.intersection(aFeat['geometry'])
exitPoint = None
if cuThalwegLine.type == 'LineString':
exitPoint = cuThalwegLine.coords[0]
else:
exitPoint = cuThalwegLine[0].coords[0]
# Retrieve a list of points along the Thalweg in the channel unit
thalwegPoints = ChannelUnitMetrics.interpolatePointsAlongLine(cuThalwegLine, 0.13)
thalwegDepths = ChannelUnitMetrics.lookupRasterValuesAtPoints(thalwegPoints, depthRaster)
unitMetrics['MaxDepth'] = np.nanmax(thalwegDepths['values'])
unitMetrics['DepthAtThalwegExit'] = depthRaster.getPixelVal(exitPoint)
unitMetrics['ResidualDepth'] = unitMetrics['MaxDepth'] - unitMetrics['DepthAtThalwegExit']
unitMetrics['Length'] = cuThalwegLine.length
unitMetrics['ThalwegIntersect'] = 1
# Tier 1 and tier 2 topometrics. Note that metric dictionary keys are used for XML tags & require cleaning
tier1NameClean = getCleanTierName(tier1Name)
self._calcTierLevelMetrics(dResultsTier1[tier1NameClean], tier1Name, unitMetrics, siteLength, dResultsChannelSummary['Main']['Area'])
tier2NameClean = getCleanTierName(tier2Name)
self._calcTierLevelMetrics(dResultsTier2[tier2NameClean], tier2Name, unitMetrics, siteLength, dResultsChannelSummary['Main']['Area'])
# Calculate the average of the channel unit max depths for each tier 1 and tier 2 type
for tierKey, tierMetrics in { 'Tier1': dResultsTier1, 'Tier2': dResultsTier2}.items():
for tierName, metricDict in tierMetrics.items():
maxDepthList = [aResult['MaxDepth'] for aResult in resultsCU if getCleanTierName(aResult[tierKey]) == tierName]
if len(maxDepthList) > 0:
metricDict['AvgMaxDepth'] = np.average(maxDepthList)
# Convert the sum of residual depth and depth at thalweg exist
# to average residual depth for each tier 1 and tier 2 type
for tierMetricDict in [dResultsTier1, dResultsTier2]:
for tierName, tierMetrics in tierMetricDict.items():
# channel unit types that don't occur should retain the value None for Residual Depth and Depth at Thalweg exit
if tierMetrics['Count'] > 0 and tierMetrics['ThalwegIntersectCount'] > 0:
for metricName in ['ResidualDepth', 'DepthAtThalwegExit']:
if tierMetrics[metricName] is not None and tierMetrics[metricName] != 0:
tierMetrics[metricName] = tierMetrics[metricName] / tierMetrics['ThalwegIntersectCount']
else:
tierMetrics[metricName] = 0