def integrateMetricDictionaryWithTopLevelType(topo_metrics, prefix, newCollection): """ Integrate a dictionary of metrics into an existing dictionary. Note that this differs from the dictionary merging for other metric engines. This version looks for :param topo_metrics: :param prefix: :param newCollection: :return: """ if 'VisitMetrics' in newCollection: if not 'VisitMetrics' in topo_metrics: topo_metrics['VisitMetrics'] = {} integrateMetricDictionary(topo_metrics['VisitMetrics'], prefix, newCollection['VisitMetrics']) if 'Tier1Metrics' in newCollection: if not 'Tier1Metrics' in topo_metrics: topo_metrics['Tier1Metrics'] = {} for t1Type in dUnitDefs: safet1Type = getCleanTierName(t1Type) if not safet1Type in topo_metrics['Tier1Metrics']: topo_metrics['Tier1Metrics'][safet1Type] = {'Name' : t1Type} integrateMetricDictionary(topo_metrics['Tier1Metrics'][safet1Type], prefix, newCollection['Tier1Metrics'][safet1Type])
def calculateMetricsForTier1Summary(visitobj): visitid = visitobj["visit_id"] channelUnits = visitobj['channelUnits'] tier1Metrics = [] # create the tier 1 summary metric rows with the correct tier 1 if channelUnits is not None: tier1s = [t["value"]["Tier1"] for t in channelUnits["values"]] tier1s = list(set(tier1s)) # this is a quick distinct for c in tier1s: t = dict() t["Tier1"] = c populateDefaultColumns(t, visitid) tier1Metrics.append(t) tier1FishCountMetrics(tier1Metrics, visitobj) tier1LWDMetrics(tier1Metrics, visitobj) results = {} for t1Metrics in tier1Metrics: rawt1Name = t1Metrics['Tier1'] safet1Name = getCleanTierName(rawt1Name) results[safet1Name] = t1Metrics return results
def emptiesByChannelUnit(metricDict): """ Instantiate an "Empty" object full of null values in case our calculation fails :param metricDict: :return: """ cleantypes = [getCleanTierName(t1Type) for t1Type in dUnitDefs] cuMetrics = {t1type: {} for t1type in cleantypes} for t1Name, t1Obj in cuMetrics.iteritems(): for metricName in metricDict.iterkeys(): # The Tier 1 types from the API need to be sanitized for the metric XML file t1Obj[metricName] = None t1Obj['Total'] = None visitMetrics = {metricName: None for metricName in metricDict.iterkeys()} visitMetrics['Total'] = None dResults = {'VisitMetrics': visitMetrics, 'Tier1Metrics': cuMetrics} return dResults
def metricsByChannelUnit(metricDict, channelUnitMetrics, apiValues, channelUnitMeasurements): # Retrieve channel unit areas and tier 1 type from channel unit metrics dChannelUnits = {} for unit in channelUnitMetrics: unitNumber = int(unit['ChUnitNumber']) unitID = next(u['value']['ChannelUnitID'] for u in channelUnitMeasurements['values'] if u['value']['ChannelUnitNumber'] == unitNumber) dChannelUnits[unitNumber] = {} dChannelUnits[unitNumber]['Area'] = unit['AreaTotal'] dChannelUnits[unitNumber]['Tier1'] = unit['Tier1'] # Loop over each metric. # See the metric definitions dict in each calling function. # Some metrics group several measurement classes into one metric for metricName, subClasses in metricDict.iteritems(): dChannelUnits[unitNumber][metricName] = 0.0 # Loop over the API measurements that are used by this metric for subClass in subClasses[0]: # Sum the proportions of the measurement for this class vals = [ val[subClass] for val in apiValues if val['ChannelUnitID'] == unitID and val[subClass] != None ] if len(vals) == 0: dChannelUnits[unitNumber][metricName] = None else: if dChannelUnits[unitNumber][metricName] is None: dChannelUnits[unitNumber][metricName] = 0 dChannelUnits[unitNumber][metricName] += np.sum(vals) cuMetrics = {} for t1Type in dUnitDefs: # The Tier 1 types from the API need to be sanitized for the metric XML file safet1Type = getCleanTierName(t1Type) cuMetrics[safet1Type] = {} tier1Total = 0.0 # Set a template where eveyrthing is zero cuMetrics[safet1Type]['Total'] = None for metricName, subClasses in metricDict.iteritems(): cuMetrics[safet1Type][metricName] = None t1AreaTot = np.sum([ val['Area'] for val in dChannelUnits.values() if val['Tier1'] == t1Type ]) cuMetrics[safet1Type]['Area'] = t1AreaTot # Only do this if we have some area to work with. Otherwise we'll be dividing by zero if t1AreaTot > 0: for metricName, subClasses in metricDict.iteritems(): vals = [ val[metricName] * val['Area'] for val in dChannelUnits.values() if val[metricName] is not None and val['Tier1'] == t1Type ] if len(vals) > 0: # Sum product of substrate class proportions for a specific tier 1 type t1SumProd = np.sum(vals) cuMetrics[safet1Type][metricName] = t1SumProd / t1AreaTot if subClasses[1] and cuMetrics[safet1Type][metricName]: tier1Total += cuMetrics[safet1Type][metricName] cuMetrics[safet1Type]['Total'] = tier1Total visitMetrics = {} for metricName, subClasses in metricDict.iteritems(): visitMetrics[metricName] = None visitMetrics['Total'] = None # Throw areas into XML areaTot = 0 for val in dChannelUnits.values(): if val['Tier1'] != "Small Side Channel": areaTot += val['Area'] visitMetrics['TotalArea'] = areaTot metricTotal = 0.0 for metricName, subClasses in metricDict.iteritems(): # Sum product of metric value proportions and areas, divided by sum area for all channel units vals = [ val[metricName] * val['Area'] for val in dChannelUnits.values() if val[metricName] is not None and val['Tier1'] != "Small Side Channel" ] if len(vals) > 0: visitMetrics[metricName] = np.sum(vals) / areaTot # Only include the metric in the overall total if the argument tuple indicates that the metric should be included if subClasses[1]: metricTotal += visitMetrics[metricName] visitMetrics['Total'] = metricTotal dResults = {'VisitMetrics': visitMetrics, 'Tier1Metrics': cuMetrics} return dResults
def _templateMaker(initValue, unitDefinitions): """ This template object is super complex so we need a whole method just to build it :param initValue: :return: """ base = { "resultsCU": [], "ResultsTier1": {}, "ResultsTier2": {}, "ResultsChannelSummary": {} } vallist = [ 'Area', 'Volume', 'Count', 'Spacing', 'Percent', 'Frequency', 'ThalwegIntersectCount' ] nonelist = [ 'ResidualDepth', 'DepthAtThalwegExit', 'MaxMaxDepth', 'AvgMaxDepth' ] if not unitDefinitions: unitDefinitions = dUnitDefs # Dictionaries for tier 1 and tier 2 results # These require area and volume and by creating them here there will always be values for all types. for tier1Name in unitDefinitions: base['ResultsTier1'][getCleanTierName(tier1Name)] = {} base['ResultsTier1'][getCleanTierName( tier1Name)]['Name'] = tier1Name for val in vallist: base['ResultsTier1'][getCleanTierName( tier1Name)][val] = initValue for val in nonelist: base['ResultsTier1'][getCleanTierName(tier1Name)][val] = None for tier2Name in unitDefinitions[tier1Name]: base['ResultsTier2'][getCleanTierName(tier2Name)] = {} base['ResultsTier2'][getCleanTierName( tier2Name)]['Name'] = tier2Name for val in vallist: base['ResultsTier2'][getCleanTierName( tier2Name)][val] = initValue for val in nonelist: base['ResultsTier2'][getCleanTierName( tier2Name)][val] = None # Dictionary for the side channel summary topometrics base['ResultsChannelSummary'] = { 'Main': { 'Area': initValue }, 'SmallSideChannel': { 'Area': initValue, 'Count': initValue, 'Percent': initValue, 'Volume': initValue }, 'LargeSideChannel': { 'Area': initValue, 'Count': initValue, 'Percent': initValue, 'Volume': initValue }, 'SideChannelSummary': { 'Area': initValue, 'Count': initValue, 'Percent': initValue, 'Volume': initValue }, 'ChannelUnitBreakdown': { 'SmallSideChannel': initValue, 'Other': initValue } } return base
def calc(self, shpCUPath, shpThalweg, rasDepth, visitMetrics, dUnits, unitDefs): if not os.path.isfile(shpCUPath): raise MissingException("Channel units file not found") if not os.path.isfile(shpThalweg): raise MissingException("Thalweg shape file not found") if not os.path.isfile(rasDepth): raise MissingException("Depth raster file not found") siteLength = visitMetrics['Wetted']['Centerline']['MainstemLength'] if siteLength is None: raise DataException("No valid site length found in visit metrics") # Give us a fresh template with 0's in the value positions self.metrics = self._templateMaker(0, unitDefs) dResultsChannelSummary = self.metrics['ResultsChannelSummary'] dResultsTier1 = self.metrics['ResultsTier1'] dResultsTier2 = self.metrics['ResultsTier2'] resultsCU = self.metrics['resultsCU'] #Load the Thalweg feature thalweg = Shapefile(shpThalweg).featuresToShapely() thalwegLine = thalweg[0]['geometry'] # Load the depth raster depthRaster = Raster(rasDepth) # Load the channel unit polygons and calculate the total area # The channel units should be clipped to the wetted extent and so this # can be used as the site area wetted shpCU = Shapefile(shpCUPath) arrCU = depthRaster.rasterMaskLayer(shpCUPath, "UnitNumber") feats = shpCU.featuresToShapely() for aFeat in feats: dResultsChannelSummary['Main']['Area'] += aFeat['geometry'].area # Loop over each channel unit and calculate topometrics for aFeat in feats: nCUNumber = int(aFeat['fields']['UnitNumber']) if nCUNumber not in dUnits: self.log.error( "Channel Unit: '{0}' not present in the aux data.".format( nCUNumber)) # Keep it general for the exception so we can aggregate them raise DataException( "The Channel Unit ShapeFile contains a unit number that is not present in the aux data." ) tier1Name = dUnits[nCUNumber][0] tier2Name = dUnits[nCUNumber][1] nSegment = dUnits[nCUNumber][2] #print "Channel Unit Number {0}, Segment {1}, Tier 1 - {2}, Tier 2 - {3}".format(nCUNumber, nSegment, tier1Name, tier2Name) unitMetrics = {} resultsCU.append(unitMetrics) unitMetrics['ChannelUnitNumber'] = nCUNumber unitMetrics['Area'] = aFeat['geometry'].area unitMetrics['Tier1'] = tier1Name unitMetrics['Tier2'] = tier2Name unitMetrics['Length'] = None unitMetrics['ResidualDepth'] = None unitMetrics['DepthAtThalwegExit'] = None unitMetrics['ThalwegIntersect'] = 0 # Get the depth raster for this unit as variable so we can check # whether it is entirely masked below. depArr = depthRaster.array[arrCU == nCUNumber] if depArr.count() == 0: unitMetrics['MaxDepth'] = 0 unitMetrics['Volume'] = 0 else: unitMetrics['MaxDepth'] = np.max(depArr) unitMetrics['Volume'] = np.sum(depthRaster.array[ arrCU == nCUNumber]) * (depthRaster.cellWidth**2) if nSegment != 1: dSideChannelSummary = dResultsChannelSummary[ 'SideChannelSummary'] dMain = dResultsChannelSummary['Main'] # Side channel summary captures both small and large side channels dSideChannelSummary['Area'] += aFeat['geometry'].area dSideChannelSummary['Count'] += 1 dSideChannelSummary['Percent'] = 100 * dSideChannelSummary[ 'Area'] / dMain['Area'] dSideChannelSummary['Volume'] += unitMetrics['Volume'] if 'side' in tier1Name.lower(): dSmallSideChannel = dResultsChannelSummary[ 'SmallSideChannel'] dSmallSideChannel['Area'] += aFeat['geometry'].area dSmallSideChannel['Count'] += 1 dSmallSideChannel['Percent'] = 100 * dSmallSideChannel[ 'Area'] / dMain['Area'] dSmallSideChannel['Volume'] += unitMetrics['Volume'] else: dLargeSideChannel = dResultsChannelSummary[ 'LargeSideChannel'] dLargeSideChannel['Area'] += aFeat['geometry'].area dLargeSideChannel['Count'] += 1 dLargeSideChannel['Percent'] = 100 * dLargeSideChannel[ 'Area'] / dMain['Area'] dLargeSideChannel['Volume'] += unitMetrics['Volume'] if tier1Name is None: raise DataException("tier1Name cannot be 'None'") if 'side' in tier1Name.lower(): dResultsChannelSummary['ChannelUnitBreakdown'][ 'SmallSideChannel'] += 1 else: dResultsChannelSummary['ChannelUnitBreakdown']['Other'] += 1 if (thalwegLine.intersects(aFeat['geometry'])): cuThalwegLine = thalwegLine.intersection(aFeat['geometry']) exitPoint = None if cuThalwegLine.type == 'LineString': exitPoint = cuThalwegLine.coords[0] else: exitPoint = cuThalwegLine[0].coords[0] # Retrieve a list of points along the Thalweg in the channel unit thalwegPoints = ChannelUnitMetrics.interpolatePointsAlongLine( cuThalwegLine, 0.13) thalwegDepths = ChannelUnitMetrics.lookupRasterValuesAtPoints( thalwegPoints, depthRaster) unitMetrics['MaxDepth'] = np.nanmax(thalwegDepths['values']) unitMetrics['DepthAtThalwegExit'] = depthRaster.getPixelVal( exitPoint) unitMetrics['ResidualDepth'] = unitMetrics[ 'MaxDepth'] - unitMetrics['DepthAtThalwegExit'] unitMetrics['Length'] = cuThalwegLine.length unitMetrics['ThalwegIntersect'] = 1 # Tier 1 and tier 2 topometrics. Note that metric dictionary keys are used for XML tags & require cleaning tier1NameClean = getCleanTierName(tier1Name) self._calcTierLevelMetrics(dResultsTier1[tier1NameClean], tier1Name, unitMetrics, siteLength, dResultsChannelSummary['Main']['Area']) tier2NameClean = getCleanTierName(tier2Name) self._calcTierLevelMetrics(dResultsTier2[tier2NameClean], tier2Name, unitMetrics, siteLength, dResultsChannelSummary['Main']['Area']) # Calculate the average of the channel unit max depths for each tier 1 and tier 2 type for tierKey, tierMetrics in { 'Tier1': dResultsTier1, 'Tier2': dResultsTier2 }.iteritems(): for tierName, metricDict in tierMetrics.iteritems(): maxDepthList = [ aResult['MaxDepth'] for aResult in resultsCU if getCleanTierName(aResult[tierKey]) == tierName ] if len(maxDepthList) > 0: metricDict['AvgMaxDepth'] = np.average(maxDepthList) # Convert the sum of residual depth and depth at thalweg exist # to average residual depth for each tier 1 and tier 2 type for tierMetricDict in [dResultsTier1, dResultsTier2]: for tierName, tierMetrics in tierMetricDict.iteritems(): # channel unit types that don't occur should retain the value None for Residual Depth and Depth at Thalweg exit if tierMetrics['Count'] > 0 and tierMetrics[ 'ThalwegIntersectCount'] > 0: for metricName in ['ResidualDepth', 'DepthAtThalwegExit']: if tierMetrics[metricName] is not None and tierMetrics[ metricName] != 0: tierMetrics[metricName] = tierMetrics[ metricName] / tierMetrics[ 'ThalwegIntersectCount'] else: tierMetrics[metricName] = 0