def test_real(self): from lib.raster import Raster, PrintArr from lib.shapefileloader import Shapefile r = Raster("D:/CHaMP/Harold/2014/Entiat/ENT00001-1BC1/VISIT_2447/Topo/GISLayers/DEM.tif") theArr = r.rasterMaskLayer("D:/CHaMP/Harold/2014/Entiat/ENT00001-1BC1/VISIT_2447/Topo/GISLayers/Channel_Units.shp", 'Unit_Numbe') PrintArr(theArr)
def rasterToCSV(rasterPath, outputCSVPath): print "Writing raster {0} to {1}".format(os.path.basename(rasterPath), outputCSVPath) raster = Raster(rasterPath) npArr = raster.array # Retrieve and return the extent extent = {} extent['Top'] = raster.top extent['Left'] = raster.left extent['Right'] = raster.getRight() extent['Bottom'] = raster.getBottom() writer = csv.writer(open(outputCSVPath, 'wb')) header = ['x', 'y', 'value'] writer.writerow(header) for i, iVal in enumerate(npArr): for j, jVal in enumerate(iVal): if not np.ma.is_masked(jVal): x = raster.left + (j * raster.cellWidth) + (raster.cellWidth / 2) y = raster.top + (i * raster.cellHeight) + (raster.cellHeight / 2) writer.writerow([x, y, raster.array[i][j]]) return extent
def get_z_on_dem(self, filterlist=None): feats = self.features if feats and self.dem: r = Raster(self.dem) if filterlist: feats = [feat for feat in feats if feat['fields'][self.fieldName_Description] in filterlist] feats = [feat for feat in feats for poly in self.demDataExtent if poly.contains(feat['geometry'])] list_z = [r.getPixelVal([feat['geometry'].x, feat['geometry'].y]) for feat in feats] return list_z else: return []
def RasterMetrics(rpath): result = {'StDev': None} theRaster = Raster(rpath) result['StDev'] = np.std(theRaster.array) return result
def calc(self, channelName, shpWaterExtent, shpCenterline, rasDepth): if not os.path.isfile(shpCenterline): raise MissingException("Missing centerline shapefile") if not os.path.isfile(shpWaterExtent): raise MissingException("Missing water extent shapefile") if not os.path.isfile(rasDepth): raise MissingException("Missing depth raster") # Load the depth raster depthRaster = Raster(rasDepth) # Load the water extent ShapeFile and sum the area of all features shpExtent = Shapefile(shpWaterExtent) feats = shpExtent.featuresToShapely() arrMask = depthRaster.rasterMaskLayer(shpWaterExtent) self.metrics['Area'] = 0 # TODO: See issue #79 and #15. We need to use a different strategy if we want volume for bankfull https://github.com/SouthForkResearch/CHaMP_Metrics/issues/79 if channelName == "Bankfull": self.metrics['Volume'] = None for aFeat in feats: self.metrics['Area'] += aFeat['geometry'].area else: self.metrics['Volume'] = np.sum( depthRaster.array[arrMask > 0]) * (depthRaster.cellWidth**2) for aFeat in feats: self.metrics['Area'] += aFeat['geometry'].area dDepth = {} dDepth['Max'] = float(np.amax(depthRaster.array[arrMask > 0])) dDepth['Mean'] = np.mean(depthRaster.array[arrMask > 0]) self.metrics['Depth'] = dDepth # Retrieve the centerline mainstem. If the ShapeFile possesses a 'Channel field then # find the 1 feature with a value of 'Main', otherwise this is simply the only ShapeFile feature mainstem = self._getMainstemGeometry(shpCenterline) self.metrics['IntegratedWidth'] = None if mainstem and mainstem.length > 0: self.metrics[ 'IntegratedWidth'] = self.metrics['Area'] / mainstem.length
def getDEMExtents(projects): outerExtent = {'Top': None, 'Left': None, 'Right': None, 'Bottom': None} extents = {} for proj in projects: dempath = proj['project'].getpath('DEM') vid = proj['visit'] dem = Raster(dempath) extents[vid] = { 'Top': dem.top, 'Left': dem.left, 'Right': dem.getRight(), 'Bottom': dem.getBottom() } if outerExtent['Top']: outerExtent['Top'] = max(dem.top, outerExtent['Top']) else: outerExtent['Top'] = dem.top if outerExtent['Left']: outerExtent['Left'] = min(dem.left, outerExtent['Left']) else: outerExtent['Left'] = dem.left if outerExtent['Right']: outerExtent['Right'] = max(dem.getRight(), outerExtent['Right']) else: outerExtent['Right'] = dem.getRight() if outerExtent['Bottom']: outerExtent['Bottom'] = min(dem.getBottom(), outerExtent['Bottom']) else: outerExtent['Bottom'] = dem.getBottom() extents['OuterExtent'] = copy.deepcopy(outerExtent) return extents
def calc(self, sThalwegshp, sDepthRaster, sWaterSurfaceRaster, fDist, visitMetrics): if not path.isfile(sThalwegshp): raise MissingException("Thalweg shapefile missing") if not path.isfile(sDepthRaster): raise MissingException("Depth raster missing") if not path.isfile(sWaterSurfaceRaster): raise MissingException("Surface raster missing") wettedMainstemLength = visitMetrics['Wetted']['Centerline'][ 'MainstemLength'] if wettedMainstemLength is None: raise MissingException( "No wetted mainstem length found in visit metrics") sfile = Shapefile(sThalwegshp).featuresToShapely() if len(sfile) < 1: raise DataException("Thalweg shapefile has no features") thalweg = sfile[0]['geometry'] depthRaster = Raster(sDepthRaster) waterSurfaceRaster = Raster(sWaterSurfaceRaster) samplepts = ThalwegMetrics.interpolateRasterAlongLine(thalweg, fDist) results = ThalwegMetrics.lookupRasterValues(samplepts, depthRaster)['values'] # Get the elevation at the first (downstream) point on the Thalweg dsElev = waterSurfaceRaster.getPixelVal(thalweg.coords[0]) usElev = waterSurfaceRaster.getPixelVal(thalweg.coords[-1]) if (np.isnan(dsElev)): raise DataException( 'nodata detected in the raster for downstream point on the thalweg' ) elif np.isnan(usElev): raise DataException( 'nodata detected in the raster for upstream point on the thalweg' ) waterSurfaceGradientRatio = (usElev - dsElev) / thalweg.length waterSurfaceGradientPC = waterSurfaceGradientRatio * 100.0 # Thalweg straight length and sinuosity firstPoint = Point(thalweg.coords[0]) lastPoint = Point(thalweg.coords[-1]) straightLength = firstPoint.distance(lastPoint) sinuosity = thalweg.length / straightLength self.metrics = { 'Min': np.nanmin(results), 'Max': np.nanmax(results), 'Mean': np.mean(results), 'StDev': np.std(results), 'Count': np.count_nonzero(results), 'Length': thalweg.length, 'WSGradientRatio': waterSurfaceGradientRatio, 'WSGradientPC': waterSurfaceGradientPC, 'Sinuosity': sinuosity, 'CV': 0.0, 'ThalwegToCenterlineRatio': thalweg.length / wettedMainstemLength #, 'Values': results.data } if self.metrics['StDev'] != 0 and self.metrics['Mean'] != 0: self.metrics['CV'] = self.metrics['StDev'] / self.metrics['Mean']
def get_raster(self): return Raster(self.filename)
def calc(self, shpCUPath, shpThalweg, rasDepth, visitMetrics, dUnits, unitDefs): if not os.path.isfile(shpCUPath): raise MissingException("Channel units file not found") if not os.path.isfile(shpThalweg): raise MissingException("Thalweg shape file not found") if not os.path.isfile(rasDepth): raise MissingException("Depth raster file not found") siteLength = visitMetrics['Wetted']['Centerline']['MainstemLength'] if siteLength is None: raise DataException("No valid site length found in visit metrics") # Give us a fresh template with 0's in the value positions self.metrics = self._templateMaker(0, unitDefs) dResultsChannelSummary = self.metrics['ResultsChannelSummary'] dResultsTier1 = self.metrics['ResultsTier1'] dResultsTier2 = self.metrics['ResultsTier2'] resultsCU = self.metrics['resultsCU'] #Load the Thalweg feature thalweg = Shapefile(shpThalweg).featuresToShapely() thalwegLine = thalweg[0]['geometry'] # Load the depth raster depthRaster = Raster(rasDepth) # Load the channel unit polygons and calculate the total area # The channel units should be clipped to the wetted extent and so this # can be used as the site area wetted shpCU = Shapefile(shpCUPath) arrCU = depthRaster.rasterMaskLayer(shpCUPath, "UnitNumber") feats = shpCU.featuresToShapely() for aFeat in feats: dResultsChannelSummary['Main']['Area'] += aFeat['geometry'].area # Loop over each channel unit and calculate topometrics for aFeat in feats: nCUNumber = int(aFeat['fields']['UnitNumber']) if nCUNumber not in dUnits: self.log.error( "Channel Unit: '{0}' not present in the aux data.".format( nCUNumber)) # Keep it general for the exception so we can aggregate them raise DataException( "The Channel Unit ShapeFile contains a unit number that is not present in the aux data." ) tier1Name = dUnits[nCUNumber][0] tier2Name = dUnits[nCUNumber][1] nSegment = dUnits[nCUNumber][2] #print "Channel Unit Number {0}, Segment {1}, Tier 1 - {2}, Tier 2 - {3}".format(nCUNumber, nSegment, tier1Name, tier2Name) unitMetrics = {} resultsCU.append(unitMetrics) unitMetrics['ChannelUnitNumber'] = nCUNumber unitMetrics['Area'] = aFeat['geometry'].area unitMetrics['Tier1'] = tier1Name unitMetrics['Tier2'] = tier2Name unitMetrics['Length'] = None unitMetrics['ResidualDepth'] = None unitMetrics['DepthAtThalwegExit'] = None unitMetrics['ThalwegIntersect'] = 0 # Get the depth raster for this unit as variable so we can check # whether it is entirely masked below. depArr = depthRaster.array[arrCU == nCUNumber] if depArr.count() == 0: unitMetrics['MaxDepth'] = 0 unitMetrics['Volume'] = 0 else: unitMetrics['MaxDepth'] = np.max(depArr) unitMetrics['Volume'] = np.sum(depthRaster.array[ arrCU == nCUNumber]) * (depthRaster.cellWidth**2) if nSegment != 1: dSideChannelSummary = dResultsChannelSummary[ 'SideChannelSummary'] dMain = dResultsChannelSummary['Main'] # Side channel summary captures both small and large side channels dSideChannelSummary['Area'] += aFeat['geometry'].area dSideChannelSummary['Count'] += 1 dSideChannelSummary['Percent'] = 100 * dSideChannelSummary[ 'Area'] / dMain['Area'] dSideChannelSummary['Volume'] += unitMetrics['Volume'] if 'side' in tier1Name.lower(): dSmallSideChannel = dResultsChannelSummary[ 'SmallSideChannel'] dSmallSideChannel['Area'] += aFeat['geometry'].area dSmallSideChannel['Count'] += 1 dSmallSideChannel['Percent'] = 100 * dSmallSideChannel[ 'Area'] / dMain['Area'] dSmallSideChannel['Volume'] += unitMetrics['Volume'] else: dLargeSideChannel = dResultsChannelSummary[ 'LargeSideChannel'] dLargeSideChannel['Area'] += aFeat['geometry'].area dLargeSideChannel['Count'] += 1 dLargeSideChannel['Percent'] = 100 * dLargeSideChannel[ 'Area'] / dMain['Area'] dLargeSideChannel['Volume'] += unitMetrics['Volume'] if tier1Name is None: raise DataException("tier1Name cannot be 'None'") if 'side' in tier1Name.lower(): dResultsChannelSummary['ChannelUnitBreakdown'][ 'SmallSideChannel'] += 1 else: dResultsChannelSummary['ChannelUnitBreakdown']['Other'] += 1 if (thalwegLine.intersects(aFeat['geometry'])): cuThalwegLine = thalwegLine.intersection(aFeat['geometry']) exitPoint = None if cuThalwegLine.type == 'LineString': exitPoint = cuThalwegLine.coords[0] else: exitPoint = cuThalwegLine[0].coords[0] # Retrieve a list of points along the Thalweg in the channel unit thalwegPoints = ChannelUnitMetrics.interpolatePointsAlongLine( cuThalwegLine, 0.13) thalwegDepths = ChannelUnitMetrics.lookupRasterValuesAtPoints( thalwegPoints, depthRaster) unitMetrics['MaxDepth'] = np.nanmax(thalwegDepths['values']) unitMetrics['DepthAtThalwegExit'] = depthRaster.getPixelVal( exitPoint) unitMetrics['ResidualDepth'] = unitMetrics[ 'MaxDepth'] - unitMetrics['DepthAtThalwegExit'] unitMetrics['Length'] = cuThalwegLine.length unitMetrics['ThalwegIntersect'] = 1 # Tier 1 and tier 2 topometrics. Note that metric dictionary keys are used for XML tags & require cleaning tier1NameClean = getCleanTierName(tier1Name) self._calcTierLevelMetrics(dResultsTier1[tier1NameClean], tier1Name, unitMetrics, siteLength, dResultsChannelSummary['Main']['Area']) tier2NameClean = getCleanTierName(tier2Name) self._calcTierLevelMetrics(dResultsTier2[tier2NameClean], tier2Name, unitMetrics, siteLength, dResultsChannelSummary['Main']['Area']) # Calculate the average of the channel unit max depths for each tier 1 and tier 2 type for tierKey, tierMetrics in { 'Tier1': dResultsTier1, 'Tier2': dResultsTier2 }.iteritems(): for tierName, metricDict in tierMetrics.iteritems(): maxDepthList = [ aResult['MaxDepth'] for aResult in resultsCU if getCleanTierName(aResult[tierKey]) == tierName ] if len(maxDepthList) > 0: metricDict['AvgMaxDepth'] = np.average(maxDepthList) # Convert the sum of residual depth and depth at thalweg exist # to average residual depth for each tier 1 and tier 2 type for tierMetricDict in [dResultsTier1, dResultsTier2]: for tierName, tierMetrics in tierMetricDict.iteritems(): # channel unit types that don't occur should retain the value None for Residual Depth and Depth at Thalweg exit if tierMetrics['Count'] > 0 and tierMetrics[ 'ThalwegIntersectCount'] > 0: for metricName in ['ResidualDepth', 'DepthAtThalwegExit']: if tierMetrics[metricName] is not None and tierMetrics[ metricName] != 0: tierMetrics[metricName] = tierMetrics[ metricName] / tierMetrics[ 'ThalwegIntersectCount'] else: tierMetrics[metricName] = 0
def validate(self): results = super(CHaMP_Thalweg, self).validate() validate_maxfeaturecount = ValidationResult(self.__class__.__name__, "MaxFeatureCount") validate_wsedemExtent = ValidationResult(self.__class__.__name__, "WithinWSEDEMExtent") validate_in_end_dist = ValidationResult(self.__class__.__name__, "InPointNearEnd") validate_out_start_dist = ValidationResult(self.__class__.__name__, "OutPointNearStart") validate_out_higher_inflow = ValidationResult(self.__class__.__name__, "StartPointLowerEndPoint") validate_thalwegstartstopraster = ValidationResult(self.__class__.__name__, "ThalwegStartStopOnDEM") if self.exists(): if len(self.features) > self.maxFeatureCount: validate_maxfeaturecount.error("Number of features (" + str(len(self.features)) + ") exceeds the maximum number allowed (" + str(self.maxFeatureCount) + ")") else: validate_maxfeaturecount.pass_validation() if self.wsedemExtent: if self.start_stop_on_raster(raster_extent=self.wsedemExtent): validate_wsedemExtent.pass_validation() else: validate_wsedemExtent.error("Thalweg not entirely contained within WSEDEM.") if self.topo_in_point: inbuffer = self.topo_in_point.buffer(15) if inbuffer.contains(self.thalweg_end_pt()): validate_in_end_dist.pass_validation() else: validate_in_end_dist.warning("End point is greater than 15m from Topo 'in' point") if self.topo_out_point: if self.topo_out_point.buffer(15).contains(self.thalweg_start_pnt()): validate_out_start_dist.pass_validation() else: validate_out_start_dist.warning("Start point is greater than 15m from Topo 'out' point") if self.dem: if self.demDataExtent and self.features: if self.start_stop_on_raster(): validate_thalwegstartstopraster.pass_validation() else: validate_thalwegstartstopraster.error("One or more line features does not start or stop on the DEM") r = Raster(self.dem) tStart = self.thalweg_start_pnt() tEnd = self.thalweg_end_pt() if tStart is None or tEnd is None: validate_out_higher_inflow.error("Could not determine thalweg start and finish") else: z_start = r.getPixelVal([tStart.x, tStart.y]) z_end = r.getPixelVal([tEnd.x, tEnd.y]) if z_start > z_end + 0.1: validate_out_higher_inflow.error("Thalweg Start (outflow) more than 10cm higher than end (inflow)") else: validate_out_higher_inflow.pass_validation() results.append(validate_maxfeaturecount.get_dict()) results.append(validate_wsedemExtent.get_dict()) results.append(validate_in_end_dist.get_dict()) results.append(validate_out_start_dist.get_dict()) results.append(validate_out_higher_inflow.get_dict()) results.append(validate_thalwegstartstopraster.get_dict()) return results
def calc(self, crosssections, waterExtent, demPath, stationInterval): # Save space by only loading the desired fields from the ShapeFile. # We also need the 'Channel' and 'IsValid' fields if they exist. desiredFields = CrossSectionMetrics.dMetricTypes.keys() desiredFields.append('IsValid') # Open the cross section ShapeFile & build a list of all features with a dictionary of the desired fields clShp = Shapefile(crosssections) lChannels = ['Main'] if not clShp.loaded: return if "Channel" in clShp.fields: desiredFields.append('Channel') lChannels.append('Side') # Older Cross Section Layers don't have the width and depth fields calculated. # So if all the necessary metric fields are present then continue to load # ShapeFile features. Otherwise we need to calculate the topometrics from scratch bMetricCalculationNeeded = False for aMetric in desiredFields: if not aMetric in clShp.fields: bMetricCalculationNeeded = True break allFeatures = [] if bMetricCalculationNeeded: # Retrieve the water extent polygon exterior rivershp = Shapefile(waterExtent) polyRiverShapeFeats = rivershp.featuresToShapely() # Try and find a channel shape. Thers's a lot of variance here. if len(polyRiverShapeFeats) == 0 or 'geometry' not in polyRiverShapeFeats[0]: raise DataException("No features in crosssection shape file") # If there's only one shape then just use it elif len(polyRiverShapeFeats) == 1: polyRiverShape = polyRiverShapeFeats[0]['geometry'] # If there's more than one shape then see if else: channelShapes = [feat['geometry'] for feat in polyRiverShapeFeats if feat['fields']['ExtentType'] == 'Channel'] if len(channelShapes) == 0: raise DataException("No features in crosssection shape file") polyRiverShape = channelShapes[0] # Calculate the topometrics from scratch for a single cross section shpXS = Shapefile(crosssections) demRaster = Raster(demPath) if shpXS.loaded: for aFeat in shpXS.featuresToShapely(): # Calculate the topometrics for this cross section. They will be stored on the aFeat dict under key 'topometrics' calcXSMetrics(aFeat, polyRiverShape , demRaster, stationInterval) # Build the all features dictionary that would be expect had the topometrics already # existed in the XS shapefile and simply got loaded. This is a combination of the new topometrics # and also the existing fields on the XS ShapeFile. singleXSMetrics = copy.deepcopy(aFeat['topometrics']) singleXSMetrics.update(aFeat['fields']) allFeatures.append(singleXSMetrics) # Destroying the raster object appears to prevent warning messages on Windows demRaster = None else: allFeatures = clShp.attributesToList(desiredFields) # For simple ShapeFiles, make every feature part of the main channel, and # set every feature as valid. This helps keep code below generic for x in allFeatures: if 'Channel' not in x: x['Channel'] = 'Main' if 'IsValid' not in x: x['IsValid'] = 1 for channelName in lChannels: # Filter the list of features to just those in this channel # PGB - 24 Apr 2017 - observed NULL values in 'Channel' ShapeFile field in Harold results. # Cast field contents to string to avoid crash here. channelFeatures = [x for x in allFeatures if str(x['Channel']).lower() == channelName.lower()] # Filter the list of features to just those that the crew considered valid validFeatures = [x for x in channelFeatures if x['IsValid'] <> 0] # Filter the features to just those with a length that is within 4 standard deviations of mean wetted width channelStatistics = getStatistics(channelFeatures, 'WetWidth') autoFeatures = None if channelStatistics['StdDev'] is not None: wetWidthThreshold = channelStatistics['StdDev'] * 4 autoFeatures = [x for x in channelFeatures if abs(x['WetWidth'] - channelStatistics['Mean']) < wetWidthThreshold] # Loop over each desired metric and calculate the statistics for each filtering type for metricName, bestFiltering in CrossSectionMetrics.dMetricTypes.iteritems(): populateChannelStatistics(self.metrics[channelName], 'None', metricName, channelFeatures) populateChannelStatistics(self.metrics[channelName], 'Crew', metricName, validFeatures) if channelStatistics['StdDev'] is not None: populateChannelStatistics(self.metrics[channelName], 'Auto', metricName, autoFeatures) self.metrics[channelName]['Best'] = self.metrics[channelName][bestFiltering] # The topometrics for the whole channel are always the results for 'Main'. # For complex ShapeFiles this will be just the results for the main channel. # For simple, single threaded, ShapeFiles this will all cross sections. self.metrics['Channel'] = self.metrics['Main']