def assert_rasters_are_equal(file1, file2): reader1 = io.RasterReader(file1) reader2 = io.RasterReader(file2) assert reader1.transform == reader2.transform assert reader1.crs == reader2.crs data1 = reader1.read() data2 = reader2.read() assert np.all(data1 == data2), "Files not equal {} != {}".format(file1, file2)
def process_wsheds(bluespots, flowdir, out): """Bluespot watersheds. Assign bluespot ID to all cells within the local bluespot watershed. """ bspot_reader = io.RasterReader(bluespots) flowdir_reader = io.RasterReader(flowdir) wshed_writer = io.RasterWriter(out, bspot_reader.transform, bspot_reader.crs, 0) watersheds = bspot_reader.read() flowdir = flowdir_reader.read() flow.watersheds_from_labels(flowdir, watersheds, unassigned=0) wshed_writer.write(watersheds)
def process_pourpoints(bluespots, depths, watersheds, dem, accum, out, format, layername, dsco, lco): """Determine pour points. \b Determines a pour point for each bluespot using one of two methods: * Random candidate. Requires DEM only * Maximum accumulated flow candidate. Requires accumulated flow The output of the two methods only differ when there are more than one pour point candidate (ie multiple threshold cells with identical Z) for a given bluespot. For documentation of OGR features (format, dsco and lco) see http://www.gdal.org/ogr_formats.html """ bspot_reader = io.RasterReader(bluespots) depths_reader = io.RasterReader(depths) wsheds_reader = io.RasterReader(watersheds) data = accum if accum else dem if not data: raise Exception('Either accum or dem must be specified') data_reader = io.RasterReader(data) format = str(format) layername = str(layername) pourpnt_writer = io.VectorWriter(format, out, layername, [], ogr.wkbPoint, depths_reader.crs, dsco, lco) # Recalculate stats on filtered bluespots labeled_data = bspot_reader.read() depths_data = depths_reader.read() bluespot_stats = label.label_stats(depths_data, labeled_data) del depths_data if accum: pp_pix = label.label_max_index(data_reader.read(), labeled_data) elif dem: dem_data = data_reader.read() short, diag = fill.minimum_safe_short_and_diag(dem_data) filled_no_flats = fill.fill_terrain_no_flats(dem_data, short, diag) pp_pix = label.label_min_index(filled_no_flats, labeled_data) del dem_data watershed_stats = label.label_count(wsheds_reader.read()) pour_points = assemble_pourpoints(depths_reader.transform, pp_pix, bluespot_stats, watershed_stats) feature_collection = dict(type="FeatureCollection", features=pour_points) pourpnt_writer.write_geojson_features(feature_collection)
def process_bspots(depths, out, filter): """Label bluespots. Assign unique bluespot ID to all cells belonging to a bluespot. Optionally disregarding some bluespots based on a filter expression. ID 0 (zero) is used for cells not belonging to a bluespot. """ depths_reader = io.RasterReader(depths) labeled_writer = io.RasterWriter(out, depths_reader.transform, depths_reader.crs, 0) filter_function = parse_filter(filter) transform = depths_reader.transform cell_width = abs(transform[1]) cell_height = abs(transform[5]) cell_area = cell_width * cell_height depths_data = depths_reader.read() raw_labeled, raw_nlabels = label.connected_components(depths_data) if not filter: # This is the end my friend labeled_writer.write(raw_labeled) return del depths raw_bluespot_stats = label.label_stats(depths_data, raw_labeled) keepers = filterbluespots(filter_function, cell_area, raw_bluespot_stats) new_components = label.keep_labels(raw_labeled, keepers) labeled, nlabels = label.connected_components(new_components) labeled_writer.write(labeled)
def process_volumes(nodes, nodes_layer, mm, pr, pr_unit, bluespots, out, out_layer, out_attribute, format, dsco, lco): """Set up initial water volumes for each watershed. The output from this process can be used as input for the finalvolumes calculation. \b Examples: malstroem initvolumes -mm 10 -nodes results.gpkg -out results.gpkg -format gpkg malstroem initvolumes -pr precip_raster.tif -bluespots bluespots.tif -nodes results.gpkg -out results.gpkg -format gpkg For documentation of OGR features (format, dsco and lco) see http://www.gdal.org/ogr_formats.html """ # Validate one and only one if mm and pr: raise click.UsageError("-mm and -pr are mutually exclusive") if not mm and not pr: raise click.UsageError("One of -mm and -pr must be specified") nodes_layer = nodes_layer format = str(format) out_layer = str(out_layer) # Calculate volumes nodes_reader = io.VectorReader(nodes, nodes_layer) volumes_writer = io.VectorWriter(format, out, out_layer, None, ogr.wkbPoint, nodes_reader.crs, dsco, lco) if mm: logger.info( f"Processing initial volumes using evenly distributed rain event of {mm}mm" ) rain_tool = raintool.SimpleVolumeTool(nodes_reader, volumes_writer, out_attribute, mm) rain_tool.process() else: if not bluespots: raise click.UsageError("Mising -bluespots") logger.info( f"Processing initial volumes using [{pr_unit}] raster input from {pr}" ) precip_reader = io.RasterReader(pr) bspotlabels_reader = io.RasterReader(bluespots) tool = raintool.RasterVolumeTool(nodes_reader, bspotlabels_reader, precip_reader, pr_unit, volumes_writer, out_attribute) tool.process()
def process_network(bluespots, flowdir, pourpoints, pourpoints_layer, out, out_nodes_layer, out_streams_layer, format, dsco, lco): """Calculate stream network between bluespots. For documentation of OGR features (format, dsco and lco) see http://www.gdal.org/ogr_formats.html """ pourpoints_reader = io.VectorReader(pourpoints, str(pourpoints_layer)) bluespot_reader = io.RasterReader(bluespots) flowdir_reader = io.RasterReader(flowdir) format = str(format) out_nodes_layer = str(out_nodes_layer) out_streams_layer = str(out_streams_layer) nodes_writer = io.VectorWriter(format, out, out_nodes_layer, None, ogr.wkbPoint, flowdir_reader.crs, dsco, lco) streams_writer = io.VectorWriter(format, out, out_streams_layer, None, ogr.wkbLineString, flowdir_reader.crs, dsco, lco) stream_tool = streams.StreamTool(pourpoints_reader, bluespot_reader, flowdir_reader, nodes_writer, streams_writer) stream_tool.process()
def test_bluespots(tmpdir): flowdir_reader = io.RasterReader(flowdirnoflatsfile) dem_reader = io.RasterReader(dtmfile) filled_reader = io.RasterReader(filledfile) depths_reader = NumpyRasterReader(filled_reader.read() - dem_reader.read(), dem_reader.transform) outdbfile = str(tmpdir.join('test.gpkg')) # At least 5cm deep, 5 cells wide and at least one cell-meter volume filter_function = lambda r: r['max'] > 0.05 and r['count'] > 5 and r['sum' ] > 1 pourpoint_writer = io.VectorWriter('gpkg', outdbfile, 'pourpoints', None, ogr.wkbPoint, dem_reader.crs) watershed_writer = io.RasterWriter(str(tmpdir.join('watersheds.tif')), dem_reader.transform, dem_reader.crs, 0) watershed_vector_writer = io.VectorWriter('gpkg', outdbfile, 'watersheds', None, ogr.wkbMultiPolygon, dem_reader.crs) labeled_writer = io.RasterWriter(str(tmpdir.join('labeled.tif')), dem_reader.transform, dem_reader.crs, 0) labeled_vector_writer = io.VectorWriter('gpkg', outdbfile, 'bluespots', None, ogr.wkbMultiPolygon, dem_reader.crs) bluespot_tool = bluespots.BluespotTool( input_depths=depths_reader, input_flowdir=flowdir_reader, input_bluespot_filter_function=filter_function, input_accum=None, input_dem=dem_reader, output_labeled_raster=labeled_writer, output_labeled_vector=labeled_vector_writer, output_pourpoints=pourpoint_writer, output_watersheds_raster=watershed_writer, output_watersheds_vector=watershed_vector_writer) bluespot_tool.process() assert os.path.isfile(outdbfile) assert os.path.isfile(watershed_writer.filepath) assert os.path.isfile(labeled_writer.filepath)
def process_bluespots(bluespots, dem, finallevels, finallevels_layer, out_depths, out_bluespots): """Approximate extent and depths rasters of bluespots in the final state. """ if not out_depths and not out_bluespots: raise click.UsageError("No output specified") bspot_reader = io.RasterReader(bluespots) dem_reader = io.RasterReader(dem, nodatasubst=-999) levels_reader = io.VectorReader(finallevels, finallevels_layer) depths_writer = io.RasterWriter(str(out_depths), bspot_reader.transform, bspot_reader.crs) if out_depths else None final_bs_writer = io.RasterWriter( str(out_bluespots), bspot_reader.transform, bspot_reader.crs, nodata=0) if out_bluespots else None approx.approx_bluespots_io(bspot_reader, levels_reader, dem_reader, depths_writer, final_bs_writer)
def process_hypsometry(bluespots, dem, pourpoints, pourpoints_layer, zresolution, out, out_hyps_layer, format, dsco, lco): """Statistical terrain elevation measures for each bluespot. For each bluespot these values describing the terrain within the bluespot are returned: - A DEM Z value histogram with user definable bin width (resolution) - Number og bins, effective upper and lower bounds of the histogram - Actual minimum and maximum Z values The values of the histogram are formatted as a single string using pipe '|' as seperator. Like: 2|1|0|3 For documentation of OGR features (format, dsco and lco) see http://www.gdal.org/ogr_formats.html """ pourpoints_reader = io.VectorReader(pourpoints, pourpoints_layer) labeled_reader = io.RasterReader(bluespots) dem_reader = io.RasterReader(dem, nodatasubst=NODATASUBST) ogr_format = str(format) hyps_writer = io.VectorWriter(ogr_format, out, out_hyps_layer, None, ogr.wkbNone, dem_reader.crs) hyps.bluespot_hypsometry_io(labeled_reader, dem_reader, pourpoints_reader, zresolution, hyps_writer)
def test_complete_nofilter(tmpdir): runner = CliRunner() result = runner.invoke(cli, [ 'complete', '-r', 10, '-r', 100, '-dem', dtmfile, '-outdir', str(tmpdir) ]) assert result.exit_code == 0, result.output assert os.path.isfile(str(tmpdir.join('filled.tif'))) r = io.RasterReader(str(tmpdir.join('bluespots.tif'))) data = r.read() assert np.max(data) == 523 v = io.VectorReader(str(tmpdir.join('vector')), 'events') data = v.read_geojson_features() assert len(data) == 587, result.output
def test_dem_processor(tmpdir): dem_reader = io.RasterReader(dtmfile) tr = dem_reader.transform crs = dem_reader.crs filled_writer = io.RasterWriter(str(tmpdir.join('filled.tif')), tr, crs) flowdir_writer = io.RasterWriter(str(tmpdir.join('flowdir.tif')), tr, crs) depths_writer = io.RasterWriter(str(tmpdir.join('depths.tif')), tr, crs) accum_writer = io.RasterWriter(str(tmpdir.join('accum.tif')), tr, crs) tool = dem.DemTool(dem_reader, filled_writer, flowdir_writer, depths_writer, accum_writer) tool.process() assert_rasters_are_equal(filledfile, filled_writer.filepath) assert_rasters_are_equal(flowdirnoflatsfile, flowdir_writer.filepath)
def test_write_raster_datatypes(tmpdir): data_types = [ "float64", "float32", "int32", "uint32", "uint8", "int64", "uint16", "int16" ] shape = (25, 30) data = np.arange(shape[0] * shape[1]).reshape(shape) gt = (510000, 0.2, 0, 6150000, 0, -0.2) tmppath = Path(tmpdir) for dt in data_types: filepath = tmppath / "dt.tif" cast_data = data.astype(dt) writer = io.RasterWriter(str(filepath), gt, "", nodata=None) writer.write(cast_data) assert filepath.exists(), f"File was not written" reader = io.RasterReader(str(filepath)) read_data = reader.read() np.testing.assert_array_equal(read_data, cast_data)
def test_complete(tmpdir): runner = CliRunner() result = runner.invoke(cli, ['complete', '-mm', 100, '-zresolution', 0.1, '-filter', 'area > 20.5 and maxdepth > 0.5 or volume > 2.5', '-dem', dtmfile, '-outdir', str(tmpdir)]) assert result.exit_code == 0, result.output assert os.path.isfile(str(tmpdir.join('filled.tif'))) r = io.RasterReader(str(tmpdir.join('bluespots.tif'))) data = r.read() assert np.max(data) == 486, result.output v = io.VectorReader(str(tmpdir.join('malstroem.gpkg')), 'finalstate') data = v.read_geojson_features() assert len(data) == 544, result.output v = io.VectorReader(str(tmpdir.join('malstroem.gpkg')), 'finalbluespots') data = v.read_geojson_features() assert len(data) == 500, result.output
def process_all(dem, outdir, accum, filter, rain, vector): """Quick option to run all processes. \b Example: malstroem complete -r 10 -r 30 -filter "volume > 2.5" -dem dem.tif -outdir ./outdir/ """ # Check that outdir exists and is empty if not os.path.isdir(outdir) or not os.path.exists(outdir) or os.listdir(outdir): logger.error("outdir isn't an empty directory") return 1 #outvector = os.path.join(outdir, 'malstroem.gpkg') outvector = os.path.join(outdir, 'vector') #ogr_drv = 'gpkg' ogr_dsco = [] ogr_drv = 'ESRI shapefile' nodatasubst = -999 filter_function = parse_filter(filter) dem_reader = io.RasterReader(dem, nodatasubst=nodatasubst) tr = dem_reader.transform crs = dem_reader.crs logger.info('Processing') logger.info(' dem: {}'.format(dem)) logger.info(' outdir: {}'.format(outdir)) logger.info(' rain: {}'.format(', '.join(['{}mm'.format(r) for r in rain]))) logger.info(' accum: {}'.format(accum)) logger.info(' filter: {}'.format(filter)) # Process DEM filled_writer = io.RasterWriter(os.path.join(outdir, 'filled.tif'), tr, crs, nodatasubst) flowdir_writer = io.RasterWriter(os.path.join(outdir, 'flowdir.tif'), tr, crs) depths_writer = io.RasterWriter(os.path.join(outdir, 'bs_depths.tif'), tr, crs) accum_writer = io.RasterWriter(os.path.join(outdir, 'accum.tif'), tr, crs) if accum else None dtmtool = demtool.DemTool(dem_reader, filled_writer, flowdir_writer, depths_writer, accum_writer) dtmtool.process() # Process bluespots depths_reader = io.RasterReader(depths_writer.filepath) flowdir_reader = io.RasterReader(flowdir_writer.filepath) accum_reader = io.RasterReader(accum_writer.filepath) if accum_writer else None pourpoint_writer = io.VectorWriter(ogr_drv, outvector, 'pourpoints', None, ogr.wkbPoint, crs, dsco=ogr_dsco) watershed_writer = io.RasterWriter(os.path.join(outdir, 'watersheds.tif'), tr, crs, 0) watershed_vector_writer = io.VectorWriter(ogr_drv, outvector, 'watersheds', None, ogr.wkbMultiPolygon, crs, dsco=ogr_dsco) if vector else None labeled_writer = io.RasterWriter(os.path.join(outdir, 'bluespots.tif'), tr, crs, 0) labeled_vector_writer = io.VectorWriter(ogr_drv, outvector, 'bluespots', None, ogr.wkbMultiPolygon, crs, dsco=ogr_dsco) if vector else None bluespot_tool = bluespots.BluespotTool( input_depths=depths_reader, input_flowdir=flowdir_reader, input_bluespot_filter_function=filter_function, input_accum=accum_reader, input_dem=dem_reader, output_labeled_raster=labeled_writer, output_labeled_vector=labeled_vector_writer, output_pourpoints=pourpoint_writer, output_watersheds_raster=watershed_writer, output_watersheds_vector=watershed_vector_writer ) bluespot_tool.process() # Process pourpoints pourpoints_reader = io.VectorReader(outvector, pourpoint_writer.layername) bluespot_reader = io.RasterReader(labeled_writer.filepath) flowdir_reader = io.RasterReader(flowdir_writer.filepath) nodes_writer = io.VectorWriter(ogr_drv, outvector, 'nodes', None, ogr.wkbPoint, crs, dsco=ogr_dsco) streams_writer = io.VectorWriter(ogr_drv, outvector, 'streams', None, ogr.wkbLineString, crs, dsco=ogr_dsco) stream_tool = streams.StreamTool(pourpoints_reader, bluespot_reader, flowdir_reader, nodes_writer, streams_writer) stream_tool.process() # Process rain events nodes_reader = io.VectorReader(outvector, nodes_writer.layername) events_writer = io.VectorWriter(ogr_drv, outvector, 'events', None, ogr.wkbPoint, crs, dsco=ogr_dsco) rain_tool = raintool.RainTool(nodes_reader, events_writer, rain) rain_tool.process()
def process_all(dem, outdir, accum, filter, mm, zresolution, vector): """Quick option to run all processes. \b Example: malstroem complete -mm 20 -filter "volume > 2.5" -dem dem.tif -zresolution 0.1 -outdir ./outdir/ """ # Check that outdir exists and is empty if not os.path.isdir(outdir) or not os.path.exists(outdir) or os.listdir( outdir): logger.error("outdir isn't an empty directory") return 1 outvector = os.path.join(outdir, 'malstroem.gpkg') ogr_drv = 'gpkg' ogr_dsco = [] ogr_lco = ["SPATIAL_INDEX=NO"] nodatasubst = -999 filter_function = parse_filter(filter) dem_reader = io.RasterReader(dem, nodatasubst=nodatasubst) tr = dem_reader.transform crs = dem_reader.crs logger.info('Processing') logger.info(' dem: {}'.format(dem)) logger.info(' outdir: {}'.format(outdir)) logger.info(' mm: {}mm'.format(mm)) logger.info(' zresolution: {}m'.format(zresolution)) logger.info(' accum: {}'.format(accum)) logger.info(' filter: {}'.format(filter)) # Process DEM filled_writer = io.RasterWriter(os.path.join(outdir, 'filled.tif'), tr, crs, nodatasubst) flowdir_writer = io.RasterWriter(os.path.join(outdir, 'flowdir.tif'), tr, crs) depths_writer = io.RasterWriter(os.path.join(outdir, 'bs_depths.tif'), tr, crs) accum_writer = io.RasterWriter(os.path.join(outdir, 'accum.tif'), tr, crs) if accum else None dtmtool = demtool.DemTool(dem_reader, filled_writer, flowdir_writer, depths_writer, accum_writer) dtmtool.process() # Process bluespots depths_reader = io.RasterReader(depths_writer.filepath) flowdir_reader = io.RasterReader(flowdir_writer.filepath) accum_reader = io.RasterReader( accum_writer.filepath) if accum_writer else None pourpoint_writer = io.VectorWriter(ogr_drv, outvector, 'pourpoints', None, ogr.wkbPoint, crs, dsco=ogr_dsco, lco=ogr_lco) watershed_writer = io.RasterWriter(os.path.join(outdir, 'watersheds.tif'), tr, crs, 0) watershed_vector_writer = io.VectorWriter(ogr_drv, outvector, 'watersheds', None, ogr.wkbMultiPolygon, crs, dsco=ogr_dsco, lco=ogr_lco) if vector else None labeled_writer = io.RasterWriter(os.path.join(outdir, 'bluespots.tif'), tr, crs, 0) labeled_vector_writer = io.VectorWriter(ogr_drv, outvector, 'bluespots', None, ogr.wkbMultiPolygon, crs, dsco=ogr_dsco, lco=ogr_lco) if vector else None bluespot_tool = bluespots.BluespotTool( input_depths=depths_reader, input_flowdir=flowdir_reader, input_bluespot_filter_function=filter_function, input_accum=accum_reader, input_dem=dem_reader, output_labeled_raster=labeled_writer, output_labeled_vector=labeled_vector_writer, output_pourpoints=pourpoint_writer, output_watersheds_raster=watershed_writer, output_watersheds_vector=watershed_vector_writer) bluespot_tool.process() # Process pourpoints pourpoints_reader = io.VectorReader(outvector, pourpoint_writer.layername) bluespot_reader = io.RasterReader(labeled_writer.filepath) flowdir_reader = io.RasterReader(flowdir_writer.filepath) nodes_writer = io.VectorWriter(ogr_drv, outvector, 'nodes', None, ogr.wkbPoint, crs, dsco=ogr_dsco, lco=ogr_lco) streams_writer = io.VectorWriter(ogr_drv, outvector, 'streams', None, ogr.wkbLineString, crs, dsco=ogr_dsco, lco=ogr_lco) stream_tool = streams.StreamTool(pourpoints_reader, bluespot_reader, flowdir_reader, nodes_writer, streams_writer) stream_tool.process() # Calculate volumes nodes_reader = io.VectorReader(outvector, nodes_writer.layername) volumes_writer = io.VectorWriter(ogr_drv, outvector, 'initvolumes', None, ogr.wkbPoint, crs, dsco=ogr_dsco, lco=ogr_lco) rain_tool = raintool.SimpleVolumeTool(nodes_reader, volumes_writer, "inputv", mm) rain_tool.process() # Process final state volumes_reader = io.VectorReader(outvector, volumes_writer.layername) events_writer = io.VectorWriter(ogr_drv, outvector, 'finalstate', None, ogr.wkbPoint, crs, dsco=ogr_dsco, lco=ogr_lco) calculator = network.FinalStateCalculator(volumes_reader, "inputv", events_writer) calculator.process() # Hypsometry pourpoints_reader = io.VectorReader(outvector, pourpoint_writer.layername) hyps_writer = io.VectorWriter(ogr_drv, outvector, "hypsometry", None, ogr.wkbNone, dem_reader.crs) hyps.bluespot_hypsometry_io(bluespot_reader, dem_reader, pourpoints_reader, zresolution, hyps_writer) # Approximation on levels finalvols_reader = io.VectorReader(outvector, events_writer.layername) hyps_reader = io.VectorReader(outvector, hyps_writer.layername) levels_writer = io.VectorWriter(ogr_drv, outvector, "finallevels", None, ogr.wkbNone, dem_reader.crs) approx.approx_water_level_io(finalvols_reader, hyps_reader, levels_writer) # Approximation on bluespots levels_reader = io.VectorReader(outvector, levels_writer.layername) final_depths_writer = io.RasterWriter( os.path.join(outdir, 'finaldepths.tif'), tr, crs) final_bs_writer = io.RasterWriter( os.path.join(outdir, 'finalbluespots.tif'), tr, crs, 0) approx.approx_bluespots_io(bluespot_reader, levels_reader, dem_reader, final_depths_writer, final_bs_writer) # Polygonize final bluespots logger.info("Polygonizing final bluespots") vectorize_labels_file_io(final_bs_writer.filepath, outvector, "finalbluespots", ogr_drv, ogr_dsco, ogr_lco) logger.info("Complete done...")