def assert_rasters_are_equal(file1, file2):
reader1 = io.RasterReader(file1)
reader2 = io.RasterReader(file2)
assert reader1.transform == reader2.transform
assert reader1.crs == reader2.crs
data1 = reader1.read()
data2 = reader2.read()
assert np.all(data1 == data2), "Files not equal {} != {}".format(file1, file2)
def process_wsheds(bluespots, flowdir, out):
"""Bluespot watersheds.
Assign bluespot ID to all cells within the local bluespot watershed.
"""
bspot_reader = io.RasterReader(bluespots)
flowdir_reader = io.RasterReader(flowdir)
wshed_writer = io.RasterWriter(out, bspot_reader.transform,
bspot_reader.crs, 0)
watersheds = bspot_reader.read()
flowdir = flowdir_reader.read()
flow.watersheds_from_labels(flowdir, watersheds, unassigned=0)
wshed_writer.write(watersheds)
def process_pourpoints(bluespots, depths, watersheds, dem, accum, out, format,
layername, dsco, lco):
"""Determine pour points.
\b
Determines a pour point for each bluespot using one of two methods:
* Random candidate. Requires DEM only
* Maximum accumulated flow candidate. Requires accumulated flow
The output of the two methods only differ when there are more than one pour point candidate (ie multiple threshold
cells with identical Z) for a given bluespot.
For documentation of OGR features (format, dsco and lco) see http://www.gdal.org/ogr_formats.html
"""
bspot_reader = io.RasterReader(bluespots)
depths_reader = io.RasterReader(depths)
wsheds_reader = io.RasterReader(watersheds)
data = accum if accum else dem
if not data:
raise Exception('Either accum or dem must be specified')
data_reader = io.RasterReader(data)
format = str(format)
layername = str(layername)
pourpnt_writer = io.VectorWriter(format, out, layername, [], ogr.wkbPoint,
depths_reader.crs, dsco, lco)
# Recalculate stats on filtered bluespots
labeled_data = bspot_reader.read()
depths_data = depths_reader.read()
bluespot_stats = label.label_stats(depths_data, labeled_data)
del depths_data
if accum:
pp_pix = label.label_max_index(data_reader.read(), labeled_data)
elif dem:
dem_data = data_reader.read()
short, diag = fill.minimum_safe_short_and_diag(dem_data)
filled_no_flats = fill.fill_terrain_no_flats(dem_data, short, diag)
pp_pix = label.label_min_index(filled_no_flats, labeled_data)
del dem_data
watershed_stats = label.label_count(wsheds_reader.read())
pour_points = assemble_pourpoints(depths_reader.transform, pp_pix,
bluespot_stats, watershed_stats)
feature_collection = dict(type="FeatureCollection", features=pour_points)
pourpnt_writer.write_geojson_features(feature_collection)
def process_bspots(depths, out, filter):
"""Label bluespots.
Assign unique bluespot ID to all cells belonging to a bluespot. Optionally disregarding some bluespots based on a
filter expression. ID 0 (zero) is used for cells not belonging to a bluespot.
"""
depths_reader = io.RasterReader(depths)
labeled_writer = io.RasterWriter(out, depths_reader.transform,
depths_reader.crs, 0)
filter_function = parse_filter(filter)
transform = depths_reader.transform
cell_width = abs(transform[1])
cell_height = abs(transform[5])
cell_area = cell_width * cell_height
depths_data = depths_reader.read()
raw_labeled, raw_nlabels = label.connected_components(depths_data)
if not filter:
# This is the end my friend
labeled_writer.write(raw_labeled)
return
del depths
raw_bluespot_stats = label.label_stats(depths_data, raw_labeled)
keepers = filterbluespots(filter_function, cell_area, raw_bluespot_stats)
new_components = label.keep_labels(raw_labeled, keepers)
labeled, nlabels = label.connected_components(new_components)
labeled_writer.write(labeled)
def process_volumes(nodes, nodes_layer, mm, pr, pr_unit, bluespots, out,
out_layer, out_attribute, format, dsco, lco):
"""Set up initial water volumes for each watershed.
The output from this process can be used as input for the finalvolumes calculation.
\b
Examples:
malstroem initvolumes -mm 10 -nodes results.gpkg -out results.gpkg -format gpkg
malstroem initvolumes -pr precip_raster.tif -bluespots bluespots.tif -nodes results.gpkg -out results.gpkg -format gpkg
For documentation of OGR features (format, dsco and lco) see http://www.gdal.org/ogr_formats.html
"""
# Validate one and only one
if mm and pr:
raise click.UsageError("-mm and -pr are mutually exclusive")
if not mm and not pr:
raise click.UsageError("One of -mm and -pr must be specified")
nodes_layer = nodes_layer
format = str(format)
out_layer = str(out_layer)
# Calculate volumes
nodes_reader = io.VectorReader(nodes, nodes_layer)
volumes_writer = io.VectorWriter(format, out, out_layer, None,
ogr.wkbPoint, nodes_reader.crs, dsco, lco)
if mm:
logger.info(
f"Processing initial volumes using evenly distributed rain event of {mm}mm"
)
rain_tool = raintool.SimpleVolumeTool(nodes_reader, volumes_writer,
out_attribute, mm)
rain_tool.process()
else:
if not bluespots:
raise click.UsageError("Mising -bluespots")
logger.info(
f"Processing initial volumes using [{pr_unit}] raster input from {pr}"
)
precip_reader = io.RasterReader(pr)
bspotlabels_reader = io.RasterReader(bluespots)
tool = raintool.RasterVolumeTool(nodes_reader, bspotlabels_reader,
precip_reader, pr_unit,
volumes_writer, out_attribute)
tool.process()
def process_network(bluespots, flowdir, pourpoints, pourpoints_layer, out, out_nodes_layer, out_streams_layer, format, dsco, lco):
"""Calculate stream network between bluespots.
For documentation of OGR features (format, dsco and lco) see http://www.gdal.org/ogr_formats.html
"""
pourpoints_reader = io.VectorReader(pourpoints, str(pourpoints_layer))
bluespot_reader = io.RasterReader(bluespots)
flowdir_reader = io.RasterReader(flowdir)
format = str(format)
out_nodes_layer = str(out_nodes_layer)
out_streams_layer = str(out_streams_layer)
nodes_writer = io.VectorWriter(format, out, out_nodes_layer, None, ogr.wkbPoint, flowdir_reader.crs, dsco, lco)
streams_writer = io.VectorWriter(format, out, out_streams_layer, None, ogr.wkbLineString, flowdir_reader.crs, dsco, lco)
stream_tool = streams.StreamTool(pourpoints_reader, bluespot_reader, flowdir_reader, nodes_writer, streams_writer)
stream_tool.process()
def test_bluespots(tmpdir):
flowdir_reader = io.RasterReader(flowdirnoflatsfile)
dem_reader = io.RasterReader(dtmfile)
filled_reader = io.RasterReader(filledfile)
depths_reader = NumpyRasterReader(filled_reader.read() - dem_reader.read(),
dem_reader.transform)
outdbfile = str(tmpdir.join('test.gpkg'))
# At least 5cm deep, 5 cells wide and at least one cell-meter volume
filter_function = lambda r: r['max'] > 0.05 and r['count'] > 5 and r['sum'
] > 1
pourpoint_writer = io.VectorWriter('gpkg', outdbfile, 'pourpoints', None,
ogr.wkbPoint, dem_reader.crs)
watershed_writer = io.RasterWriter(str(tmpdir.join('watersheds.tif')),
dem_reader.transform, dem_reader.crs, 0)
watershed_vector_writer = io.VectorWriter('gpkg', outdbfile, 'watersheds',
None, ogr.wkbMultiPolygon,
dem_reader.crs)
labeled_writer = io.RasterWriter(str(tmpdir.join('labeled.tif')),
dem_reader.transform, dem_reader.crs, 0)
labeled_vector_writer = io.VectorWriter('gpkg', outdbfile, 'bluespots',
None, ogr.wkbMultiPolygon,
dem_reader.crs)
bluespot_tool = bluespots.BluespotTool(
input_depths=depths_reader,
input_flowdir=flowdir_reader,
input_bluespot_filter_function=filter_function,
input_accum=None,
input_dem=dem_reader,
output_labeled_raster=labeled_writer,
output_labeled_vector=labeled_vector_writer,
output_pourpoints=pourpoint_writer,
output_watersheds_raster=watershed_writer,
output_watersheds_vector=watershed_vector_writer)
bluespot_tool.process()
assert os.path.isfile(outdbfile)
assert os.path.isfile(watershed_writer.filepath)
assert os.path.isfile(labeled_writer.filepath)
def process_bluespots(bluespots, dem, finallevels, finallevels_layer,
out_depths, out_bluespots):
"""Approximate extent and depths rasters of bluespots in the final state.
"""
if not out_depths and not out_bluespots:
raise click.UsageError("No output specified")
bspot_reader = io.RasterReader(bluespots)
dem_reader = io.RasterReader(dem, nodatasubst=-999)
levels_reader = io.VectorReader(finallevels, finallevels_layer)
depths_writer = io.RasterWriter(str(out_depths), bspot_reader.transform,
bspot_reader.crs) if out_depths else None
final_bs_writer = io.RasterWriter(
str(out_bluespots), bspot_reader.transform, bspot_reader.crs,
nodata=0) if out_bluespots else None
approx.approx_bluespots_io(bspot_reader, levels_reader, dem_reader,
depths_writer, final_bs_writer)
def process_hypsometry(bluespots, dem, pourpoints, pourpoints_layer,
zresolution, out, out_hyps_layer, format, dsco, lco):
"""Statistical terrain elevation measures for each bluespot.
For each bluespot these values describing the terrain within the bluespot are returned:
- A DEM Z value histogram with user definable bin width (resolution)
- Number og bins, effective upper and lower bounds of the histogram
- Actual minimum and maximum Z values
The values of the histogram are formatted as a single string using pipe '|' as seperator. Like:
2|1|0|3
For documentation of OGR features (format, dsco and lco) see http://www.gdal.org/ogr_formats.html
"""
pourpoints_reader = io.VectorReader(pourpoints, pourpoints_layer)
labeled_reader = io.RasterReader(bluespots)
dem_reader = io.RasterReader(dem, nodatasubst=NODATASUBST)
ogr_format = str(format)
hyps_writer = io.VectorWriter(ogr_format, out, out_hyps_layer, None,
ogr.wkbNone, dem_reader.crs)
hyps.bluespot_hypsometry_io(labeled_reader, dem_reader, pourpoints_reader,
zresolution, hyps_writer)
def test_complete_nofilter(tmpdir):
runner = CliRunner()
result = runner.invoke(cli, [
'complete', '-r', 10, '-r', 100, '-dem', dtmfile, '-outdir',
str(tmpdir)
])
assert result.exit_code == 0, result.output
assert os.path.isfile(str(tmpdir.join('filled.tif')))
r = io.RasterReader(str(tmpdir.join('bluespots.tif')))
data = r.read()
assert np.max(data) == 523
v = io.VectorReader(str(tmpdir.join('vector')), 'events')
data = v.read_geojson_features()
assert len(data) == 587, result.output
def test_dem_processor(tmpdir):
dem_reader = io.RasterReader(dtmfile)
tr = dem_reader.transform
crs = dem_reader.crs
filled_writer = io.RasterWriter(str(tmpdir.join('filled.tif')), tr, crs)
flowdir_writer = io.RasterWriter(str(tmpdir.join('flowdir.tif')), tr, crs)
depths_writer = io.RasterWriter(str(tmpdir.join('depths.tif')), tr, crs)
accum_writer = io.RasterWriter(str(tmpdir.join('accum.tif')), tr, crs)
tool = dem.DemTool(dem_reader, filled_writer, flowdir_writer, depths_writer, accum_writer)
tool.process()
assert_rasters_are_equal(filledfile, filled_writer.filepath)
assert_rasters_are_equal(flowdirnoflatsfile, flowdir_writer.filepath)
def test_write_raster_datatypes(tmpdir):
data_types = [
"float64", "float32", "int32", "uint32", "uint8", "int64", "uint16",
"int16"
]
shape = (25, 30)
data = np.arange(shape[0] * shape[1]).reshape(shape)
gt = (510000, 0.2, 0, 6150000, 0, -0.2)
tmppath = Path(tmpdir)
for dt in data_types:
filepath = tmppath / "dt.tif"
cast_data = data.astype(dt)
writer = io.RasterWriter(str(filepath), gt, "", nodata=None)
writer.write(cast_data)
assert filepath.exists(), f"File was not written"
reader = io.RasterReader(str(filepath))
read_data = reader.read()
np.testing.assert_array_equal(read_data, cast_data)
def test_complete(tmpdir):
runner = CliRunner()
result = runner.invoke(cli, ['complete',
'-mm', 100,
'-zresolution', 0.1,
'-filter', 'area > 20.5 and maxdepth > 0.5 or volume > 2.5',
'-dem', dtmfile,
'-outdir', str(tmpdir)])
assert result.exit_code == 0, result.output
assert os.path.isfile(str(tmpdir.join('filled.tif')))
r = io.RasterReader(str(tmpdir.join('bluespots.tif')))
data = r.read()
assert np.max(data) == 486, result.output
v = io.VectorReader(str(tmpdir.join('malstroem.gpkg')), 'finalstate')
data = v.read_geojson_features()
assert len(data) == 544, result.output
v = io.VectorReader(str(tmpdir.join('malstroem.gpkg')), 'finalbluespots')
data = v.read_geojson_features()
assert len(data) == 500, result.output
def process_all(dem, outdir, accum, filter, rain, vector):
"""Quick option to run all processes.
\b
Example:
malstroem complete -r 10 -r 30 -filter "volume > 2.5" -dem dem.tif -outdir ./outdir/
"""
# Check that outdir exists and is empty
if not os.path.isdir(outdir) or not os.path.exists(outdir) or os.listdir(outdir):
logger.error("outdir isn't an empty directory")
return 1
#outvector = os.path.join(outdir, 'malstroem.gpkg')
outvector = os.path.join(outdir, 'vector')
#ogr_drv = 'gpkg'
ogr_dsco = []
ogr_drv = 'ESRI shapefile'
nodatasubst = -999
filter_function = parse_filter(filter)
dem_reader = io.RasterReader(dem, nodatasubst=nodatasubst)
tr = dem_reader.transform
crs = dem_reader.crs
logger.info('Processing')
logger.info(' dem: {}'.format(dem))
logger.info(' outdir: {}'.format(outdir))
logger.info(' rain: {}'.format(', '.join(['{}mm'.format(r) for r in rain])))
logger.info(' accum: {}'.format(accum))
logger.info(' filter: {}'.format(filter))
# Process DEM
filled_writer = io.RasterWriter(os.path.join(outdir, 'filled.tif'), tr, crs, nodatasubst)
flowdir_writer = io.RasterWriter(os.path.join(outdir, 'flowdir.tif'), tr, crs)
depths_writer = io.RasterWriter(os.path.join(outdir, 'bs_depths.tif'), tr, crs)
accum_writer = io.RasterWriter(os.path.join(outdir, 'accum.tif'), tr, crs) if accum else None
dtmtool = demtool.DemTool(dem_reader, filled_writer, flowdir_writer, depths_writer, accum_writer)
dtmtool.process()
# Process bluespots
depths_reader = io.RasterReader(depths_writer.filepath)
flowdir_reader = io.RasterReader(flowdir_writer.filepath)
accum_reader = io.RasterReader(accum_writer.filepath) if accum_writer else None
pourpoint_writer = io.VectorWriter(ogr_drv, outvector, 'pourpoints', None, ogr.wkbPoint, crs, dsco=ogr_dsco)
watershed_writer = io.RasterWriter(os.path.join(outdir, 'watersheds.tif'), tr, crs, 0)
watershed_vector_writer = io.VectorWriter(ogr_drv, outvector, 'watersheds', None, ogr.wkbMultiPolygon, crs, dsco=ogr_dsco) if vector else None
labeled_writer = io.RasterWriter(os.path.join(outdir, 'bluespots.tif'), tr, crs, 0)
labeled_vector_writer = io.VectorWriter(ogr_drv, outvector, 'bluespots', None, ogr.wkbMultiPolygon, crs, dsco=ogr_dsco) if vector else None
bluespot_tool = bluespots.BluespotTool(
input_depths=depths_reader,
input_flowdir=flowdir_reader,
input_bluespot_filter_function=filter_function,
input_accum=accum_reader,
input_dem=dem_reader,
output_labeled_raster=labeled_writer,
output_labeled_vector=labeled_vector_writer,
output_pourpoints=pourpoint_writer,
output_watersheds_raster=watershed_writer,
output_watersheds_vector=watershed_vector_writer
)
bluespot_tool.process()
# Process pourpoints
pourpoints_reader = io.VectorReader(outvector, pourpoint_writer.layername)
bluespot_reader = io.RasterReader(labeled_writer.filepath)
flowdir_reader = io.RasterReader(flowdir_writer.filepath)
nodes_writer = io.VectorWriter(ogr_drv, outvector, 'nodes', None, ogr.wkbPoint, crs, dsco=ogr_dsco)
streams_writer = io.VectorWriter(ogr_drv, outvector, 'streams', None, ogr.wkbLineString, crs, dsco=ogr_dsco)
stream_tool = streams.StreamTool(pourpoints_reader, bluespot_reader, flowdir_reader, nodes_writer, streams_writer)
stream_tool.process()
# Process rain events
nodes_reader = io.VectorReader(outvector, nodes_writer.layername)
events_writer = io.VectorWriter(ogr_drv, outvector, 'events', None, ogr.wkbPoint, crs, dsco=ogr_dsco)
rain_tool = raintool.RainTool(nodes_reader, events_writer, rain)
rain_tool.process()
def process_all(dem, outdir, accum, filter, mm, zresolution, vector):
"""Quick option to run all processes.
\b
Example:
malstroem complete -mm 20 -filter "volume > 2.5" -dem dem.tif -zresolution 0.1 -outdir ./outdir/
"""
# Check that outdir exists and is empty
if not os.path.isdir(outdir) or not os.path.exists(outdir) or os.listdir(
outdir):
logger.error("outdir isn't an empty directory")
return 1
outvector = os.path.join(outdir, 'malstroem.gpkg')
ogr_drv = 'gpkg'
ogr_dsco = []
ogr_lco = ["SPATIAL_INDEX=NO"]
nodatasubst = -999
filter_function = parse_filter(filter)
dem_reader = io.RasterReader(dem, nodatasubst=nodatasubst)
tr = dem_reader.transform
crs = dem_reader.crs
logger.info('Processing')
logger.info(' dem: {}'.format(dem))
logger.info(' outdir: {}'.format(outdir))
logger.info(' mm: {}mm'.format(mm))
logger.info(' zresolution: {}m'.format(zresolution))
logger.info(' accum: {}'.format(accum))
logger.info(' filter: {}'.format(filter))
# Process DEM
filled_writer = io.RasterWriter(os.path.join(outdir, 'filled.tif'), tr,
crs, nodatasubst)
flowdir_writer = io.RasterWriter(os.path.join(outdir, 'flowdir.tif'), tr,
crs)
depths_writer = io.RasterWriter(os.path.join(outdir, 'bs_depths.tif'), tr,
crs)
accum_writer = io.RasterWriter(os.path.join(outdir, 'accum.tif'), tr,
crs) if accum else None
dtmtool = demtool.DemTool(dem_reader, filled_writer, flowdir_writer,
depths_writer, accum_writer)
dtmtool.process()
# Process bluespots
depths_reader = io.RasterReader(depths_writer.filepath)
flowdir_reader = io.RasterReader(flowdir_writer.filepath)
accum_reader = io.RasterReader(
accum_writer.filepath) if accum_writer else None
pourpoint_writer = io.VectorWriter(ogr_drv,
outvector,
'pourpoints',
None,
ogr.wkbPoint,
crs,
dsco=ogr_dsco,
lco=ogr_lco)
watershed_writer = io.RasterWriter(os.path.join(outdir, 'watersheds.tif'),
tr, crs, 0)
watershed_vector_writer = io.VectorWriter(ogr_drv,
outvector,
'watersheds',
None,
ogr.wkbMultiPolygon,
crs,
dsco=ogr_dsco,
lco=ogr_lco) if vector else None
labeled_writer = io.RasterWriter(os.path.join(outdir, 'bluespots.tif'), tr,
crs, 0)
labeled_vector_writer = io.VectorWriter(ogr_drv,
outvector,
'bluespots',
None,
ogr.wkbMultiPolygon,
crs,
dsco=ogr_dsco,
lco=ogr_lco) if vector else None
bluespot_tool = bluespots.BluespotTool(
input_depths=depths_reader,
input_flowdir=flowdir_reader,
input_bluespot_filter_function=filter_function,
input_accum=accum_reader,
input_dem=dem_reader,
output_labeled_raster=labeled_writer,
output_labeled_vector=labeled_vector_writer,
output_pourpoints=pourpoint_writer,
output_watersheds_raster=watershed_writer,
output_watersheds_vector=watershed_vector_writer)
bluespot_tool.process()
# Process pourpoints
pourpoints_reader = io.VectorReader(outvector, pourpoint_writer.layername)
bluespot_reader = io.RasterReader(labeled_writer.filepath)
flowdir_reader = io.RasterReader(flowdir_writer.filepath)
nodes_writer = io.VectorWriter(ogr_drv,
outvector,
'nodes',
None,
ogr.wkbPoint,
crs,
dsco=ogr_dsco,
lco=ogr_lco)
streams_writer = io.VectorWriter(ogr_drv,
outvector,
'streams',
None,
ogr.wkbLineString,
crs,
dsco=ogr_dsco,
lco=ogr_lco)
stream_tool = streams.StreamTool(pourpoints_reader, bluespot_reader,
flowdir_reader, nodes_writer,
streams_writer)
stream_tool.process()
# Calculate volumes
nodes_reader = io.VectorReader(outvector, nodes_writer.layername)
volumes_writer = io.VectorWriter(ogr_drv,
outvector,
'initvolumes',
None,
ogr.wkbPoint,
crs,
dsco=ogr_dsco,
lco=ogr_lco)
rain_tool = raintool.SimpleVolumeTool(nodes_reader, volumes_writer,
"inputv", mm)
rain_tool.process()
# Process final state
volumes_reader = io.VectorReader(outvector, volumes_writer.layername)
events_writer = io.VectorWriter(ogr_drv,
outvector,
'finalstate',
None,
ogr.wkbPoint,
crs,
dsco=ogr_dsco,
lco=ogr_lco)
calculator = network.FinalStateCalculator(volumes_reader, "inputv",
events_writer)
calculator.process()
# Hypsometry
pourpoints_reader = io.VectorReader(outvector, pourpoint_writer.layername)
hyps_writer = io.VectorWriter(ogr_drv, outvector, "hypsometry", None,
ogr.wkbNone, dem_reader.crs)
hyps.bluespot_hypsometry_io(bluespot_reader, dem_reader, pourpoints_reader,
zresolution, hyps_writer)
# Approximation on levels
finalvols_reader = io.VectorReader(outvector, events_writer.layername)
hyps_reader = io.VectorReader(outvector, hyps_writer.layername)
levels_writer = io.VectorWriter(ogr_drv, outvector, "finallevels", None,
ogr.wkbNone, dem_reader.crs)
approx.approx_water_level_io(finalvols_reader, hyps_reader, levels_writer)
# Approximation on bluespots
levels_reader = io.VectorReader(outvector, levels_writer.layername)
final_depths_writer = io.RasterWriter(
os.path.join(outdir, 'finaldepths.tif'), tr, crs)
final_bs_writer = io.RasterWriter(
os.path.join(outdir, 'finalbluespots.tif'), tr, crs, 0)
approx.approx_bluespots_io(bluespot_reader, levels_reader, dem_reader,
final_depths_writer, final_bs_writer)
# Polygonize final bluespots
logger.info("Polygonizing final bluespots")
vectorize_labels_file_io(final_bs_writer.filepath, outvector,
"finalbluespots", ogr_drv, ogr_dsco, ogr_lco)
logger.info("Complete done...")
