def test_write_vector(tmpdir):
crs = create_utm32_crs()
fields = create_fields()
features = create_geojson()
layername = 'xxx'
outfile = str(tmpdir.join('test.geojson'))
writer = io.VectorWriter('geojson', outfile, layername, fields,
ogr.wkbPoint, crs)
writer.write_geojson_features(features)
del writer
ds, lyr = get_datasource_layer(outfile, None)
fdefn = lyr.GetLayerDefn()
assert lyr is not None
assert lyr.GetGeomType() == ogr.wkbPoint
assert lyr.GetSpatialRef().ExportToWkt() == crs
assert lyr.GetFeatureCount() == len(features['features'])
# Feature defn
assert fdefn.GetFieldCount() == 2
assert len(fields) == 2
# Field 0
for ix, f_exp in enumerate(fields):
f = fdefn.GetFieldDefn(ix)
assert f.GetName() == f_exp.GetName()
def process_network(bluespots, flowdir, pourpoints, pourpoints_layer, out, out_nodes_layer, out_streams_layer, format, dsco, lco):
"""Calculate stream network between bluespots.
For documentation of OGR features (format, dsco and lco) see http://www.gdal.org/ogr_formats.html
"""
pourpoints_reader = io.VectorReader(pourpoints, str(pourpoints_layer))
bluespot_reader = io.RasterReader(bluespots)
flowdir_reader = io.RasterReader(flowdir)
format = str(format)
out_nodes_layer = str(out_nodes_layer)
out_streams_layer = str(out_streams_layer)
nodes_writer = io.VectorWriter(format, out, out_nodes_layer, None, ogr.wkbPoint, flowdir_reader.crs, dsco, lco)
streams_writer = io.VectorWriter(format, out, out_streams_layer, None, ogr.wkbLineString, flowdir_reader.crs, dsco, lco)
stream_tool = streams.StreamTool(pourpoints_reader, bluespot_reader, flowdir_reader, nodes_writer, streams_writer)
stream_tool.process()
def test_bluespots(tmpdir):
flowdir_reader = io.RasterReader(flowdirnoflatsfile)
dem_reader = io.RasterReader(dtmfile)
filled_reader = io.RasterReader(filledfile)
depths_reader = NumpyRasterReader(filled_reader.read() - dem_reader.read(),
dem_reader.transform)
outdbfile = str(tmpdir.join('test.gpkg'))
# At least 5cm deep, 5 cells wide and at least one cell-meter volume
filter_function = lambda r: r['max'] > 0.05 and r['count'] > 5 and r['sum'
] > 1
pourpoint_writer = io.VectorWriter('gpkg', outdbfile, 'pourpoints', None,
ogr.wkbPoint, dem_reader.crs)
watershed_writer = io.RasterWriter(str(tmpdir.join('watersheds.tif')),
dem_reader.transform, dem_reader.crs, 0)
watershed_vector_writer = io.VectorWriter('gpkg', outdbfile, 'watersheds',
None, ogr.wkbMultiPolygon,
dem_reader.crs)
labeled_writer = io.RasterWriter(str(tmpdir.join('labeled.tif')),
dem_reader.transform, dem_reader.crs, 0)
labeled_vector_writer = io.VectorWriter('gpkg', outdbfile, 'bluespots',
None, ogr.wkbMultiPolygon,
dem_reader.crs)
bluespot_tool = bluespots.BluespotTool(
input_depths=depths_reader,
input_flowdir=flowdir_reader,
input_bluespot_filter_function=filter_function,
input_accum=None,
input_dem=dem_reader,
output_labeled_raster=labeled_writer,
output_labeled_vector=labeled_vector_writer,
output_pourpoints=pourpoint_writer,
output_watersheds_raster=watershed_writer,
output_watersheds_vector=watershed_vector_writer)
bluespot_tool.process()
assert os.path.isfile(outdbfile)
assert os.path.isfile(watershed_writer.filepath)
assert os.path.isfile(labeled_writer.filepath)
def process_pourpoints(bluespots, depths, watersheds, dem, accum, out, format,
layername, dsco, lco):
"""Determine pour points.
\b
Determines a pour point for each bluespot using one of two methods:
* Random candidate. Requires DEM only
* Maximum accumulated flow candidate. Requires accumulated flow
The output of the two methods only differ when there are more than one pour point candidate (ie multiple threshold
cells with identical Z) for a given bluespot.
For documentation of OGR features (format, dsco and lco) see http://www.gdal.org/ogr_formats.html
"""
bspot_reader = io.RasterReader(bluespots)
depths_reader = io.RasterReader(depths)
wsheds_reader = io.RasterReader(watersheds)
data = accum if accum else dem
if not data:
raise Exception('Either accum or dem must be specified')
data_reader = io.RasterReader(data)
format = str(format)
layername = str(layername)
pourpnt_writer = io.VectorWriter(format, out, layername, [], ogr.wkbPoint,
depths_reader.crs, dsco, lco)
# Recalculate stats on filtered bluespots
labeled_data = bspot_reader.read()
depths_data = depths_reader.read()
bluespot_stats = label.label_stats(depths_data, labeled_data)
del depths_data
if accum:
pp_pix = label.label_max_index(data_reader.read(), labeled_data)
elif dem:
dem_data = data_reader.read()
short, diag = fill.minimum_safe_short_and_diag(dem_data)
filled_no_flats = fill.fill_terrain_no_flats(dem_data, short, diag)
pp_pix = label.label_min_index(filled_no_flats, labeled_data)
del dem_data
watershed_stats = label.label_count(wsheds_reader.read())
pour_points = assemble_pourpoints(depths_reader.transform, pp_pix,
bluespot_stats, watershed_stats)
feature_collection = dict(type="FeatureCollection", features=pour_points)
pourpnt_writer.write_geojson_features(feature_collection)
def process_finallevels(finalvols, finalvols_layer, hyps, hyps_layer, out,
out_layer, format, dsco, lco):
"""Approximate water levels of bluespots in the final state.
This proces assumes that a given bluespot is filled in cell Z order (from lowest to highest cells). No attempt is made
to model how water actually flows within the bluespot.
For documentation of OGR features (format, dsco and lco) see http://www.gdal.org/ogr_formats.html
"""
finalvols_reader = io.VectorReader(finalvols, finalvols_layer)
hyps_reader = io.VectorReader(hyps, hyps_layer)
levels_writer = io.VectorWriter(format, out, out_layer, None, ogr.wkbNone,
finalvols_reader.crs, dsco, lco)
approx.approx_water_level_io(finalvols_reader, hyps_reader, levels_writer)
def process_volumes(nodes, nodes_layer, mm, pr, pr_unit, bluespots, out,
out_layer, out_attribute, format, dsco, lco):
"""Set up initial water volumes for each watershed.
The output from this process can be used as input for the finalvolumes calculation.
\b
Examples:
malstroem initvolumes -mm 10 -nodes results.gpkg -out results.gpkg -format gpkg
malstroem initvolumes -pr precip_raster.tif -bluespots bluespots.tif -nodes results.gpkg -out results.gpkg -format gpkg
For documentation of OGR features (format, dsco and lco) see http://www.gdal.org/ogr_formats.html
"""
# Validate one and only one
if mm and pr:
raise click.UsageError("-mm and -pr are mutually exclusive")
if not mm and not pr:
raise click.UsageError("One of -mm and -pr must be specified")
nodes_layer = nodes_layer
format = str(format)
out_layer = str(out_layer)
# Calculate volumes
nodes_reader = io.VectorReader(nodes, nodes_layer)
volumes_writer = io.VectorWriter(format, out, out_layer, None,
ogr.wkbPoint, nodes_reader.crs, dsco, lco)
if mm:
logger.info(
f"Processing initial volumes using evenly distributed rain event of {mm}mm"
)
rain_tool = raintool.SimpleVolumeTool(nodes_reader, volumes_writer,
out_attribute, mm)
rain_tool.process()
else:
if not bluespots:
raise click.UsageError("Mising -bluespots")
logger.info(
f"Processing initial volumes using [{pr_unit}] raster input from {pr}"
)
precip_reader = io.RasterReader(pr)
bspotlabels_reader = io.RasterReader(bluespots)
tool = raintool.RasterVolumeTool(nodes_reader, bspotlabels_reader,
precip_reader, pr_unit,
volumes_writer, out_attribute)
tool.process()
def process_rain(nodes, nodes_layer, rain, out, out_layer, format, dsco, lco):
"""Calculate bluespot fill and spill volumes for specific rain event.
The rain event is evenly distributed across the entire area.
Note that multiple rain events can be calculated at once by repeating the '-r' option.
\b
Example:
malstroem rain -r 10 -r 30 -nodes results.gpkg -out results.gpkg -format gpkg
For documentation of OGR features (format, dsco and lco) see http://www.gdal.org/ogr_formats.html
"""
nodes_layer = str(nodes_layer)
format = str(format)
out_layer = str(out_layer)
nodes_reader = io.VectorReader(nodes, nodes_layer)
events_writer = io.VectorWriter(format, out, out_layer, None, ogr.wkbPoint,
nodes_reader.crs, dsco, lco)
rain_tool = raintool.RainTool(nodes_reader, events_writer, rain)
rain_tool.process()
def process_hypsometry(bluespots, dem, pourpoints, pourpoints_layer,
zresolution, out, out_hyps_layer, format, dsco, lco):
"""Statistical terrain elevation measures for each bluespot.
For each bluespot these values describing the terrain within the bluespot are returned:
- A DEM Z value histogram with user definable bin width (resolution)
- Number og bins, effective upper and lower bounds of the histogram
- Actual minimum and maximum Z values
The values of the histogram are formatted as a single string using pipe '|' as seperator. Like:
2|1|0|3
For documentation of OGR features (format, dsco and lco) see http://www.gdal.org/ogr_formats.html
"""
pourpoints_reader = io.VectorReader(pourpoints, pourpoints_layer)
labeled_reader = io.RasterReader(bluespots)
dem_reader = io.RasterReader(dem, nodatasubst=NODATASUBST)
ogr_format = str(format)
hyps_writer = io.VectorWriter(ogr_format, out, out_hyps_layer, None,
ogr.wkbNone, dem_reader.crs)
hyps.bluespot_hypsometry_io(labeled_reader, dem_reader, pourpoints_reader,
zresolution, hyps_writer)
def process_net(inputvolumes, inputvolumes_layer, attribute, out, out_layer,
format, dsco, lco):
"""Bluespot fill and stream network volumes in the final state of an event.
The rain event is defined by the initial water volumes per node.
\b
Example:
malstroem finalvolumes -inputvolumes results.gpkg -out results.gpkg -format gpkg
For documentation of OGR features (format, dsco and lco) see http://www.gdal.org/ogr_formats.html
"""
inputvolumes_layer = inputvolumes_layer
format = str(format)
out_layer = str(out_layer)
volumes_reader = io.VectorReader(inputvolumes, inputvolumes_layer)
events_writer = io.VectorWriter(format, out, out_layer, None, ogr.wkbPoint,
volumes_reader.crs, dsco, lco)
# Process events
calculator = network.FinalStateCalculator(volumes_reader, attribute,
events_writer)
calculator.process()
def process_all(dem, outdir, accum, filter, rain, vector):
"""Quick option to run all processes.
\b
Example:
malstroem complete -r 10 -r 30 -filter "volume > 2.5" -dem dem.tif -outdir ./outdir/
"""
# Check that outdir exists and is empty
if not os.path.isdir(outdir) or not os.path.exists(outdir) or os.listdir(outdir):
logger.error("outdir isn't an empty directory")
return 1
#outvector = os.path.join(outdir, 'malstroem.gpkg')
outvector = os.path.join(outdir, 'vector')
#ogr_drv = 'gpkg'
ogr_dsco = []
ogr_drv = 'ESRI shapefile'
nodatasubst = -999
filter_function = parse_filter(filter)
dem_reader = io.RasterReader(dem, nodatasubst=nodatasubst)
tr = dem_reader.transform
crs = dem_reader.crs
logger.info('Processing')
logger.info(' dem: {}'.format(dem))
logger.info(' outdir: {}'.format(outdir))
logger.info(' rain: {}'.format(', '.join(['{}mm'.format(r) for r in rain])))
logger.info(' accum: {}'.format(accum))
logger.info(' filter: {}'.format(filter))
# Process DEM
filled_writer = io.RasterWriter(os.path.join(outdir, 'filled.tif'), tr, crs, nodatasubst)
flowdir_writer = io.RasterWriter(os.path.join(outdir, 'flowdir.tif'), tr, crs)
depths_writer = io.RasterWriter(os.path.join(outdir, 'bs_depths.tif'), tr, crs)
accum_writer = io.RasterWriter(os.path.join(outdir, 'accum.tif'), tr, crs) if accum else None
dtmtool = demtool.DemTool(dem_reader, filled_writer, flowdir_writer, depths_writer, accum_writer)
dtmtool.process()
# Process bluespots
depths_reader = io.RasterReader(depths_writer.filepath)
flowdir_reader = io.RasterReader(flowdir_writer.filepath)
accum_reader = io.RasterReader(accum_writer.filepath) if accum_writer else None
pourpoint_writer = io.VectorWriter(ogr_drv, outvector, 'pourpoints', None, ogr.wkbPoint, crs, dsco=ogr_dsco)
watershed_writer = io.RasterWriter(os.path.join(outdir, 'watersheds.tif'), tr, crs, 0)
watershed_vector_writer = io.VectorWriter(ogr_drv, outvector, 'watersheds', None, ogr.wkbMultiPolygon, crs, dsco=ogr_dsco) if vector else None
labeled_writer = io.RasterWriter(os.path.join(outdir, 'bluespots.tif'), tr, crs, 0)
labeled_vector_writer = io.VectorWriter(ogr_drv, outvector, 'bluespots', None, ogr.wkbMultiPolygon, crs, dsco=ogr_dsco) if vector else None
bluespot_tool = bluespots.BluespotTool(
input_depths=depths_reader,
input_flowdir=flowdir_reader,
input_bluespot_filter_function=filter_function,
input_accum=accum_reader,
input_dem=dem_reader,
output_labeled_raster=labeled_writer,
output_labeled_vector=labeled_vector_writer,
output_pourpoints=pourpoint_writer,
output_watersheds_raster=watershed_writer,
output_watersheds_vector=watershed_vector_writer
)
bluespot_tool.process()
# Process pourpoints
pourpoints_reader = io.VectorReader(outvector, pourpoint_writer.layername)
bluespot_reader = io.RasterReader(labeled_writer.filepath)
flowdir_reader = io.RasterReader(flowdir_writer.filepath)
nodes_writer = io.VectorWriter(ogr_drv, outvector, 'nodes', None, ogr.wkbPoint, crs, dsco=ogr_dsco)
streams_writer = io.VectorWriter(ogr_drv, outvector, 'streams', None, ogr.wkbLineString, crs, dsco=ogr_dsco)
stream_tool = streams.StreamTool(pourpoints_reader, bluespot_reader, flowdir_reader, nodes_writer, streams_writer)
stream_tool.process()
# Process rain events
nodes_reader = io.VectorReader(outvector, nodes_writer.layername)
events_writer = io.VectorWriter(ogr_drv, outvector, 'events', None, ogr.wkbPoint, crs, dsco=ogr_dsco)
rain_tool = raintool.RainTool(nodes_reader, events_writer, rain)
rain_tool.process()
def process_all(dem, outdir, accum, filter, mm, zresolution, vector):
"""Quick option to run all processes.
\b
Example:
malstroem complete -mm 20 -filter "volume > 2.5" -dem dem.tif -zresolution 0.1 -outdir ./outdir/
"""
# Check that outdir exists and is empty
if not os.path.isdir(outdir) or not os.path.exists(outdir) or os.listdir(
outdir):
logger.error("outdir isn't an empty directory")
return 1
outvector = os.path.join(outdir, 'malstroem.gpkg')
ogr_drv = 'gpkg'
ogr_dsco = []
ogr_lco = ["SPATIAL_INDEX=NO"]
nodatasubst = -999
filter_function = parse_filter(filter)
dem_reader = io.RasterReader(dem, nodatasubst=nodatasubst)
tr = dem_reader.transform
crs = dem_reader.crs
logger.info('Processing')
logger.info(' dem: {}'.format(dem))
logger.info(' outdir: {}'.format(outdir))
logger.info(' mm: {}mm'.format(mm))
logger.info(' zresolution: {}m'.format(zresolution))
logger.info(' accum: {}'.format(accum))
logger.info(' filter: {}'.format(filter))
# Process DEM
filled_writer = io.RasterWriter(os.path.join(outdir, 'filled.tif'), tr,
crs, nodatasubst)
flowdir_writer = io.RasterWriter(os.path.join(outdir, 'flowdir.tif'), tr,
crs)
depths_writer = io.RasterWriter(os.path.join(outdir, 'bs_depths.tif'), tr,
crs)
accum_writer = io.RasterWriter(os.path.join(outdir, 'accum.tif'), tr,
crs) if accum else None
dtmtool = demtool.DemTool(dem_reader, filled_writer, flowdir_writer,
depths_writer, accum_writer)
dtmtool.process()
# Process bluespots
depths_reader = io.RasterReader(depths_writer.filepath)
flowdir_reader = io.RasterReader(flowdir_writer.filepath)
accum_reader = io.RasterReader(
accum_writer.filepath) if accum_writer else None
pourpoint_writer = io.VectorWriter(ogr_drv,
outvector,
'pourpoints',
None,
ogr.wkbPoint,
crs,
dsco=ogr_dsco,
lco=ogr_lco)
watershed_writer = io.RasterWriter(os.path.join(outdir, 'watersheds.tif'),
tr, crs, 0)
watershed_vector_writer = io.VectorWriter(ogr_drv,
outvector,
'watersheds',
None,
ogr.wkbMultiPolygon,
crs,
dsco=ogr_dsco,
lco=ogr_lco) if vector else None
labeled_writer = io.RasterWriter(os.path.join(outdir, 'bluespots.tif'), tr,
crs, 0)
labeled_vector_writer = io.VectorWriter(ogr_drv,
outvector,
'bluespots',
None,
ogr.wkbMultiPolygon,
crs,
dsco=ogr_dsco,
lco=ogr_lco) if vector else None
bluespot_tool = bluespots.BluespotTool(
input_depths=depths_reader,
input_flowdir=flowdir_reader,
input_bluespot_filter_function=filter_function,
input_accum=accum_reader,
input_dem=dem_reader,
output_labeled_raster=labeled_writer,
output_labeled_vector=labeled_vector_writer,
output_pourpoints=pourpoint_writer,
output_watersheds_raster=watershed_writer,
output_watersheds_vector=watershed_vector_writer)
bluespot_tool.process()
# Process pourpoints
pourpoints_reader = io.VectorReader(outvector, pourpoint_writer.layername)
bluespot_reader = io.RasterReader(labeled_writer.filepath)
flowdir_reader = io.RasterReader(flowdir_writer.filepath)
nodes_writer = io.VectorWriter(ogr_drv,
outvector,
'nodes',
None,
ogr.wkbPoint,
crs,
dsco=ogr_dsco,
lco=ogr_lco)
streams_writer = io.VectorWriter(ogr_drv,
outvector,
'streams',
None,
ogr.wkbLineString,
crs,
dsco=ogr_dsco,
lco=ogr_lco)
stream_tool = streams.StreamTool(pourpoints_reader, bluespot_reader,
flowdir_reader, nodes_writer,
streams_writer)
stream_tool.process()
# Calculate volumes
nodes_reader = io.VectorReader(outvector, nodes_writer.layername)
volumes_writer = io.VectorWriter(ogr_drv,
outvector,
'initvolumes',
None,
ogr.wkbPoint,
crs,
dsco=ogr_dsco,
lco=ogr_lco)
rain_tool = raintool.SimpleVolumeTool(nodes_reader, volumes_writer,
"inputv", mm)
rain_tool.process()
# Process final state
volumes_reader = io.VectorReader(outvector, volumes_writer.layername)
events_writer = io.VectorWriter(ogr_drv,
outvector,
'finalstate',
None,
ogr.wkbPoint,
crs,
dsco=ogr_dsco,
lco=ogr_lco)
calculator = network.FinalStateCalculator(volumes_reader, "inputv",
events_writer)
calculator.process()
# Hypsometry
pourpoints_reader = io.VectorReader(outvector, pourpoint_writer.layername)
hyps_writer = io.VectorWriter(ogr_drv, outvector, "hypsometry", None,
ogr.wkbNone, dem_reader.crs)
hyps.bluespot_hypsometry_io(bluespot_reader, dem_reader, pourpoints_reader,
zresolution, hyps_writer)
# Approximation on levels
finalvols_reader = io.VectorReader(outvector, events_writer.layername)
hyps_reader = io.VectorReader(outvector, hyps_writer.layername)
levels_writer = io.VectorWriter(ogr_drv, outvector, "finallevels", None,
ogr.wkbNone, dem_reader.crs)
approx.approx_water_level_io(finalvols_reader, hyps_reader, levels_writer)
# Approximation on bluespots
levels_reader = io.VectorReader(outvector, levels_writer.layername)
final_depths_writer = io.RasterWriter(
os.path.join(outdir, 'finaldepths.tif'), tr, crs)
final_bs_writer = io.RasterWriter(
os.path.join(outdir, 'finalbluespots.tif'), tr, crs, 0)
approx.approx_bluespots_io(bluespot_reader, levels_reader, dem_reader,
final_depths_writer, final_bs_writer)
# Polygonize final bluespots
logger.info("Polygonizing final bluespots")
vectorize_labels_file_io(final_bs_writer.filepath, outvector,
"finalbluespots", ogr_drv, ogr_dsco, ogr_lco)
logger.info("Complete done...")