def test_dem_invalid_resample1():
with pytest.raises(ValueError):
bounds = [1046891, 704778, 1055345, 709629]
bcdata.get_dem(bounds,
"test_dem.tif",
interpolation="cubic",
resolution=50)
def dem(bounds, dst_crs, out_file, resolution):
"""Dump BC DEM to TIFF
"""
if not dst_crs:
dst_crs = "EPSG:3005"
bcdata.get_dem(bounds,
dst_crs=dst_crs,
out_file=out_file,
resolution=resolution)
def dem(bounds, bounds_crs, dst_crs, out_file, resolution, interpolation,
verbose, quiet):
"""Dump BC DEM to TIFF
"""
verbosity = verbose - quiet
configure_logging(verbosity)
if not dst_crs:
dst_crs = "EPSG:3005"
bcdata.get_dem(
bounds,
out_file=out_file,
src_crs=bounds_crs,
dst_crs=dst_crs,
resolution=resolution,
interpolation=interpolation,
)
def test_dem_rasterio(tmpdir):
bounds = [1046891, 704778, 1055345, 709629]
src = bcdata.get_dem(bounds, as_rasterio=True)
stats = [{
"min": float(b.min()),
"max": float(b.max()),
"mean": float(b.mean())
} for b in src.read()]
assert stats[0]["max"] == 3982
def test_dem(tmpdir):
bounds = [1046891, 704778, 1055345, 709629]
out_file = bcdata.get_dem(bounds, os.path.join(tmpdir, "test_dem.tif"))
assert os.path.exists(out_file)
with rasterio.open(out_file) as src:
stats = [{
"min": float(b.min()),
"max": float(b.max()),
"mean": float(b.mean())
} for b in src.read()]
assert stats[0]["max"] == 3982
def test_dem():
bounds = [1046891, 704778, 1055345, 709629]
out_file = bcdata.get_dem(bounds, "test_dem.tif")
assert os.path.exists("test_dem.tif")
with rasterio.open("test_dem.tif") as src:
stats = [{
'min': float(b.min()),
'max': float(b.max()),
'mean': float(b.mean())
} for b in src.read()]
assert stats[0]['max'] == 3982
os.remove("test_dem.tif")
def test_dem_invalid_resample2():
with pytest.raises(ValueError):
bounds = [1046891, 704778, 1055345, 709629]
bcdata.get_dem(bounds, "test_dem.tif", interpolation="bilinear")
def load(self, overwrite=False):
"""Load input data, do all model calculations and filters"""
# shortcuts
config = self.config
data = self.data
# note workspace
LOG.info("Temp data are here: {}".format(self.config["temp_folder"]))
# create dict that maps beclabel to becvalue
uniques = (data["elevation"][["beclabel", "becvalue"
]].drop_duplicates().to_dict("list"))
self.becvalue_lookup = {}
for i, v in enumerate(uniques["beclabel"]):
self.becvalue_lookup[v] = uniques["becvalue"][i]
# create a reverse lookup
self.beclabel_lookup = {
value: key
for key, value in self.becvalue_lookup.items()
}
# add zeros to reverse lookup
self.beclabel_lookup[0] = None
# convert slope dependent filter sizes from m to cells
self.filtersize_low = ceil(
(config["majority_filter_size_slope_low_metres"] /
config["cell_size_metres"]))
self.filtersize_steep = ceil(
(config["majority_filter_size_slope_steep_metres"] /
config["cell_size_metres"]))
# get bounds from gdf and bump out by specified expansion
bounds = list(data["rulepolys"].geometry.total_bounds)
xmin = bounds[0] - config["expand_bounds_metres"]
ymin = bounds[1] - config["expand_bounds_metres"]
xmax = bounds[2] + config["expand_bounds_metres"]
ymax = bounds[3] + config["expand_bounds_metres"]
expanded_bounds = (xmin, ymin, xmax, ymax)
# align to Hectares BC raster
data["bounds"] = util.align(expanded_bounds)
LOG.info("Bounds: " + " ".join([str(b) for b in data["bounds"]]))
# confirm workspace exists, overwrite if specified
if overwrite and os.path.exists(config["wksp"]):
shutil.rmtree(config["wksp"])
# create workspace and a subfolder for the non-numbered rasters
# (so that they can be cached for repeated model runs on the same
# study area)
srcpath = os.path.join(config["wksp"], "src")
Path(srcpath).mkdir(parents=True, exist_ok=True)
# do bounds extend outside of BC?
bounds_ll = transform_bounds("EPSG:3005", "EPSG:4326", *data["bounds"])
bounds_gdf = util.bbox2gdf(bounds_ll).set_crs("EPSG:4326")
# load neighbours
# Note that the natural earth dataset is only 1:10m,
# buffer it by 2km to be sure it captures the edge of the province
nbr = (gpd.read_file(
os.path.join(os.path.dirname(__file__),
"data/neighbours.geojson")).dissolve(by="scalerank"))
nbr = nbr.to_crs("EPSG:3005").buffer(2000).to_crs("EPSG:4326")
neighbours = (gpd.GeoDataFrame(nbr).rename(columns={
0: "geometry"
}).set_geometry("geometry"))
outside_bc = gpd.overlay(neighbours, bounds_gdf, how="intersection")
# use file based dem if provided in config
if config["dem"]:
LOG.info("Using DEM: {}".format(config["dem"]))
self.dempath = config["dem"]
else:
self.dempath = os.path.join(srcpath, "dem.tif")
# We cache the result of WCS / terraintiles requests, so only
# rerun if the file is not present
dem_bc = os.path.join(srcpath, "dem_bc.tif")
dem_exbc = os.path.join(srcpath, "dem_exbc.tif")
if not config["dem"]:
# get TRIM dem
if not os.path.exists(dem_bc):
LOG.info("Downloading and processing BC DEM")
# request at native resolution and resample locally
# because requesting a bilinear resampled DEM is slow
bcdata.get_dem(data["bounds"],
os.path.join(srcpath, "dem_bc25.tif"),
resolution=25)
# resample if "cell_size_metres" is not 25m
if config["cell_size_metres"] != 25:
LOG.info("Resampling BC DEM")
cmd = [
"gdalwarp",
"-r",
"bilinear",
"-tr",
str(config["cell_size_metres"]),
str(config["cell_size_metres"]),
os.path.join(srcpath, "dem_bc25.tif"),
dem_bc,
]
subprocess.run(cmd)
# otherwise, just rename
else:
LOG.info("xxx")
os.rename(os.path.join(srcpath, "dem_bc25.tif"), dem_bc)
# if not requesting terrain-tiles, again just rename the bc dem
if outside_bc.empty is True:
LOG.info("yyy")
os.rename(dem_bc, self.dempath)
# get terrain-tiles
# - if the bbox does extend outside of BC
# - if _exbc file is not already present
if not os.path.exists(dem_exbc) and not outside_bc.empty:
# find path to cached terrain-tiles
if "TERRAINCACHE" in os.environ.keys():
terraincache_path = os.environ["TERRAINCACHE"]
else:
terraincache_path = os.path.join(config["wksp"],
"terrain-tiles")
LOG.info(
"Study area bounding box extends outside of BC, using MapZen terrain tiles to fill gaps"
)
tt = TerrainTiles(
data["bounds"],
11,
cache_dir=terraincache_path,
bounds_crs="EPSG:3005",
dst_crs="EPSG:3005",
resolution=config["cell_size_metres"],
)
tt.save(out_file=dem_exbc)
# combine the sources
a = rasterio.open(dem_bc)
b = rasterio.open(dem_exbc)
mosaic, out_trans = riomerge([b, a])
out_meta = a.meta.copy()
out_meta.update({
"driver": "GTiff",
"height": mosaic.shape[1],
"width": mosaic.shape[2],
"transform": out_trans,
"crs": "EPSG:3005",
})
# write merged tiff
with rasterio.open(self.dempath, "w", **out_meta) as dest:
dest.write(mosaic)
# ----------------------------------------------------------------
# DEM processing
# ----------------------------------------------------------------
# load dem into memory and get the shape / transform
with rasterio.open(self.dempath) as src:
self.shape = src.shape
self.transform = src.transform
data["dem"] = src.read(1)
# generate slope and aspect
if not os.path.exists(os.path.join(srcpath, "slope.tif")):
gdal.DEMProcessing(
os.path.join(srcpath, "slope.tif"),
self.dempath,
"slope",
slopeFormat="percent",
)
if not os.path.exists(os.path.join(srcpath, "aspect.tif")):
gdal.DEMProcessing(os.path.join(srcpath, "aspect.tif"),
self.dempath, "aspect")
# load slope from file
with rasterio.open(os.path.join(srcpath, "slope.tif")) as src:
data["slope"] = src.read(1)
# load aspect and convert to unsigned integer
with rasterio.open(os.path.join(srcpath, "aspect.tif")) as src:
data["aspect"] = src.read(1).astype(np.uint16)
# We consider slopes less that 15% to be neutral.
# Set aspect to aspect_midpoint_neutral_east (ie, typically 90 degrees)
data["aspect"][
data["slope"] <
config["aspect_neutral_slope_threshold_percent"]] = config[
"aspect_midpoint_neutral_east_degrees"]
# ----------------------------------------------------------------
# convert rule polygons to raster and expand the outer rule bounds
# ----------------------------------------------------------------
# load to raster
rules = features.rasterize(
((geom, value) for geom, value in zip(
data["rulepolys"].geometry, data["rulepolys"].polygon_number)),
out_shape=self.shape,
transform=self.transform,
all_touched=False,
dtype=np.uint16,
)
# create binary inverted rules image to calc distance from
# zero values to a rule poly
a = np.where(rules == 0, 1, 0)
# compute the distance (euclidian distance in cell units)
# and also return the index of the nearest rule poly
# (allowing us to perform 'Euclidean Allocation')
b, c = ndimage.distance_transform_edt(a, return_indices=True)
# extract only the part of the feature transform within
# our expansion distance
expand_bounds_cells = ceil(
(config["expand_bounds_metres"] / config["cell_size_metres"]))
data["ruleimg"] = np.where(b < expand_bounds_cells, rules[c[0], c[1]],
0)
self.data = data
def create_watersheds(in_file, in_id, in_name=None, in_layer=None, points_only=None):
"""Get watershed boundaries upstream of provided points
"""
# load input points
in_points = []
in_points = geopandas.read_file(in_file, layer=in_layer)
# This just makes things simpler.
if in_points.crs.to_epsg() != 3005:
return "Input points must be BC Albers"
# iterate through input points
for index, pt in in_points.iterrows():
click.echo("-----------------------------------------------------------")
click.echo("* INPUT POINT")
click.echo(pt)
# create temp folder structure
temp_folder = os.path.join("tempfiles", "t_" + str(pt[in_id]))
Path(temp_folder).mkdir(parents=True, exist_ok=True)
# find 10 closest streams in BC, within 500m
nearest_streams = fwa_indexpoint(
pt.geometry.x,
pt.geometry.y,
3005,
tolerance=500,
num_features=10,
as_gdf=True,
)
# The closest stream is not necessarily the one we want!
# If we have a name column to compare against, try getting the best combination
# of name and distance matching by comparing to the stream gnis_name
if not nearest_streams.empty:
if in_name:
matched_stream = distance_name_match(nearest_streams, pt[in_name])
# if no name provided, just use the first result
else:
matched_stream = nearest_streams.head(1)
# simplify the schema for standardization between BC/USA
matched_stream = matched_stream.drop(
["wscode_ltree", "localcode_ltree", "linear_feature_id"], axis=1
)
matched_stream["comid"] = ""
# try the EPA service if:
# - no results from fwa_indexpoint() or
# - fwa_indexpoint() says notbc and point is >150m from stream
if nearest_streams.empty or (
matched_stream.iloc[0]["bc_ind"] is False
and matched_stream.iloc[0]["distance_to_stream"] >= 150
):
matched_stream = epa_index_point(
pt.geometry.x, pt.geometry.y, 3005, 150, as_gdf=True
)
if not matched_stream.empty:
# add input id column and value to point
matched_stream.at[0, in_id] = pt[in_id]
# write indexed point to shp
matched_stream.to_file(os.path.join(temp_folder, "point.shp"))
# drop geom for easy dump to stdout so user know what stream we've matched to
click.echo("")
click.echo("* MATCHED STREAM")
click.echo(pprint(matched_stream.iloc[0].drop("geometry").to_dict()))
# extract the required values from matched_stream gdf, just to
# keep code below tidier
blue_line_key = matched_stream.iloc[0]["blue_line_key"]
downstream_route_measure = matched_stream.iloc[0]["downstream_route_measure"]
comid = matched_stream.iloc[0]["comid"]
# if not just indexing points, start deriving the watershed
if not points_only:
# Canadian streams
if matched_stream.iloc[0]["bc_ind"] != "USA":
wsd = fwa_watershedatmeasure(
blue_line_key, downstream_route_measure, as_gdf=True
)
# USA streams (only lower 48 states supported)
else:
wsd = epa_delineate_watershed(
comid, downstream_route_measure, as_gdf=True
)
# if we have a wsd poly, add id and write to shape
if not wsd.empty:
wsd.at[0, in_id] = pt[in_id]
wsd.to_file(os.path.join(temp_folder, "wsd.shp"))
# We are presuming that if nothing is returned from the
# FWA_WatershedAtMeasure call, DEM postprocessing is required.
# (to handle cases where a point is in a watershed with nothing
# else upstream). This is only true because we are only matching to
# streams in BC and lower 48 - there should not be any other
# cases where the wsd gdf is empty
else:
wsd = pandas.DataFrame(data={'refine_method': ["DEM"]})
# if we are postprocessing with DEM, get additional data
if wsd.iloc[0]["refine_method"] == "DEM":
click.echo("requesting additional data for {}".format(pt[in_id]))
# fwapg requests
hexgrid = fwa_watershedhex(
blue_line_key, downstream_route_measure, as_gdf=True
)
hexgrid.to_file(os.path.join(temp_folder, "hexgrid.shp"))
pourpoints = fwa_watershedstream(
blue_line_key, downstream_route_measure, as_gdf=True
)
pourpoints.to_file(os.path.join(temp_folder, "pourpoints.shp"))
# DEM of hex watershed plus 250m
bounds = list(hexgrid.geometry.total_bounds)
expansion = 250
xmin = bounds[0] - expansion
ymin = bounds[1] - expansion
xmax = bounds[2] + expansion
ymax = bounds[3] + expansion
expanded_bounds = (xmin, ymin, xmax, ymax)
bcdata.get_dem(
expanded_bounds,
out_file=os.path.join(temp_folder, "dem.tif"),
src_crs="EPSG:3005",
dst_crs="EPSG:3005",
resolution=25,
)
else:
click.echo("")
click.echo("NO MATCHED STREAM - IS POINT IN BC or USA LOWER 48?")
if not points_only:
click.echo("Attempting to process point with hydrosheds data")
click.echo("WARNING - hydroshed boundaries are much lower precision than FWA")
click.echo("WARNING - this script does not refine hydroshed boundaries, all of intersecting polygon is included!")
click.echo("WARNING - if watershed for this point includes areas in BC, the portion of output boundary in BC will not match FWA watershed boundaries!")
wsd = hydroshed(pt.geometry.x, pt.geometry.y, 3005, as_gdf=True)
# if we have a wsd poly, add id and write to shape
if not wsd.empty:
wsd.at[0, in_id] = pt[in_id]
wsd.to_file(os.path.join(temp_folder, "wsd.shp"))