def test_dem_invalid_resample1(): with pytest.raises(ValueError): bounds = [1046891, 704778, 1055345, 709629] bcdata.get_dem(bounds, "test_dem.tif", interpolation="cubic", resolution=50)
def dem(bounds, dst_crs, out_file, resolution): """Dump BC DEM to TIFF """ if not dst_crs: dst_crs = "EPSG:3005" bcdata.get_dem(bounds, dst_crs=dst_crs, out_file=out_file, resolution=resolution)
def dem(bounds, bounds_crs, dst_crs, out_file, resolution, interpolation, verbose, quiet): """Dump BC DEM to TIFF """ verbosity = verbose - quiet configure_logging(verbosity) if not dst_crs: dst_crs = "EPSG:3005" bcdata.get_dem( bounds, out_file=out_file, src_crs=bounds_crs, dst_crs=dst_crs, resolution=resolution, interpolation=interpolation, )
def test_dem_rasterio(tmpdir): bounds = [1046891, 704778, 1055345, 709629] src = bcdata.get_dem(bounds, as_rasterio=True) stats = [{ "min": float(b.min()), "max": float(b.max()), "mean": float(b.mean()) } for b in src.read()] assert stats[0]["max"] == 3982
def test_dem(tmpdir): bounds = [1046891, 704778, 1055345, 709629] out_file = bcdata.get_dem(bounds, os.path.join(tmpdir, "test_dem.tif")) assert os.path.exists(out_file) with rasterio.open(out_file) as src: stats = [{ "min": float(b.min()), "max": float(b.max()), "mean": float(b.mean()) } for b in src.read()] assert stats[0]["max"] == 3982
def test_dem(): bounds = [1046891, 704778, 1055345, 709629] out_file = bcdata.get_dem(bounds, "test_dem.tif") assert os.path.exists("test_dem.tif") with rasterio.open("test_dem.tif") as src: stats = [{ 'min': float(b.min()), 'max': float(b.max()), 'mean': float(b.mean()) } for b in src.read()] assert stats[0]['max'] == 3982 os.remove("test_dem.tif")
def test_dem_invalid_resample2(): with pytest.raises(ValueError): bounds = [1046891, 704778, 1055345, 709629] bcdata.get_dem(bounds, "test_dem.tif", interpolation="bilinear")
def load(self, overwrite=False): """Load input data, do all model calculations and filters""" # shortcuts config = self.config data = self.data # note workspace LOG.info("Temp data are here: {}".format(self.config["temp_folder"])) # create dict that maps beclabel to becvalue uniques = (data["elevation"][["beclabel", "becvalue" ]].drop_duplicates().to_dict("list")) self.becvalue_lookup = {} for i, v in enumerate(uniques["beclabel"]): self.becvalue_lookup[v] = uniques["becvalue"][i] # create a reverse lookup self.beclabel_lookup = { value: key for key, value in self.becvalue_lookup.items() } # add zeros to reverse lookup self.beclabel_lookup[0] = None # convert slope dependent filter sizes from m to cells self.filtersize_low = ceil( (config["majority_filter_size_slope_low_metres"] / config["cell_size_metres"])) self.filtersize_steep = ceil( (config["majority_filter_size_slope_steep_metres"] / config["cell_size_metres"])) # get bounds from gdf and bump out by specified expansion bounds = list(data["rulepolys"].geometry.total_bounds) xmin = bounds[0] - config["expand_bounds_metres"] ymin = bounds[1] - config["expand_bounds_metres"] xmax = bounds[2] + config["expand_bounds_metres"] ymax = bounds[3] + config["expand_bounds_metres"] expanded_bounds = (xmin, ymin, xmax, ymax) # align to Hectares BC raster data["bounds"] = util.align(expanded_bounds) LOG.info("Bounds: " + " ".join([str(b) for b in data["bounds"]])) # confirm workspace exists, overwrite if specified if overwrite and os.path.exists(config["wksp"]): shutil.rmtree(config["wksp"]) # create workspace and a subfolder for the non-numbered rasters # (so that they can be cached for repeated model runs on the same # study area) srcpath = os.path.join(config["wksp"], "src") Path(srcpath).mkdir(parents=True, exist_ok=True) # do bounds extend outside of BC? bounds_ll = transform_bounds("EPSG:3005", "EPSG:4326", *data["bounds"]) bounds_gdf = util.bbox2gdf(bounds_ll).set_crs("EPSG:4326") # load neighbours # Note that the natural earth dataset is only 1:10m, # buffer it by 2km to be sure it captures the edge of the province nbr = (gpd.read_file( os.path.join(os.path.dirname(__file__), "data/neighbours.geojson")).dissolve(by="scalerank")) nbr = nbr.to_crs("EPSG:3005").buffer(2000).to_crs("EPSG:4326") neighbours = (gpd.GeoDataFrame(nbr).rename(columns={ 0: "geometry" }).set_geometry("geometry")) outside_bc = gpd.overlay(neighbours, bounds_gdf, how="intersection") # use file based dem if provided in config if config["dem"]: LOG.info("Using DEM: {}".format(config["dem"])) self.dempath = config["dem"] else: self.dempath = os.path.join(srcpath, "dem.tif") # We cache the result of WCS / terraintiles requests, so only # rerun if the file is not present dem_bc = os.path.join(srcpath, "dem_bc.tif") dem_exbc = os.path.join(srcpath, "dem_exbc.tif") if not config["dem"]: # get TRIM dem if not os.path.exists(dem_bc): LOG.info("Downloading and processing BC DEM") # request at native resolution and resample locally # because requesting a bilinear resampled DEM is slow bcdata.get_dem(data["bounds"], os.path.join(srcpath, "dem_bc25.tif"), resolution=25) # resample if "cell_size_metres" is not 25m if config["cell_size_metres"] != 25: LOG.info("Resampling BC DEM") cmd = [ "gdalwarp", "-r", "bilinear", "-tr", str(config["cell_size_metres"]), str(config["cell_size_metres"]), os.path.join(srcpath, "dem_bc25.tif"), dem_bc, ] subprocess.run(cmd) # otherwise, just rename else: LOG.info("xxx") os.rename(os.path.join(srcpath, "dem_bc25.tif"), dem_bc) # if not requesting terrain-tiles, again just rename the bc dem if outside_bc.empty is True: LOG.info("yyy") os.rename(dem_bc, self.dempath) # get terrain-tiles # - if the bbox does extend outside of BC # - if _exbc file is not already present if not os.path.exists(dem_exbc) and not outside_bc.empty: # find path to cached terrain-tiles if "TERRAINCACHE" in os.environ.keys(): terraincache_path = os.environ["TERRAINCACHE"] else: terraincache_path = os.path.join(config["wksp"], "terrain-tiles") LOG.info( "Study area bounding box extends outside of BC, using MapZen terrain tiles to fill gaps" ) tt = TerrainTiles( data["bounds"], 11, cache_dir=terraincache_path, bounds_crs="EPSG:3005", dst_crs="EPSG:3005", resolution=config["cell_size_metres"], ) tt.save(out_file=dem_exbc) # combine the sources a = rasterio.open(dem_bc) b = rasterio.open(dem_exbc) mosaic, out_trans = riomerge([b, a]) out_meta = a.meta.copy() out_meta.update({ "driver": "GTiff", "height": mosaic.shape[1], "width": mosaic.shape[2], "transform": out_trans, "crs": "EPSG:3005", }) # write merged tiff with rasterio.open(self.dempath, "w", **out_meta) as dest: dest.write(mosaic) # ---------------------------------------------------------------- # DEM processing # ---------------------------------------------------------------- # load dem into memory and get the shape / transform with rasterio.open(self.dempath) as src: self.shape = src.shape self.transform = src.transform data["dem"] = src.read(1) # generate slope and aspect if not os.path.exists(os.path.join(srcpath, "slope.tif")): gdal.DEMProcessing( os.path.join(srcpath, "slope.tif"), self.dempath, "slope", slopeFormat="percent", ) if not os.path.exists(os.path.join(srcpath, "aspect.tif")): gdal.DEMProcessing(os.path.join(srcpath, "aspect.tif"), self.dempath, "aspect") # load slope from file with rasterio.open(os.path.join(srcpath, "slope.tif")) as src: data["slope"] = src.read(1) # load aspect and convert to unsigned integer with rasterio.open(os.path.join(srcpath, "aspect.tif")) as src: data["aspect"] = src.read(1).astype(np.uint16) # We consider slopes less that 15% to be neutral. # Set aspect to aspect_midpoint_neutral_east (ie, typically 90 degrees) data["aspect"][ data["slope"] < config["aspect_neutral_slope_threshold_percent"]] = config[ "aspect_midpoint_neutral_east_degrees"] # ---------------------------------------------------------------- # convert rule polygons to raster and expand the outer rule bounds # ---------------------------------------------------------------- # load to raster rules = features.rasterize( ((geom, value) for geom, value in zip( data["rulepolys"].geometry, data["rulepolys"].polygon_number)), out_shape=self.shape, transform=self.transform, all_touched=False, dtype=np.uint16, ) # create binary inverted rules image to calc distance from # zero values to a rule poly a = np.where(rules == 0, 1, 0) # compute the distance (euclidian distance in cell units) # and also return the index of the nearest rule poly # (allowing us to perform 'Euclidean Allocation') b, c = ndimage.distance_transform_edt(a, return_indices=True) # extract only the part of the feature transform within # our expansion distance expand_bounds_cells = ceil( (config["expand_bounds_metres"] / config["cell_size_metres"])) data["ruleimg"] = np.where(b < expand_bounds_cells, rules[c[0], c[1]], 0) self.data = data
def create_watersheds(in_file, in_id, in_name=None, in_layer=None, points_only=None): """Get watershed boundaries upstream of provided points """ # load input points in_points = [] in_points = geopandas.read_file(in_file, layer=in_layer) # This just makes things simpler. if in_points.crs.to_epsg() != 3005: return "Input points must be BC Albers" # iterate through input points for index, pt in in_points.iterrows(): click.echo("-----------------------------------------------------------") click.echo("* INPUT POINT") click.echo(pt) # create temp folder structure temp_folder = os.path.join("tempfiles", "t_" + str(pt[in_id])) Path(temp_folder).mkdir(parents=True, exist_ok=True) # find 10 closest streams in BC, within 500m nearest_streams = fwa_indexpoint( pt.geometry.x, pt.geometry.y, 3005, tolerance=500, num_features=10, as_gdf=True, ) # The closest stream is not necessarily the one we want! # If we have a name column to compare against, try getting the best combination # of name and distance matching by comparing to the stream gnis_name if not nearest_streams.empty: if in_name: matched_stream = distance_name_match(nearest_streams, pt[in_name]) # if no name provided, just use the first result else: matched_stream = nearest_streams.head(1) # simplify the schema for standardization between BC/USA matched_stream = matched_stream.drop( ["wscode_ltree", "localcode_ltree", "linear_feature_id"], axis=1 ) matched_stream["comid"] = "" # try the EPA service if: # - no results from fwa_indexpoint() or # - fwa_indexpoint() says notbc and point is >150m from stream if nearest_streams.empty or ( matched_stream.iloc[0]["bc_ind"] is False and matched_stream.iloc[0]["distance_to_stream"] >= 150 ): matched_stream = epa_index_point( pt.geometry.x, pt.geometry.y, 3005, 150, as_gdf=True ) if not matched_stream.empty: # add input id column and value to point matched_stream.at[0, in_id] = pt[in_id] # write indexed point to shp matched_stream.to_file(os.path.join(temp_folder, "point.shp")) # drop geom for easy dump to stdout so user know what stream we've matched to click.echo("") click.echo("* MATCHED STREAM") click.echo(pprint(matched_stream.iloc[0].drop("geometry").to_dict())) # extract the required values from matched_stream gdf, just to # keep code below tidier blue_line_key = matched_stream.iloc[0]["blue_line_key"] downstream_route_measure = matched_stream.iloc[0]["downstream_route_measure"] comid = matched_stream.iloc[0]["comid"] # if not just indexing points, start deriving the watershed if not points_only: # Canadian streams if matched_stream.iloc[0]["bc_ind"] != "USA": wsd = fwa_watershedatmeasure( blue_line_key, downstream_route_measure, as_gdf=True ) # USA streams (only lower 48 states supported) else: wsd = epa_delineate_watershed( comid, downstream_route_measure, as_gdf=True ) # if we have a wsd poly, add id and write to shape if not wsd.empty: wsd.at[0, in_id] = pt[in_id] wsd.to_file(os.path.join(temp_folder, "wsd.shp")) # We are presuming that if nothing is returned from the # FWA_WatershedAtMeasure call, DEM postprocessing is required. # (to handle cases where a point is in a watershed with nothing # else upstream). This is only true because we are only matching to # streams in BC and lower 48 - there should not be any other # cases where the wsd gdf is empty else: wsd = pandas.DataFrame(data={'refine_method': ["DEM"]}) # if we are postprocessing with DEM, get additional data if wsd.iloc[0]["refine_method"] == "DEM": click.echo("requesting additional data for {}".format(pt[in_id])) # fwapg requests hexgrid = fwa_watershedhex( blue_line_key, downstream_route_measure, as_gdf=True ) hexgrid.to_file(os.path.join(temp_folder, "hexgrid.shp")) pourpoints = fwa_watershedstream( blue_line_key, downstream_route_measure, as_gdf=True ) pourpoints.to_file(os.path.join(temp_folder, "pourpoints.shp")) # DEM of hex watershed plus 250m bounds = list(hexgrid.geometry.total_bounds) expansion = 250 xmin = bounds[0] - expansion ymin = bounds[1] - expansion xmax = bounds[2] + expansion ymax = bounds[3] + expansion expanded_bounds = (xmin, ymin, xmax, ymax) bcdata.get_dem( expanded_bounds, out_file=os.path.join(temp_folder, "dem.tif"), src_crs="EPSG:3005", dst_crs="EPSG:3005", resolution=25, ) else: click.echo("") click.echo("NO MATCHED STREAM - IS POINT IN BC or USA LOWER 48?") if not points_only: click.echo("Attempting to process point with hydrosheds data") click.echo("WARNING - hydroshed boundaries are much lower precision than FWA") click.echo("WARNING - this script does not refine hydroshed boundaries, all of intersecting polygon is included!") click.echo("WARNING - if watershed for this point includes areas in BC, the portion of output boundary in BC will not match FWA watershed boundaries!") wsd = hydroshed(pt.geometry.x, pt.geometry.y, 3005, as_gdf=True) # if we have a wsd poly, add id and write to shape if not wsd.empty: wsd.at[0, in_id] = pt[in_id] wsd.to_file(os.path.join(temp_folder, "wsd.shp"))