def test_set_width(self):
entity = gpd.read_file(self.rgi_file).iloc[0]
gdir = oggm.GlacierDirectory(entity, base_dir=self.testdir)
gis.define_glacier_region(gdir)
gis.glacier_masks(gdir)
centerlines.compute_centerlines(gdir)
centerlines.initialize_flowlines(gdir)
centerlines.compute_downstream_line(gdir)
centerlines.compute_downstream_bedshape(gdir)
centerlines.catchment_area(gdir)
centerlines.catchment_intersections(gdir)
centerlines.catchment_width_geom(gdir)
centerlines.catchment_width_correction(gdir)
# Test that area and area-altitude elev is fine
with utils.ncDataset(gdir.get_filepath('gridded_data')) as nc:
mask = nc.variables['glacier_mask'][:]
topo = nc.variables['topo_smoothed'][:]
rhgt = topo[np.where(mask)][:]
fls = gdir.read_pickle('inversion_flowlines')
hgt, widths = gdir.get_inversion_flowline_hw()
bs = 100
bins = np.arange(utils.nicenumber(np.min(hgt), bs, lower=True),
utils.nicenumber(np.max(hgt), bs) + 1,
bs)
h1, b = np.histogram(hgt, weights=widths, density=True, bins=bins)
h2, b = np.histogram(rhgt, density=True, bins=bins)
h1 = h1 / np.sum(h1)
h2 = h2 / np.sum(h2)
assert utils.rmsd(h1, h2) < 0.02 # less than 2% error
new_area = np.sum(widths * fls[-1].dx * gdir.grid.dx)
np.testing.assert_allclose(new_area, gdir.rgi_area_m2)
centerlines.terminus_width_correction(gdir, new_width=714)
fls = gdir.read_pickle('inversion_flowlines')
hgt, widths = gdir.get_inversion_flowline_hw()
# Check that the width is ok
np.testing.assert_allclose(fls[-1].widths[-1] * gdir.grid.dx, 714)
# Check for area distrib
bins = np.arange(utils.nicenumber(np.min(hgt), bs, lower=True),
utils.nicenumber(np.max(hgt), bs) + 1,
bs)
h1, b = np.histogram(hgt, weights=widths, density=True, bins=bins)
h2, b = np.histogram(rhgt, density=True, bins=bins)
h1 = h1 / np.sum(h1)
h2 = h2 / np.sum(h2)
assert utils.rmsd(h1, h2) < 0.02 # less than 2% error
new_area = np.sum(widths * fls[-1].dx * gdir.grid.dx)
np.testing.assert_allclose(new_area, gdir.rgi_area_m2)
def test_set_width(self):
entity = gpd.read_file(self.rgi_file).iloc[0]
gdir = oggm.GlacierDirectory(entity, base_dir=self.testdir)
gis.define_glacier_region(gdir, entity=entity)
gis.glacier_masks(gdir)
centerlines.compute_centerlines(gdir)
centerlines.initialize_flowlines(gdir)
centerlines.compute_downstream_line(gdir)
centerlines.compute_downstream_bedshape(gdir)
centerlines.catchment_area(gdir)
centerlines.catchment_intersections(gdir)
centerlines.catchment_width_geom(gdir)
centerlines.catchment_width_correction(gdir)
# Test that area and area-altitude elev is fine
with utils.ncDataset(gdir.get_filepath('gridded_data')) as nc:
mask = nc.variables['glacier_mask'][:]
topo = nc.variables['topo_smoothed'][:]
rhgt = topo[np.where(mask)][:]
fls = gdir.read_pickle('inversion_flowlines')
hgt, widths = gdir.get_inversion_flowline_hw()
bs = 100
bins = np.arange(utils.nicenumber(np.min(hgt), bs, lower=True),
utils.nicenumber(np.max(hgt), bs) + 1,
bs)
h1, b = np.histogram(hgt, weights=widths, density=True, bins=bins)
h2, b = np.histogram(rhgt, density=True, bins=bins)
h1 = h1 / np.sum(h1)
h2 = h2 / np.sum(h2)
assert utils.rmsd(h1, h2) < 0.02 # less than 2% error
new_area = np.sum(widths * fls[-1].dx * gdir.grid.dx)
np.testing.assert_allclose(new_area, gdir.rgi_area_m2)
centerlines.terminus_width_correction(gdir, new_width=714)
fls = gdir.read_pickle('inversion_flowlines')
hgt, widths = gdir.get_inversion_flowline_hw()
# Check that the width is ok
np.testing.assert_allclose(fls[-1].widths[-1] * gdir.grid.dx, 714)
# Check for area distrib
bins = np.arange(utils.nicenumber(np.min(hgt), bs, lower=True),
utils.nicenumber(np.max(hgt), bs) + 1,
bs)
h1, b = np.histogram(hgt, weights=widths, density=True, bins=bins)
h2, b = np.histogram(rhgt, density=True, bins=bins)
h1 = h1 / np.sum(h1)
h2 = h2 / np.sum(h2)
assert utils.rmsd(h1, h2) < 0.02 # less than 2% error
new_area = np.sum(widths * fls[-1].dx * gdir.grid.dx)
np.testing.assert_allclose(new_area, gdir.rgi_area_m2)
def test_width(self):
hef_file = get_demo_file('Hintereisferner.shp')
rgidf = gpd.GeoDataFrame.from_file(hef_file)
# loop because for some reason indexing wont work
for index, entity in rgidf.iterrows():
gdir = oggm.GlacierDirectory(entity, base_dir=self.testdir)
gis.define_glacier_region(gdir, entity=entity)
gis.glacier_masks(gdir)
centerlines.compute_centerlines(gdir)
geometry.initialize_flowlines(gdir)
geometry.catchment_area(gdir)
geometry.catchment_width_geom(gdir)
geometry.catchment_width_correction(gdir)
area = 0.
otherarea = 0.
hgt = []
harea = []
for i in gdir.divide_ids:
cls = gdir.read_pickle('inversion_flowlines', div_id=i)
for cl in cls:
harea.extend(list(cl.widths * cl.dx))
hgt.extend(list(cl.surface_h))
area += np.sum(cl.widths * cl.dx)
nc = netCDF4.Dataset(gdir.get_filepath('gridded_data', div_id=i))
otherarea += np.sum(nc.variables['glacier_mask'][:])
nc.close()
nc = netCDF4.Dataset(gdir.get_filepath('gridded_data', div_id=0))
mask = nc.variables['glacier_mask'][:]
topo = nc.variables['topo_smoothed'][:]
nc.close()
rhgt = topo[np.where(mask)][:]
tdf = gpd.GeoDataFrame.from_file(gdir.get_filepath('outlines'))
np.testing.assert_allclose(area, otherarea, rtol=0.1)
area *= (gdir.grid.dx)**2
otherarea *= (gdir.grid.dx)**2
np.testing.assert_allclose(area * 10**-6,
np.float(tdf['AREA']),
rtol=1e-4)
# Check for area distrib
bins = np.arange(utils.nicenumber(np.min(hgt), 50, lower=True),
utils.nicenumber(np.max(hgt), 50) + 1, 50.)
h1, b = np.histogram(hgt, weights=harea, density=True, bins=bins)
h2, b = np.histogram(rhgt, density=True, bins=bins)
self.assertTrue(utils.rmsd(h1 * 100 * 50, h2 * 100 * 50) < 1)
def test_width(self):
hef_file = get_demo_file('Hintereisferner.shp')
rgidf = gpd.GeoDataFrame.from_file(hef_file)
# loop because for some reason indexing wont work
for index, entity in rgidf.iterrows():
gdir = cfg.GlacierDir(entity, base_dir=self.testdir)
gis.define_glacier_region(gdir, entity)
gis.glacier_masks(gdir)
centerlines.compute_centerlines(gdir)
geometry.initialize_flowlines(gdir)
geometry.catchment_area(gdir)
geometry.catchment_width_geom(gdir)
geometry.catchment_width_correction(gdir)
area = 0.
otherarea = 0.
hgt = []
harea = []
for i in gdir.divide_ids:
cls = gdir.read_pickle('inversion_flowlines', div_id=i)
for cl in cls:
harea.extend(list(cl.widths * cl.dx))
hgt.extend(list(cl.surface_h))
area += np.sum(cl.widths * cl.dx)
nc = netCDF4.Dataset(gdir.get_filepath('grids', div_id=i))
otherarea += np.sum(nc.variables['glacier_mask'][:])
nc.close()
nc = netCDF4.Dataset(gdir.get_filepath('grids', div_id=0))
mask = nc.variables['glacier_mask'][:]
topo = nc.variables['topo_smoothed'][:]
nc.close()
rhgt = topo[np.where(mask)][:]
tdf = gpd.GeoDataFrame.from_file(gdir.get_filepath('outlines'))
np.testing.assert_allclose(area, otherarea, rtol=0.1)
area *= (gdir.grid.dx) ** 2
otherarea *= (gdir.grid.dx) ** 2
np.testing.assert_allclose(area * 10**-6, np.float(tdf['AREA']), rtol=1e-4)
# Check for area distrib
bins = np.arange(utils.nicenumber(np.min(hgt), 50, lower=True),
utils.nicenumber(np.max(hgt), 50)+1,
50.)
h1, b = np.histogram(hgt, weights=harea, density=True, bins=bins)
h2, b = np.histogram(rhgt, density=True, bins=bins)
self.assertTrue(utils.rmsd(h1*100*50, h2*100*50) < 1)
def correct_dem(gdir, glacier_mask, dem, smoothed_dem):
"""Compare with Huss and stuff."""
dem_glac = dem[np.nonzero(glacier_mask)]
# Read RGI hypso for compa
tosearch = '{:02d}'.format(np.int(gdir.rgi_region))
tosearch = os.path.join(RGI_DIR, '*', tosearch + '*_hypso.csv')
for fh in glob.glob(tosearch):
pass
df = pd.read_csv(fh)
df.columns = [c.strip() for c in df.columns]
df = df.loc[df.RGIId.isin([gdir.rgi_id])]
df = df[df.columns[3:]].T
df.columns = ['RGI (Huss)']
hs = np.asarray(df.index.values, np.int)
bins = utils.nicenumber(hs, 50, lower=True)
bins = np.append(bins, bins[-1] + 50)
myhist, _ = np.histogram(dem_glac, bins=bins)
myhist = myhist / np.sum(myhist) * 1000
df['OGGM'] = myhist
df = df / 10
df.index.rename('Alt (m)', inplace=True)
df.plot()
plt.ylabel('Freq (%)')
plt.savefig('/home/mowglie/hypso_' + gdir.rgi_id + '.png')
minz = None
if gdir.rgi_id == 'RGI50-06.00424': minz = 800
if gdir.rgi_id == 'RGI50-06.00443': minz = 600
return dem, smoothed_dem
def test_width(self):
hef_file = get_demo_file("Hintereisferner.shp")
entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0]
gdir = oggm.GlacierDirectory(entity, base_dir=self.testdir)
gis.define_glacier_region(gdir, entity=entity)
gis.glacier_masks(gdir)
centerlines.compute_centerlines(gdir)
geometry.initialize_flowlines(gdir)
geometry.catchment_area(gdir)
geometry.catchment_width_geom(gdir)
geometry.catchment_width_correction(gdir)
area = 0.0
otherarea = 0.0
hgt = []
harea = []
for i in gdir.divide_ids:
cls = gdir.read_pickle("inversion_flowlines", div_id=i)
for cl in cls:
harea.extend(list(cl.widths * cl.dx))
hgt.extend(list(cl.surface_h))
area += np.sum(cl.widths * cl.dx)
with netCDF4.Dataset(gdir.get_filepath("gridded_data", div_id=i)) as nc:
otherarea += np.sum(nc.variables["glacier_mask"][:])
with netCDF4.Dataset(gdir.get_filepath("gridded_data", div_id=0)) as nc:
mask = nc.variables["glacier_mask"][:]
topo = nc.variables["topo_smoothed"][:]
rhgt = topo[np.where(mask)][:]
tdf = gpd.GeoDataFrame.from_file(gdir.get_filepath("outlines"))
np.testing.assert_allclose(area, otherarea, rtol=0.1)
area *= (gdir.grid.dx) ** 2
otherarea *= (gdir.grid.dx) ** 2
np.testing.assert_allclose(area * 10 ** -6, np.float(tdf["AREA"]), rtol=1e-4)
# Check for area distrib
bins = np.arange(utils.nicenumber(np.min(hgt), 50, lower=True), utils.nicenumber(np.max(hgt), 50) + 1, 50.0)
h1, b = np.histogram(hgt, weights=harea, density=True, bins=bins)
h2, b = np.histogram(rhgt, density=True, bins=bins)
self.assertTrue(utils.rmsd(h1 * 100 * 50, h2 * 100 * 50) < 1)
def catchment_width_correction(gdir, div_id=None):
"""Corrects for NaNs and inconsistencies in the geometrical widths.
Interpolates missing values, ensures consistency of the
surface-area distribution AND with the geometrical area of the glacier
polygon, avoiding errors due to gridded representation.
Updates the 'inversion_flowlines' save file.
Parameters
----------
gdir : oggm.GlacierDirectory
"""
# The code below makes of this task a "special" divide task.
# We keep it as is and remove the divide task decorator
if div_id is None:
# This is the original call
# This time instead of just looping over the divides we add a test
# to check for the conservation of the shapefile's area.
area = 0.
divides = []
for i in gdir.divide_ids:
log.info('%s: width correction, divide %d', gdir.rgi_id, i)
fls = catchment_width_correction(gdir, div_id=i, reset=True)
for fl in fls:
area += np.sum(fl.widths) * fl.dx
divides.append(fls)
# Final correction - because of the raster, the gridded area of the
# glacier is not that of the actual geometry. correct for that
fac = gdir.rgi_area_km2 / (area * gdir.grid.dx**2 * 10**-6)
log.debug('%s: corrected widths with a factor %.2f', gdir.rgi_id, fac)
for i in gdir.divide_ids:
fls = divides[i-1]
for fl in fls:
fl.widths *= fac
# Overwrite centerlines
gdir.write_pickle(fls, 'inversion_flowlines', div_id=i)
return None
# variables
flowlines = gdir.read_pickle('inversion_flowlines', div_id=div_id)
catchment_indices = gdir.read_pickle('catchment_indices', div_id=div_id)
# Topography for altitude-area distribution
# I take the non-smoothed topography and remove the borders
fpath = gdir.get_filepath('gridded_data', div_id=div_id)
with netCDF4.Dataset(fpath) as nc:
topo = nc.variables['topo'][:]
ext = nc.variables['glacier_ext'][:]
topo[np.where(ext==1)] = np.NaN
# Param
nmin = int(cfg.PARAMS['min_n_per_bin'])
smooth_ws = int(cfg.PARAMS['smooth_widths_window_size'])
# Per flowline (important so that later, the indices can be moved)
catchment_heights = []
for ci in catchment_indices:
_t = topo[tuple(ci.T)][:]
catchment_heights.append(list(_t[np.isfinite(_t)]))
# Loop over lines in a reverse order
for fl, catch_h in zip(flowlines, catchment_heights):
# Interpolate widths
widths = utils.interp_nans(fl.widths)
widths = np.clip(widths, 0.1, np.max(widths))
# Get topo per catchment and per flowline point
fhgt = fl.surface_h
# Sometimes, the centerline does not reach as high as each pix on the
# glacier. (e.g. RGI40-11.00006)
catch_h = np.clip(catch_h, 0, np.max(fhgt))
# Max and mins for the histogram
maxh = np.max(fhgt)
if fl.flows_to is None:
minh = np.min(fhgt)
catch_h = np.clip(catch_h, minh, np.max(catch_h))
else:
minh = np.min(fhgt) # Min just for flowline (this has reasons)
# Now decide on a binsize which ensures at least N element per bin
bsize = cfg.PARAMS['base_binsize']
while True:
maxb = utils.nicenumber(maxh, 1)
minb = utils.nicenumber(minh, 1, lower=True)
bins = np.arange(minb, maxb+bsize+0.01, bsize)
minb = np.min(bins)
# Ignore the topo pixels below the last bin
tmp_ght = catch_h[np.where(catch_h >= minb)]
topo_digi = np.digitize(tmp_ght, bins) - 1 # I prefer the left
fl_digi = np.digitize(fhgt, bins) - 1 # I prefer the left
if nmin == 1:
# No need for complicated count
_c = set(topo_digi)
_fl = set(fl_digi)
else:
# Keep indexes with at least n counts
_c = Counter(topo_digi.tolist())
_c = set([k for (k, v) in _c.items() if v >= nmin])
_fl = Counter(fl_digi.tolist())
_fl = set([k for (k, v) in _fl.items() if v >= nmin])
ref_set = set(range(len(bins)-1))
if (_c == ref_set) and (_fl == ref_set):
# For each bin, the width(s) have to represent the "real" area
new_widths = widths.copy()
for bi in range(len(bins) - 1):
bintopoarea = len(np.where(topo_digi == bi)[0])
wherewiths = np.where(fl_digi == bi)
binflarea = np.sum(new_widths[wherewiths]) * fl.dx
new_widths[wherewiths] = (bintopoarea / binflarea) * \
new_widths[wherewiths]
break
bsize += 5
# Add a security for infinite loops
if bsize > 500:
nmin -= 1
bsize = cfg.PARAMS['base_binsize']
log.warning('%s: reduced min n per bin to %d', gdir.rgi_id,
nmin)
if nmin == 0:
raise RuntimeError('NO binsize could be chosen for: '
'{}'.format(gdir.rgi_id))
if bsize > 150:
log.warning('%s: chosen binsize %d', gdir.rgi_id, bsize)
else:
log.debug('%s: chosen binsize %d', gdir.rgi_id, bsize)
# Now keep the good topo pixels and send the unattributed ones to the
# next flowline
tosend = list(catch_h[np.where(catch_h < minb)])
if (len(tosend) > 0) and (fl.flows_to is not None):
ide = flowlines.index(fl.flows_to)
catchment_heights[ide] = np.append(catchment_heights[ide], tosend)
if (len(tosend) > 0) and (fl.flows_to is None):
raise RuntimeError('This should not happen')
# Now we have a width which is the "best" representation of our
# tributary according to the altitude area distribution.
# This sometimes leads to abrupt changes in the widths from one
# grid point to another. I think it's not too harmful to smooth them
# here, at the cost of a less perfect altitude area distribution
if smooth_ws != 0:
if smooth_ws == 1:
new_widths = utils.smooth1d(new_widths)
else:
new_widths = utils.smooth1d(new_widths, window_size=smooth_ws)
# Write it
fl.widths = new_widths
return flowlines
def simple_glacier_masks(gdir):
"""Compute glacier masks based on much simpler rules than OGGM's default.
This is therefore more robust: we use this function to compute glacier
hypsometries.
Parameters
----------
gdir : :py:class:`oggm.GlacierDirectory`
where to write the data
"""
# open srtm tif-file:
dem_dr = rasterio.open(gdir.get_filepath('dem'), 'r', driver='GTiff')
dem = dem_dr.read(1).astype(rasterio.float32)
# Grid
nx = dem_dr.width
ny = dem_dr.height
assert nx == gdir.grid.nx
assert ny == gdir.grid.ny
# Correct the DEM
# Currently we just do a linear interp -- filling is totally shit anyway
min_z = -999.
dem[dem <= min_z] = np.NaN
isfinite = np.isfinite(dem)
if np.all(~isfinite):
raise InvalidDEMError('Not a single valid grid point in DEM')
if np.any(~isfinite):
xx, yy = gdir.grid.ij_coordinates
pnan = np.nonzero(~isfinite)
pok = np.nonzero(isfinite)
points = np.array((np.ravel(yy[pok]), np.ravel(xx[pok]))).T
inter = np.array((np.ravel(yy[pnan]), np.ravel(xx[pnan]))).T
try:
dem[pnan] = griddata(points, np.ravel(dem[pok]), inter,
method='linear')
except ValueError:
raise InvalidDEMError('DEM interpolation not possible.')
log.warning(gdir.rgi_id + ': DEM needed interpolation.')
gdir.add_to_diagnostics('dem_needed_interpolation', True)
gdir.add_to_diagnostics('dem_invalid_perc', len(pnan[0]) / (nx*ny))
isfinite = np.isfinite(dem)
if np.any(~isfinite):
# this happens when extrapolation is needed
# see how many percent of the dem
if np.sum(~isfinite) > (0.5 * nx * ny):
log.warning('({}) many NaNs in DEM'.format(gdir.rgi_id))
xx, yy = gdir.grid.ij_coordinates
pnan = np.nonzero(~isfinite)
pok = np.nonzero(isfinite)
points = np.array((np.ravel(yy[pok]), np.ravel(xx[pok]))).T
inter = np.array((np.ravel(yy[pnan]), np.ravel(xx[pnan]))).T
try:
dem[pnan] = griddata(points, np.ravel(dem[pok]), inter,
method='nearest')
except ValueError:
raise InvalidDEMError('DEM extrapolation not possible.')
log.warning(gdir.rgi_id + ': DEM needed extrapolation.')
gdir.add_to_diagnostics('dem_needed_extrapolation', True)
gdir.add_to_diagnostics('dem_extrapol_perc', len(pnan[0]) / (nx*ny))
if np.min(dem) == np.max(dem):
raise InvalidDEMError('({}) min equal max in the DEM.'
.format(gdir.rgi_id))
# Proj
if LooseVersion(rasterio.__version__) >= LooseVersion('1.0'):
transf = dem_dr.transform
else:
raise ImportError('This task needs rasterio >= 1.0 to work properly')
x0 = transf[2] # UL corner
y0 = transf[5] # UL corner
dx = transf[0]
dy = transf[4] # Negative
assert dx == -dy
assert dx == gdir.grid.dx
assert y0 == gdir.grid.corner_grid.y0
assert x0 == gdir.grid.corner_grid.x0
profile = dem_dr.profile
dem_dr.close()
# Clip topography to 0 m a.s.l.
dem = dem.clip(0)
# Smooth DEM?
if cfg.PARAMS['smooth_window'] > 0.:
gsize = np.rint(cfg.PARAMS['smooth_window'] / dx)
smoothed_dem = gaussian_blur(dem, np.int(gsize))
else:
smoothed_dem = dem.copy()
if not np.all(np.isfinite(smoothed_dem)):
raise InvalidDEMError('({}) NaN in smoothed DEM'.format(gdir.rgi_id))
# Geometries
geometry = gdir.read_shapefile('outlines').geometry[0]
# simple trick to correct invalid polys:
# http://stackoverflow.com/questions/20833344/
# fix-invalid-polygon-python-shapely
geometry = geometry.buffer(0)
if not geometry.is_valid:
raise InvalidDEMError('This glacier geometry is not valid.')
# Compute the glacier mask using rasterio
# Small detour as mask only accepts DataReader objects
with rasterio.io.MemoryFile() as memfile:
with memfile.open(**profile) as dataset:
dataset.write(dem.astype(np.int16)[np.newaxis, ...])
dem_data = rasterio.open(memfile.name)
masked_dem, _ = riomask(dem_data, [shpg.mapping(geometry)],
filled=False)
glacier_mask = ~masked_dem[0, ...].mask
# Smame without nunataks
with rasterio.io.MemoryFile() as memfile:
with memfile.open(**profile) as dataset:
dataset.write(dem.astype(np.int16)[np.newaxis, ...])
dem_data = rasterio.open(memfile.name)
poly = shpg.mapping(shpg.Polygon(geometry.exterior))
masked_dem, _ = riomask(dem_data, [poly],
filled=False)
glacier_mask_nonuna = ~masked_dem[0, ...].mask
# Glacier exterior excluding nunataks
erode = binary_erosion(glacier_mask_nonuna)
glacier_ext = glacier_mask_nonuna ^ erode
glacier_ext = np.where(glacier_mask_nonuna, glacier_ext, 0)
# Last sanity check based on the masked dem
tmp_max = np.max(dem[glacier_mask])
tmp_min = np.min(dem[glacier_mask])
if tmp_max < (tmp_min + 1):
raise InvalidDEMError('({}) min equal max in the masked DEM.'
.format(gdir.rgi_id))
# hypsometry
bsize = 50.
dem_on_ice = dem[glacier_mask]
bins = np.arange(nicenumber(dem_on_ice.min(), bsize, lower=True),
nicenumber(dem_on_ice.max(), bsize) + 0.01, bsize)
h, _ = np.histogram(dem_on_ice, bins)
h = h / np.sum(h) * 1000 # in permil
# We want to convert the bins to ints but preserve their sum to 1000
# Start with everything rounded down, then round up the numbers with the
# highest fractional parts until the desired sum is reached.
hi = np.floor(h).astype(np.int)
hup = np.ceil(h).astype(np.int)
aso = np.argsort(hup - h)
for i in aso:
hi[i] = hup[i]
if np.sum(hi) == 1000:
break
# slope
sy, sx = np.gradient(dem, dx)
aspect = np.arctan2(np.mean(-sx[glacier_mask]), np.mean(sy[glacier_mask]))
aspect = np.rad2deg(aspect)
if aspect < 0:
aspect += 360
slope = np.arctan(np.sqrt(sx ** 2 + sy ** 2))
avg_slope = np.rad2deg(np.mean(slope[glacier_mask]))
# write
df = pd.DataFrame()
df['RGIId'] = [gdir.rgi_id]
df['GLIMSId'] = [gdir.glims_id]
df['Zmin'] = [dem_on_ice.min()]
df['Zmax'] = [dem_on_ice.max()]
df['Zmed'] = [np.median(dem_on_ice)]
df['Area'] = [gdir.rgi_area_km2]
df['Slope'] = [avg_slope]
df['Aspect'] = [aspect]
for b, bs in zip(hi, (bins[1:] + bins[:-1])/2):
df['{}'.format(np.round(bs).astype(int))] = [b]
df.to_csv(gdir.get_filepath('hypsometry'), index=False)
# write out the grids in the netcdf file
nc = gdir.create_gridded_ncdf_file('gridded_data')
v = nc.createVariable('topo', 'f4', ('y', 'x', ), zlib=True)
v.units = 'm'
v.long_name = 'DEM topography'
v[:] = dem
v = nc.createVariable('topo_smoothed', 'f4', ('y', 'x', ), zlib=True)
v.units = 'm'
v.long_name = ('DEM topography smoothed '
'with radius: {:.1} m'.format(cfg.PARAMS['smooth_window']))
v[:] = smoothed_dem
v = nc.createVariable('glacier_mask', 'i1', ('y', 'x', ), zlib=True)
v.units = '-'
v.long_name = 'Glacier mask'
v[:] = glacier_mask
v = nc.createVariable('glacier_ext', 'i1', ('y', 'x', ), zlib=True)
v.units = '-'
v.long_name = 'Glacier external boundaries'
v[:] = glacier_ext
# add some meta stats and close
nc.max_h_dem = np.max(dem)
nc.min_h_dem = np.min(dem)
dem_on_g = dem[np.where(glacier_mask)]
nc.max_h_glacier = np.max(dem_on_g)
nc.min_h_glacier = np.min(dem_on_g)
nc.close()
def catchment_width_correction(gdir, div_id=None):
"""Corrects for NaNs and inconsistencies in the geometrical widths.
Interpolates missing values, ensures consistency of the
surface-area distribution AND with the geometrical area of the glacier
polygon, avoiding errors due to gridded representation.
Updates the 'inversion_flowlines' save file.
Parameters
----------
gdir : oggm.GlacierDirectory
"""
# The code below makes of this task a "special" divide task.
# We keep it as is and remove the divide task decorator
if div_id is None:
# This is the original call
# This time instead of just looping over the divides we add a test
# to check for the conservation of the shapefile's area.
area = 0.
divides = []
for i in gdir.divide_ids:
log.info('%s: width correction, divide %d', gdir.rgi_id, i)
fls = catchment_width_correction(gdir, div_id=i)
for fl in fls:
area += np.sum(fl.widths) * fl.dx
divides.append(fls)
# Final correction - because of the raster, the gridded area of the
# glacier is not that of the actual geometry. correct for that
fac = gdir.rgi_area_km2 / (area * gdir.grid.dx**2 * 10**-6)
log.debug('%s: corrected widths with a factor %.2f', gdir.rgi_id, fac)
for i in gdir.divide_ids:
fls = divides[i-1]
for fl in fls:
fl.widths *= fac
# Overwrite centerlines
gdir.write_pickle(fls, 'inversion_flowlines', div_id=i)
return None
# variables
flowlines = gdir.read_pickle('inversion_flowlines', div_id=div_id)
catchment_indices = gdir.read_pickle('catchment_indices', div_id=div_id)
# Topography for altitude-area distribution
# I take the non-smoothed topography and remove the borders
fpath = gdir.get_filepath('gridded_data', div_id=div_id)
with netCDF4.Dataset(fpath) as nc:
topo = nc.variables['topo'][:]
ext = nc.variables['glacier_ext'][:]
topo[np.where(ext==1)] = np.NaN
# Param
nmin = int(cfg.PARAMS['min_n_per_bin'])
# Per flowline (important so that later, the indices can be moved)
catchment_heights = []
for ci in catchment_indices:
_t = topo[tuple(ci.T)][:]
catchment_heights.append(list(_t[np.isfinite(_t)]))
# Loop over lines in a reverse order
for fl, catch_h in zip(flowlines, catchment_heights):
# Interpolate widths
widths = utils.interp_nans(fl.widths)
widths = np.clip(widths, 0.1, np.max(widths))
# Get topo per catchment and per flowline point
fhgt = fl.surface_h
# Sometimes, the centerline does not reach as high as each pix on the
# glacier. (e.g. RGI40-11.00006)
catch_h = np.clip(catch_h, 0, np.max(fhgt))
# Max and mins for the histogram
maxh = np.max(fhgt)
if fl.flows_to is None:
minh = np.min(fhgt)
catch_h = np.clip(catch_h, minh, np.max(catch_h))
else:
minh = np.min(fhgt) # Min just for flowline (this has reasons)
# Now decide on a binsize which ensures at least N element per bin
bsize = cfg.PARAMS['base_binsize']
while True:
maxb = utils.nicenumber(maxh, 1)
minb = utils.nicenumber(minh, 1, lower=True)
bins = np.arange(minb, maxb+bsize+0.01, bsize)
minb = np.min(bins)
# Ignore the topo pixels below the last bin
tmp_ght = catch_h[np.where(catch_h >= minb)]
topo_digi = np.digitize(tmp_ght, bins) - 1 # I prefer the left
fl_digi = np.digitize(fhgt, bins) - 1 # I prefer the left
if nmin == 1:
# No need for complicated count
_c = set(topo_digi)
_fl = set(fl_digi)
else:
# Keep indexes with at least n counts
_c = Counter(topo_digi.tolist())
_c = set([k for (k, v) in _c.items() if v >= nmin])
_fl = Counter(fl_digi.tolist())
_fl = set([k for (k, v) in _fl.items() if v >= nmin])
ref_set = set(range(len(bins)-1))
if (_c == ref_set) and (_fl == ref_set):
# For each bin, the width(s) have to represent the "real" area
new_widths = widths.copy()
for bi in range(len(bins) - 1):
bintopoarea = len(np.where(topo_digi == bi)[0])
wherewiths = np.where(fl_digi == bi)
binflarea = np.sum(new_widths[wherewiths]) * fl.dx
new_widths[wherewiths] = (bintopoarea / binflarea) * \
new_widths[wherewiths]
break
# # TODO: smooth them ?
# widths = utils.smooth1d(widths)
# if np.nanmax(_width_change_factor(new_widths)[:-5]) < 2.:
# break
bsize += 5
# Add a security for infinite loops
if bsize > 250:
nmin -= 1
bsize = cfg.PARAMS['base_binsize']
log.warning('%s: reduced min n per bin to %d', gdir.rgi_id,
nmin)
if nmin == 0:
raise RuntimeError('NO binsize could be chosen for: '
'{}'.format(gdir.rgi_id))
if bsize > 150:
log.warning('%s: chosen binsize %d', gdir.rgi_id, bsize)
else:
log.debug('%s: chosen binsize %d', gdir.rgi_id, bsize)
# Now keep the good topo pixels and send the unattributed ones to the
# next flowline
tosend = list(catch_h[np.where(catch_h < minb)])
if (len(tosend) > 0) and (fl.flows_to is not None):
ide = flowlines.index(fl.flows_to)
catchment_heights[ide] = np.append(catchment_heights[ide], tosend)
if (len(tosend) > 0) and (fl.flows_to is None):
raise RuntimeError('This should not happen')
# Write it
fl.widths = new_widths
return flowlines
tasks.compute_centerlines(gdir)
tasks.initialize_flowlines(gdir)
tasks.compute_downstream_line(gdir)
tasks.catchment_area(gdir)
tasks.catchment_intersections(gdir)
tasks.catchment_width_geom(gdir)
tasks.catchment_width_correction(gdir)
with netCDF4.Dataset(gdir.get_filepath('gridded_data')) as nc:
mask = nc.variables['glacier_mask'][:]
topo = nc.variables['topo_smoothed'][:]
rhgt = topo[np.where(mask)][:]
hgt, harea = gdir.get_inversion_flowline_hw()
# Check for area distrib
bins = np.arange(nicenumber(np.min(hgt), 150, lower=True),
nicenumber(np.max(hgt), 150) + 1, 150.)
h1, b = np.histogram(hgt, weights=harea, density=True, bins=bins)
h2, b = np.histogram(rhgt, density=True, bins=bins)
h1 = h1 / np.sum(h1)
h2 = h2 / np.sum(h2)
LCMAP = plt.get_cmap('Purples')(np.linspace(0.4, 1, 5)[::-1])
MCMAP = graphics.truncate_colormap(plt.get_cmap('Blues'), 0.2, 0.8, 255)
f, axs = plt.subplots(2, 2, figsize=(8.5, 7))
axs = np.asarray(axs).flatten()
llkw = {'interval': 0}
letkm = dict(color='black',
def simple_glacier_masks(gdir):
"""Compute glacier masks based on much simpler rules than OGGM's default.
This is therefore more robust: we use this function to compute glacier
hypsometries.
Parameters
----------
gdir : :py:class:`oggm.GlacierDirectory`
where to write the data
"""
# open srtm tif-file:
dem_dr = rasterio.open(gdir.get_filepath('dem'), 'r', driver='GTiff')
dem = dem_dr.read(1).astype(rasterio.float32)
# Grid
nx = dem_dr.width
ny = dem_dr.height
assert nx == gdir.grid.nx
assert ny == gdir.grid.ny
# Correct the DEM
# Currently we just do a linear interp -- filling is totally shit anyway
min_z = -999.
dem[dem <= min_z] = np.NaN
isfinite = np.isfinite(dem)
if np.all(~isfinite):
raise InvalidDEMError('Not a single valid grid point in DEM')
if np.any(~isfinite):
xx, yy = gdir.grid.ij_coordinates
pnan = np.nonzero(~isfinite)
pok = np.nonzero(isfinite)
points = np.array((np.ravel(yy[pok]), np.ravel(xx[pok]))).T
inter = np.array((np.ravel(yy[pnan]), np.ravel(xx[pnan]))).T
try:
dem[pnan] = griddata(points,
np.ravel(dem[pok]),
inter,
method='linear')
except ValueError:
raise InvalidDEMError('DEM interpolation not possible.')
log.warning(gdir.rgi_id + ': DEM needed interpolation.')
gdir.add_to_diagnostics('dem_needed_interpolation', True)
gdir.add_to_diagnostics('dem_invalid_perc', len(pnan[0]) / (nx * ny))
isfinite = np.isfinite(dem)
if np.any(~isfinite):
# this happens when extrapolation is needed
# see how many percent of the dem
if np.sum(~isfinite) > (0.5 * nx * ny):
log.warning('({}) many NaNs in DEM'.format(gdir.rgi_id))
xx, yy = gdir.grid.ij_coordinates
pnan = np.nonzero(~isfinite)
pok = np.nonzero(isfinite)
points = np.array((np.ravel(yy[pok]), np.ravel(xx[pok]))).T
inter = np.array((np.ravel(yy[pnan]), np.ravel(xx[pnan]))).T
try:
dem[pnan] = griddata(points,
np.ravel(dem[pok]),
inter,
method='nearest')
except ValueError:
raise InvalidDEMError('DEM extrapolation not possible.')
log.warning(gdir.rgi_id + ': DEM needed extrapolation.')
gdir.add_to_diagnostics('dem_needed_extrapolation', True)
gdir.add_to_diagnostics('dem_extrapol_perc', len(pnan[0]) / (nx * ny))
if np.min(dem) == np.max(dem):
raise InvalidDEMError('({}) min equal max in the DEM.'.format(
gdir.rgi_id))
# Proj
if LooseVersion(rasterio.__version__) >= LooseVersion('1.0'):
transf = dem_dr.transform
else:
raise ImportError('This task needs rasterio >= 1.0 to work properly')
x0 = transf[2] # UL corner
y0 = transf[5] # UL corner
dx = transf[0]
dy = transf[4] # Negative
assert dx == -dy
assert dx == gdir.grid.dx
assert y0 == gdir.grid.corner_grid.y0
assert x0 == gdir.grid.corner_grid.x0
profile = dem_dr.profile
dem_dr.close()
# Clip topography to 0 m a.s.l.
dem = dem.clip(0)
# Smooth DEM?
if cfg.PARAMS['smooth_window'] > 0.:
gsize = np.rint(cfg.PARAMS['smooth_window'] / dx)
smoothed_dem = gaussian_blur(dem, np.int(gsize))
else:
smoothed_dem = dem.copy()
if not np.all(np.isfinite(smoothed_dem)):
raise InvalidDEMError('({}) NaN in smoothed DEM'.format(gdir.rgi_id))
# Geometries
geometry = gdir.read_shapefile('outlines').geometry[0]
# simple trick to correct invalid polys:
# http://stackoverflow.com/questions/20833344/
# fix-invalid-polygon-python-shapely
geometry = geometry.buffer(0)
if not geometry.is_valid:
raise InvalidDEMError('This glacier geometry is not valid.')
# Compute the glacier mask using rasterio
# Small detour as mask only accepts DataReader objects
with rasterio.io.MemoryFile() as memfile:
with memfile.open(**profile) as dataset:
dataset.write(dem.astype(np.int16)[np.newaxis, ...])
dem_data = rasterio.open(memfile.name)
masked_dem, _ = riomask(dem_data, [shpg.mapping(geometry)],
filled=False)
glacier_mask = ~masked_dem[0, ...].mask
# Smame without nunataks
with rasterio.io.MemoryFile() as memfile:
with memfile.open(**profile) as dataset:
dataset.write(dem.astype(np.int16)[np.newaxis, ...])
dem_data = rasterio.open(memfile.name)
poly = shpg.mapping(shpg.Polygon(geometry.exterior))
masked_dem, _ = riomask(dem_data, [poly], filled=False)
glacier_mask_nonuna = ~masked_dem[0, ...].mask
# Glacier exterior excluding nunataks
erode = binary_erosion(glacier_mask_nonuna)
glacier_ext = glacier_mask_nonuna ^ erode
glacier_ext = np.where(glacier_mask_nonuna, glacier_ext, 0)
# Last sanity check based on the masked dem
tmp_max = np.max(dem[glacier_mask])
tmp_min = np.min(dem[glacier_mask])
if tmp_max < (tmp_min + 1):
raise InvalidDEMError('({}) min equal max in the masked DEM.'.format(
gdir.rgi_id))
# hypsometry
bsize = 50.
dem_on_ice = dem[glacier_mask]
bins = np.arange(nicenumber(dem_on_ice.min(), bsize, lower=True),
nicenumber(dem_on_ice.max(), bsize) + 0.01, bsize)
h, _ = np.histogram(dem_on_ice, bins)
h = h / np.sum(h) * 1000 # in permil
# We want to convert the bins to ints but preserve their sum to 1000
# Start with everything rounded down, then round up the numbers with the
# highest fractional parts until the desired sum is reached.
hi = np.floor(h).astype(np.int)
hup = np.ceil(h).astype(np.int)
aso = np.argsort(hup - h)
for i in aso:
hi[i] = hup[i]
if np.sum(hi) == 1000:
break
# slope
sy, sx = np.gradient(dem, dx)
aspect = np.arctan2(np.mean(-sx[glacier_mask]), np.mean(sy[glacier_mask]))
aspect = np.rad2deg(aspect)
if aspect < 0:
aspect += 360
slope = np.arctan(np.sqrt(sx**2 + sy**2))
avg_slope = np.rad2deg(np.mean(slope[glacier_mask]))
# write
df = pd.DataFrame()
df['RGIId'] = [gdir.rgi_id]
df['GLIMSId'] = [gdir.glims_id]
df['Zmin'] = [dem_on_ice.min()]
df['Zmax'] = [dem_on_ice.max()]
df['Zmed'] = [np.median(dem_on_ice)]
df['Area'] = [gdir.rgi_area_km2]
df['Slope'] = [avg_slope]
df['Aspect'] = [aspect]
for b, bs in zip(hi, (bins[1:] + bins[:-1]) / 2):
df['{}'.format(np.round(bs).astype(int))] = [b]
df.to_csv(gdir.get_filepath('hypsometry'), index=False)
# write out the grids in the netcdf file
nc = gdir.create_gridded_ncdf_file('gridded_data')
v = nc.createVariable('topo', 'f4', (
'y',
'x',
), zlib=True)
v.units = 'm'
v.long_name = 'DEM topography'
v[:] = dem
v = nc.createVariable('topo_smoothed', 'f4', (
'y',
'x',
), zlib=True)
v.units = 'm'
v.long_name = ('DEM topography smoothed '
'with radius: {:.1} m'.format(cfg.PARAMS['smooth_window']))
v[:] = smoothed_dem
v = nc.createVariable('glacier_mask', 'i1', (
'y',
'x',
), zlib=True)
v.units = '-'
v.long_name = 'Glacier mask'
v[:] = glacier_mask
v = nc.createVariable('glacier_ext', 'i1', (
'y',
'x',
), zlib=True)
v.units = '-'
v.long_name = 'Glacier external boundaries'
v[:] = glacier_ext
# add some meta stats and close
nc.max_h_dem = np.max(dem)
nc.min_h_dem = np.min(dem)
dem_on_g = dem[np.where(glacier_mask)]
nc.max_h_glacier = np.max(dem_on_g)
nc.min_h_glacier = np.min(dem_on_g)
nc.close()
def simple_glacier_masks(gdir):
"""Compute glacier masks based on much simpler rules than OGGM's default.
This is therefore more robust: we use this function to compute glacier
hypsometries.
Parameters
----------
gdir : :py:class:`oggm.GlacierDirectory`
where to write the data
"""
if not os.path.exists(gdir.get_filepath('gridded_data')):
# In a possible future, we might actually want to raise a
# deprecation warning here
process_dem(gdir)
# Geometries
geometry = gdir.read_shapefile('outlines').geometry[0]
# rio metadata
with rasterio.open(gdir.get_filepath('dem'), 'r', driver='GTiff') as ds:
data = ds.read(1).astype(rasterio.float32)
profile = ds.profile
# simple trick to correct invalid polys:
# http://stackoverflow.com/questions/20833344/
# fix-invalid-polygon-python-shapely
geometry = geometry.buffer(0)
if not geometry.is_valid:
raise InvalidDEMError('This glacier geometry is not valid.')
# Compute the glacier mask using rasterio
# Small detour as mask only accepts DataReader objects
with rasterio.io.MemoryFile() as memfile:
with memfile.open(**profile) as dataset:
dataset.write(data.astype(np.int16)[np.newaxis, ...])
dem_data = rasterio.open(memfile.name)
masked_dem, _ = riomask(dem_data, [shpg.mapping(geometry)],
filled=False)
glacier_mask = ~masked_dem[0, ...].mask
# Same without nunataks
with rasterio.io.MemoryFile() as memfile:
with memfile.open(**profile) as dataset:
dataset.write(data.astype(np.int16)[np.newaxis, ...])
dem_data = rasterio.open(memfile.name)
poly = shpg.mapping(shpg.Polygon(geometry.exterior))
masked_dem, _ = riomask(dem_data, [poly],
filled=False)
glacier_mask_nonuna = ~masked_dem[0, ...].mask
# Glacier exterior excluding nunataks
erode = binary_erosion(glacier_mask_nonuna)
glacier_ext = glacier_mask_nonuna ^ erode
glacier_ext = np.where(glacier_mask_nonuna, glacier_ext, 0)
dem = read_geotiff_dem(gdir)
# Last sanity check based on the masked dem
tmp_max = np.max(dem[glacier_mask])
tmp_min = np.min(dem[glacier_mask])
if tmp_max < (tmp_min + 1):
raise InvalidDEMError('({}) min equal max in the masked DEM.'
.format(gdir.rgi_id))
# hypsometry
bsize = 50.
dem_on_ice = dem[glacier_mask]
bins = np.arange(nicenumber(dem_on_ice.min(), bsize, lower=True),
nicenumber(dem_on_ice.max(), bsize) + 0.01, bsize)
h, _ = np.histogram(dem_on_ice, bins)
h = h / np.sum(h) * 1000 # in permil
# We want to convert the bins to ints but preserve their sum to 1000
# Start with everything rounded down, then round up the numbers with the
# highest fractional parts until the desired sum is reached.
hi = np.floor(h).astype(np.int)
hup = np.ceil(h).astype(np.int)
aso = np.argsort(hup - h)
for i in aso:
hi[i] = hup[i]
if np.sum(hi) == 1000:
break
# slope
sy, sx = np.gradient(dem, gdir.grid.dx)
aspect = np.arctan2(np.mean(-sx[glacier_mask]), np.mean(sy[glacier_mask]))
aspect = np.rad2deg(aspect)
if aspect < 0:
aspect += 360
slope = np.arctan(np.sqrt(sx ** 2 + sy ** 2))
avg_slope = np.rad2deg(np.mean(slope[glacier_mask]))
# write
df = pd.DataFrame()
df['RGIId'] = [gdir.rgi_id]
df['GLIMSId'] = [gdir.glims_id]
df['Zmin'] = [dem_on_ice.min()]
df['Zmax'] = [dem_on_ice.max()]
df['Zmed'] = [np.median(dem_on_ice)]
df['Area'] = [gdir.rgi_area_km2]
df['Slope'] = [avg_slope]
df['Aspect'] = [aspect]
for b, bs in zip(hi, (bins[1:] + bins[:-1])/2):
df['{}'.format(np.round(bs).astype(int))] = [b]
df.to_csv(gdir.get_filepath('hypsometry'), index=False)
# write out the grids in the netcdf file
with GriddedNcdfFile(gdir) as nc:
if 'glacier_mask' not in nc.variables:
v = nc.createVariable('glacier_mask', 'i1', ('y', 'x', ),
zlib=True)
v.units = '-'
v.long_name = 'Glacier mask'
else:
v = nc.variables['glacier_mask']
v[:] = glacier_mask
if 'glacier_ext' not in nc.variables:
v = nc.createVariable('glacier_ext', 'i1', ('y', 'x', ),
zlib=True)
v.units = '-'
v.long_name = 'Glacier external boundaries'
else:
v = nc.variables['glacier_ext']
v[:] = glacier_ext
# Log DEM that needed processing within the glacier mask
valid_mask = nc.variables['topo_valid_mask'][:]
if gdir.get_diagnostics().get('dem_needed_interpolation', False):
pnan = (valid_mask == 0) & glacier_mask
gdir.add_to_diagnostics('dem_invalid_perc_in_mask',
np.sum(pnan) / np.sum(glacier_mask))
# add some meta stats and close
nc.max_h_dem = np.max(dem)
nc.min_h_dem = np.min(dem)
dem_on_g = dem[np.where(glacier_mask)]
nc.max_h_glacier = np.max(dem_on_g)
nc.min_h_glacier = np.min(dem_on_g)
