def _read_all_data(self):
"""
Reads all necessary information from files in gdir for
minimization/optimization and logging.
"""
self.true_bed = salem.GeoTiff(
self.gdir.get_filepath('dem')).get_vardata()
self.ref_surf = salem.GeoTiff(
self.gdir.get_filepath('ref_dem')).get_vardata()
self.first_guessed_bed = salem.GeoTiff(
self.get_subdir_filepath('first_guessed_bed')).get_vardata()
self.ice_mask = np.load(self.gdir.get_filepath('ref_ice_mask'))
if os.path.exists(self.gdir.get_filepath('dem_noise')): #TODO: once
# surface noise is present, it cant get rid off ...
shutil.copy(self.gdir.get_filepath('dem_noise'),
self.get_subdir_filepath('dem_noise'))
self.surf_noise = np.load(self.get_subdir_filepath('dem_noise'))
else:
self.surf_noise = None
if os.path.exists(self.gdir.get_filepath('bed_measurements')):
shutil.copy(self.gdir.get_filepath('bed_measurements'),
self.get_subdir_filepath('bed_measurements'))
self.bed_measurements = np.load(
self.get_subdir_filepath('bed_measurements'))
else:
self.bed_measurements = None
def create_glacier(gdir, run_spinup=True):
"""
Creates a DEM-file for a glacier surface by running a forward model
for spin-up to a first state and based on this state further on to a
next state
Parameters
----------
gdir: NonRGIGlacierDirectory
GlacierDirectory possibly containing spinup-state and used for
saving the final reference state
run_spinup: bool
whether to run spin-up or rely on existing state
Returns
-------
"""
inv_settings = gdir.inversion_settings
if run_spinup:
spinup(gdir,
inv_settings['case'],
inv_settings['yrs_spinup'],
mb=inv_settings['mb_spinup'])
spinup_it = np.load(gdir.get_filepath('spinup_ice_thickness'))
spinup_surf = salem.GeoTiff(gdir.get_filepath('spinup_dem')).get_vardata()
with rasterio.open(gdir.get_filepath('dem')) as src:
bed = src.read(1)
profile = src.profile
ref_surf = run_forward(gdir,
inv_settings['case'],
inv_settings['yrs_forward_run'],
bed,
mb=inv_settings['mb_forward_run'],
init_ice_thick=spinup_it)
profile['dtype'] = 'float32'
with rasterio.open(gdir.get_filepath('ref_dem'), 'w', **profile) as dst:
dst.write(ref_surf, 1)
ref_surf = salem.GeoTiff(gdir.get_filepath('ref_dem')).get_vardata()
ref_it = ref_surf - bed
ref_ice_mask = ref_it > 0
np.save(gdir.get_filepath('ref_ice_thickness'), ref_it)
np.save(gdir.get_filepath('ref_ice_mask'), ref_ice_mask)
def plot_iterative_behaviour(gdir,
subdir,
figsize=(4.5, 3),
file_extension='png',
reset=False):
fig = plt.figure(figsize=figsize)
case = gdir.case
ref_surf = salem.GeoTiff(gdir.get_filepath('ref_dem')).get_vardata()
inv_settings = load_pickle(
get_subdir_filepath(gdir, subdir, 'inversion_settings'))
noise = 0.
if os.path.exists(get_subdir_filepath(gdir, subdir, 'dem_noise')):
noise = np.load(get_subdir_filepath(gdir, subdir, 'dem_noise'))
noisy_ref_surf = ref_surf + noise
ref_ice_mask = np.load(gdir.get_filepath('ref_ice_mask'))
ref_inner_mask = compute_inner_mask(ref_ice_mask, full_array=True)
dl = load_pickle(get_subdir_filepath(gdir, subdir, 'data_logger'))
reg_parameters = inv_settings['reg_parameters']
interesting_costs = []
cost_names = []
for l, lamb in enumerate(reg_parameters):
if lamb != 0:
interesting_costs.append(l)
cost_names.append('J{:d}'.format(l))
interesting_costs.append(-1)
cost_names.append('surf_misfit')
# make sure all directories exist:
plot_dir = os.path.join(gdir.dir, subdir, 'plot')
if reset:
if os.path.exists(plot_dir):
shutil.rmtree(plot_dir)
#if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
os.makedirs(os.path.join(plot_dir, 'bed_error'))
os.makedirs(os.path.join(plot_dir, 'surf_error'))
os.makedirs(os.path.join(plot_dir, 'summed_cost'))
os.makedirs(os.path.join(plot_dir, 'gradient'))
for c_name in cost_names:
os.makedirs(os.path.join(plot_dir, c_name))
dl.plot_rmses(plot_dir)
dl.plot_c_terms(plot_dir)
for i in dl.step_indices:
plot_iterative_step(dl,
i,
interesting_costs,
cost_names,
plot_dir,
case,
ref_ice_mask,
ref_inner_mask,
noisy_ref_surf,
reg_parameters,
file_extension,
existing_fig=fig)
plt.close(fig)
def add_consensus_thickness(gdir, base_url=None):
"""Add the consensus thickness estimate to the gridded_data file.
varname: consensus_ice_thickness
Parameters
----------
gdir ::py:class:`oggm.GlacierDirectory`
the glacier directory to process
base_url : str
where to find the thickness data. Default is
https://cluster.klima.uni-bremen.de/~fmaussion/icevol/composite
"""
if base_url is None:
base_url = default_base_url
if not base_url.endswith('/'):
base_url += '/'
rgi_str = gdir.rgi_id
rgi_reg_str = rgi_str[:8]
url = base_url + rgi_reg_str + '/' + rgi_str + '_thickness.tif'
input_file = utils.file_downloader(url)
dsb = salem.GeoTiff(input_file)
thick = utils.clip_min(dsb.get_vardata(), 0)
in_volume = thick.sum() * dsb.grid.dx**2
thick = gdir.grid.map_gridded_data(thick, dsb.grid, interp='linear')
# Correct for volume
thick = utils.clip_min(thick.filled(0), 0)
out_volume = thick.sum() * gdir.grid.dx**2
if out_volume > 0:
thick *= in_volume / out_volume
# We mask zero ice as nodata
thick = np.where(thick == 0, np.NaN, thick)
# Write
with utils.ncDataset(gdir.get_filepath('gridded_data'), 'a') as nc:
vn = 'consensus_ice_thickness'
if vn in nc.variables:
v = nc.variables[vn]
else:
v = nc.createVariable(vn, 'f4', (
'y',
'x',
), zlib=True)
v.units = 'm'
ln = 'Ice thickness from the consensus estimate'
v.long_name = ln
v.base_url = base_url
v[:] = thick
def region_grid(reg):
global region_grids
if reg not in region_grids:
with utils.get_lock():
fp = utils.file_downloader(region_files[reg]['vx'])
ds = salem.GeoTiff(fp)
region_grids[reg] = ds.grid
return region_grids[reg]
def generate_bed_measurements(gdir, bed_measurements_mask, std=0):
true_bed = salem.GeoTiff(gdir.get_filepath('dem')).get_vardata()
noise = std * np.random.randn(*true_bed.shape)
bed_measurements = (true_bed + noise) * bed_measurements_mask
bed_measurements[np.logical_not(bed_measurements_mask)] = -np.inf
bed_measurements = np.ma.masked_array(
bed_measurements, mask=np.logical_not(bed_measurements_mask))
print('Actual RMSE of bed measurements: {:g}'.format(
RMSE(bed_measurements, true_bed)))
# TODO: apply std scaling after masking ...?
# TODO: investigate deviations of RMSE from numpys std
return bed_measurements
def add_noise_to_first_guess(gdir, noise, cut_noise=True, min_ice_thick=5):
"""
Adds noise to the first guess. Saves this to the file with the first guess
and also saves the applied noise.
Parameters
----------
gdir: NonRGIGlacierDirectory
GlacierDirectory containing the first guess
noise: ndarray
noise to apply to the first guess
cut_noise: bool
whether or not the noise should be cut to not penetrate the surface
and require a minimum ice thickness if applied to first guess
min_ice_thick: float
minimum ice thickness, only applied if noise is cut
"""
fg_filepath = gdir.get_filepath('first_guessed_bed')
with rasterio.open(fg_filepath) as src:
first_guessed_bed = src.read(1)
profile = src.profile
if cut_noise:
ref_ice_mask = np.load(gdir.get_filepath('ref_ice_mask'))
desired_rmse = RMSE(noise, 0, ref_ice_mask)
ref_surf = salem.GeoTiff(gdir.get_filepath('ref_dem')).get_vardata()
penetrating = (first_guessed_bed + noise - min_ice_thick > ref_surf)
penetrating *= ref_ice_mask
noise = np.where(penetrating,
ref_surf - first_guessed_bed - min_ice_thick, noise)
# TODO: will result in problems -> iteratively?
rmse = RMSE(noise, 0, ref_ice_mask)
print('desired rmse: {:g}\\rmse after cutting: {:g}'.format(
desired_rmse, rmse))
# noise *= desired_rmse / rmse # rescale to desired RMSE
# if np.any(first_guessed_bed + noise > ref_surf):
# raise ValueError('First guess is found to penetrate ice surface; '
# 'Aborting')
first_guessed_bed = first_guessed_bed + noise
profile['dtype'] = 'float64'
with rasterio.open(fg_filepath, 'w', **profile) as dst:
dst.write(first_guessed_bed, 1)
np.save(gdir.get_filepath('first_guessed_bed_noise'), noise)
def define_g2ti_glacier(path=None, base_dir=None):
fname = os.path.join(path, 'outlines.shp')
ent = gpd.read_file(fname)
rid = ent.RGIId.values[0]
if '5a' in rid:
rid = rid.replace('5a', '60')
ent['RGIId'] = rid
ent['Name'] = '' if ent['Name'][0] == 'None' else ent['Name']
gdir = utils.GlacierDirectory(ent.iloc[0], base_dir=base_dir)
ent.to_file(gdir.get_filepath('outlines'))
proj_out = salem.check_crs(ent.crs)
# Also transform the intersects if necessary
gdf = cfg.PARAMS['intersects_gdf']
if len(gdf) > 0:
gdf = gdf.loc[((gdf.RGIId_1 == gdir.rgi_id) |
(gdf.RGIId_2 == gdir.rgi_id))]
if len(gdf) > 0:
gdf = salem.transform_geopandas(gdf, to_crs=proj_out)
if hasattr(gdf.crs, 'srs'):
# salem uses pyproj
gdf.crs = gdf.crs.srs
gdf.to_file(gdir.get_filepath('intersects'))
else:
# Sanity check
if cfg.PARAMS['use_intersects']:
raise RuntimeError('You seem to have forgotten to set the '
'intersects file for this run. OGGM works '
'better with such a file. If you know what '
'your are doing, set '
"cfg.PARAMS['use_intersects'] = False to "
"suppress this error.")
# Topo
shutil.copy(os.path.join(path, 'dem.tif'), gdir.get_filepath('dem'))
mpath = gdir.get_filepath('dem').replace('dem', 'g2ti_mask')
shutil.copy(os.path.join(path, 'mask.tif'), mpath)
# Grid
ds = salem.GeoTiff(gdir.get_filepath('dem'))
ds.grid.to_json(gdir.get_filepath('glacier_grid'))
gdir.write_pickle(['G2TI'], 'dem_source')
return gdir
def oggm_to_g2ti(gdir, dirname='final'):
"""From an oggm gdir to a g2ti tiff
Parameters
----------
"""
# Get the data
grids_file = gdir.get_filepath('gridded_data')
with netCDF4.Dataset(grids_file) as nc:
with warnings.catch_warnings():
# https://github.com/Unidata/netcdf4-python/issues/766
warnings.filterwarnings("ignore", category=RuntimeWarning)
thick = nc.variables['distributed_thickness'][:]
dx = gdir.grid.dx
vol = np.nansum(thick * dx**2)
tpl_f = os.path.join(g2ti.geometry_dir, gdir.rgi_id[:8], gdir.rgi_id,
'dem.tif')
dst = salem.GeoTiff(tpl_f)
thick[~np.isfinite(thick)] = 0.
dst_thick = dst.grid.map_gridded_data(thick,
grid=gdir.grid,
interp='spline')
dst_thick[~np.isfinite(dst_thick)] = 0.
# Conserve volume
dx = dst.grid.dx
dst_thick *= vol / np.nansum(dst_thick * dx**2)
if not np.isclose(vol / np.nansum(dst_thick * dx**2), 1, atol=0.2):
raise RuntimeError('Something went wrong in reproj.')
with rasterio.open(tpl_f) as orig:
# Set up profile for writing output
profile = orig.profile
ft = os.path.join(cfg.PATHS['working_dir'], dirname,
'RGI60-{}'.format(gdir.rgi_region))
utils.mkdir(ft)
ft = os.path.join(ft, 'thickness_{}.tif'.format(gdir.rgi_id))
with rasterio.open(ft, 'w', **profile) as dest:
dest.write(dst_thick.astype(np.float32).clip(0), 1)
def plot_domain(gdirs, ax=None, smap=None):
"""Plot the glacier directory."""
# Files
gdir = gdirs[0]
topo = salem.GeoTiff(gdir.get_filepath('dem')).get_vardata()
try:
smap.set_data(topo)
except ValueError:
pass
cm = truncate_colormap(ALTITUDE_CMAP, minval=0.25, maxval=1.0, n=256)
smap.set_cmap(cm)
smap.set_plot_params(nlevels=256)
for gdir in gdirs:
crs = gdir.grid.center_grid
try:
geom = gdir.read_pickle('geometries')
# Plot boundaries
poly_pix = geom['polygon_pix']
smap.set_geometry(poly_pix,
crs=crs,
fc='white',
alpha=0.3,
zorder=2,
linewidth=.2)
poly_pix = utils.tolist(poly_pix)
for _poly in poly_pix:
for l in _poly.interiors:
smap.set_geometry(l, crs=crs, color='black', linewidth=0.5)
except FileNotFoundError:
smap.set_shapefile(gdir.read_shapefile('outlines'))
smap.plot(ax)
return dict(cbar_label='Alt. [m]')
def compile_biased_first_guess(gdir, desired_mean_bias):
ref_ice_thickness = np.load(gdir.get_filepath('ref_ice_thickness'))
ref_ice_mask = np.load(gdir.get_filepath('ref_ice_mask'))
true_bed = salem.GeoTiff(gdir.get_filepath('dem')).get_vardata()
with rasterio.open(gdir.get_filepath('ref_dem')) as src:
surf = src.read(1)
profile = src.profile
gamma = desired_mean_bias / (ref_ice_thickness.sum() / ref_ice_mask.sum())
if gamma > 1:
raise AttributeError('Given desired mean bias not applicable;'
'ice volume exceeded')
first_guessed_bed = true_bed + gamma * ref_ice_thickness
with rasterio.open(gdir.get_filepath('first_guessed_bed'), 'w',
**profile) as dst:
dst.write(first_guessed_bed, 1)
return first_guessed_bed
def compile_rmsed_first_guess(gdir, desired_rmse):
ref_ice_thickness = np.load(gdir.get_filepath('ref_ice_thickness'))
ref_ice_mask = np.load(gdir.get_filepath('ref_ice_mask'))
true_bed = salem.GeoTiff(gdir.get_filepath('dem')).get_vardata()
with rasterio.open(gdir.get_filepath('ref_dem')) as src:
surf = src.read(1)
profile = src.profile
std = desired_rmse
perturbation = std * np.random.randn(*ref_ice_mask.shape) * ref_ice_mask
if np.any(perturbation > ref_ice_thickness) > 1:
raise AttributeError('Perturbation peek through surface; Aborting')
first_guessed_bed = true_bed + perturbation
first_guessed_bed.dtype = np.float32
with rasterio.open(gdir.get_filepath('first_guessed_bed'), 'w',
**profile) as dst:
dst.write(first_guessed_bed, 1)
return first_guessed_bed
def test_distribute(self):
gdir = g2task.define_g2ti_glacier(self.idir, base_dir=self.testdir)
gis.glacier_masks(gdir)
g2task.g2ti_masks(gdir)
out = g2task.distribute_thickness_vas(gdir,
vas_c=0.034,
dis_factor=0.2,
topo_factor=0.2,
write_tiff=True)
ref = out * np.NaN
i, j, _, _, thick = g2task.get_ref_gtd_data(gdir)
ref[j, i] = thick
out = xr.DataArray(out)
ref = xr.DataArray(ref)
ft = os.path.join(cfg.PATHS['working_dir'], 'final',
'RGI60-{}'.format(gdir.rgi_region))
ft = os.path.join(ft, 'thickness_{}.tif'.format(gdir.rgi_id))
tif = xr.open_rasterio(ft)
dx2 = salem.GeoTiff(ft).grid.dx**2
np.testing.assert_allclose((tif * dx2).sum() * 1e-9,
0.034 * gdir.rgi_area_km2**1.375,
rtol=0.01)
if do_plot:
import matplotlib.pyplot as plt
out.plot()
plt.figure()
ref.plot()
plt.figure()
tif.plot()
plt.show()
def test_mb(self):
# This is a function to produce the MB function needed by Anna
# Download the RGI file for the run
# Make a new dataframe of those
rgidf = gpd.read_file(get_demo_file('SouthGlacier.shp'))
# Go - initialize working directories
gdirs = workflow.init_glacier_regions(rgidf)
# Preprocessing tasks
task_list = [
tasks.glacier_masks,
tasks.compute_centerlines,
tasks.initialize_flowlines,
tasks.catchment_area,
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
]
for task in task_list:
execute_entity_task(task, gdirs)
# Climate tasks -- only data IO and tstar interpolation!
execute_entity_task(tasks.process_cru_data, gdirs)
tasks.distribute_t_stars(gdirs)
execute_entity_task(tasks.apparent_mb, gdirs)
mbref = salem.GeoTiff(get_demo_file('mb_SouthGlacier.tif'))
demref = salem.GeoTiff(get_demo_file('dem_SouthGlacier.tif'))
mbref = mbref.get_vardata()
mbref[mbref == -9999] = np.NaN
demref = demref.get_vardata()[np.isfinite(mbref)]
mbref = mbref[np.isfinite(mbref)] * 1000
# compute the bias to make it 0 SMB on the 2D DEM
mbmod = ConstantMassBalance(gdirs[0], bias=0)
mymb = mbmod.get_annual_mb(demref) * cfg.SEC_IN_YEAR * cfg.RHO
mbmod = ConstantMassBalance(gdirs[0], bias=np.average(mymb))
mymb = mbmod.get_annual_mb(demref) * cfg.SEC_IN_YEAR * cfg.RHO
np.testing.assert_allclose(np.average(mymb), 0., atol=1e-3)
# Same for ref
mbref = mbref - np.average(mbref)
np.testing.assert_allclose(np.average(mbref), 0., atol=1e-3)
# Fit poly
p = np.polyfit(demref, mbref, deg=2)
poly = np.poly1d(p)
myfit = poly(demref)
np.testing.assert_allclose(np.average(myfit), 0., atol=1e-3)
if do_plot:
import matplotlib.pyplot as plt
plt.scatter(mbref, demref, s=5, label='Obs (2007-2012), shifted to '
'Avg(SMB) = 0')
plt.scatter(mymb, demref, s=5, label='OGGM MB at t*')
plt.scatter(myfit, demref, s=5, label='Polyfit', c='C3')
plt.xlabel('MB (mm w.e yr-1)')
plt.ylabel('Altidude (m)')
plt.legend()
plt.show()
bounds_min_max=(2, 1000)
)
# Optional, if not reset=True and already ran once
# only needed once:
#create_glacier(gdir)
fg = compile_first_guess(gdir)
#idir = InversionDirectory(gdir)
mb = gdir.inversion_settings['mb_forward_run']
if mb is None:
mb = case.get_mb_model()
guessed_bed = torch.tensor(fg, dtype=torch.float, requires_grad=False)
spinup_surf = salem.GeoTiff(gdir.get_filepath('spinup_dem')).get_vardata()
ref_surf = salem.GeoTiff(gdir.get_filepath('ref_dem')).get_vardata()
ice_mask = np.load(gdir.get_filepath('ref_ice_mask'))
# run model forward
init_ice_thick = spinup_surf - guessed_bed
model_surf = run_forward_core(gdir.inversion_settings['yrs_forward_run'],
guessed_bed, case.dx, case.get_mb_model(),
init_ice_thick).detach().numpy()
print('Bias: ' + str(np.sum(model_surf - ref_surf)/ice_mask.sum()))
plt.imshow(model_surf - ref_surf)
plt.show()
#dl = data_logging.load_pickle(idir.get_current_basedir() + '/data_logger.pkl')
bed = salem.GeoTiff(gdir.get_filepath('dem')).get_vardata()
print('Bias: ' + str(np.sum(fg - bed)/ice_mask.sum()))
def _reproject_and_scale(gdir, do_error=False):
"""Reproject and scale itslive data, avoid code duplication for error"""
reg = find_region(gdir)
if reg is None:
raise InvalidWorkflowError('There does not seem to be its_live data '
'available for this glacier')
vnx = 'vx'
vny = 'vy'
if do_error:
vnx += '_err'
vny += '_err'
with utils.get_lock():
fx = utils.file_downloader(region_files[reg][vnx])
fy = utils.file_downloader(region_files[reg][vny])
# Open the files
dsx = salem.GeoTiff(fx)
dsy = salem.GeoTiff(fy)
# subset them to our map
grid_gla = gdir.grid.center_grid
proj_vel = dsx.grid.proj
x0, x1, y0, y1 = grid_gla.extent_in_crs(proj_vel)
dsx.set_subset(corners=((x0, y0), (x1, y1)), crs=proj_vel, margin=4)
dsy.set_subset(corners=((x0, y0), (x1, y1)), crs=proj_vel, margin=4)
grid_vel = dsx.grid.center_grid
# TODO: this should be taken care of by salem
# https://github.com/fmaussion/salem/issues/171
with rasterio.Env():
with rasterio.open(fx) as src:
nodata = getattr(src, 'nodata', -32767.0)
# Error files are wrong
if nodata == 0:
nodata = -32767.0
# Get the coords at t0
xx0, yy0 = grid_vel.center_grid.xy_coordinates
# Compute coords at t1
xx1 = dsx.get_vardata()
yy1 = dsy.get_vardata()
non_valid = (xx1 == nodata) | (yy1 == nodata)
xx1[non_valid] = np.NaN
yy1[non_valid] = np.NaN
orig_vel = np.sqrt(xx1**2 + yy1**2)
xx1 += xx0
yy1 += yy0
# Transform both to glacier proj
xx0, yy0 = salem.transform_proj(proj_vel, grid_gla.proj, xx0, yy0)
xx1, yy1 = salem.transform_proj(proj_vel, grid_gla.proj, xx1, yy1)
# Correct no data after proj as well (inf)
xx1[non_valid] = np.NaN
yy1[non_valid] = np.NaN
# Compute velocities from there
vx = xx1 - xx0
vy = yy1 - yy0
# Scale back velocities - https://github.com/OGGM/oggm/issues/1014
new_vel = np.sqrt(vx**2 + vy**2)
p_ok = new_vel > 1e-5 # avoid div by zero
vx[p_ok] = vx[p_ok] * orig_vel[p_ok] / new_vel[p_ok]
vy[p_ok] = vy[p_ok] * orig_vel[p_ok] / new_vel[p_ok]
# And transform to local map
vx = grid_gla.map_gridded_data(vx, grid=grid_vel, interp='linear')
vy = grid_gla.map_gridded_data(vy, grid=grid_vel, interp='linear')
# Write
with utils.ncDataset(gdir.get_filepath('gridded_data'), 'a') as nc:
vn = 'obs_icevel_x'
if do_error:
vn = vn.replace('obs', 'err')
if vn in nc.variables:
v = nc.variables[vn]
else:
v = nc.createVariable(vn, 'f4', ('y', 'x', ), zlib=True)
v.units = 'm yr-1'
ln = 'ITS LIVE velocity data in x map direction'
if do_error:
ln = 'Uncertainty of ' + ln
v.long_name = ln
v[:] = vx.filled(np.nan)
vn = 'obs_icevel_y'
if do_error:
vn = vn.replace('obs', 'err')
if vn in nc.variables:
v = nc.variables[vn]
else:
v = nc.createVariable(vn, 'f4', ('y', 'x', ), zlib=True)
v.units = 'm yr-1'
ln = 'ITS LIVE velocity data in y map direction'
if do_error:
ln = 'Uncertainty of ' + ln
v.long_name = ln
v[:] = vy.filled(np.nan)
from combine2d.core.arithmetics import compute_inner_mask
cfg.initialize()
basedir = '/home/philipp/HR_01/'
file_extension = 'png'
case = test_cases.BordenHR
dx = case.dx
gdir = NonRGIGlacierDirectory(case, basedir)
experiment = 'identical-twin 0'
output_dir = os.path.join(gdir.dir, experiment, 'plots')
figsize = (4.5, 3)
dl = load_pickle(os.path.join(gdir.dir, experiment, 'data_logger.pkl'))
ref_surf = salem.GeoTiff(gdir.get_filepath('ref_dem')).get_vardata()
ref_ice_mask = np.load(gdir.get_filepath('ref_ice_mask'))
#true_bed = salem.GeoTiff(gdir.get_filepath('dem')).get_vardata()
ref_inner_mask = compute_inner_mask(ref_ice_mask, full_array=True)
#plt.figure()
#plt.imshow(ref_ice_mask)
#plt.imshow(ref_inner_mask, cmap='RdBu')
#plt.show()
inversion_settings = load_pickle(
os.path.join(gdir.dir, experiment, 'inversion_settings.pkl'))
reg_parameters = inversion_settings['reg_parameters']
margin = np.logical_xor(ref_ice_mask, ref_inner_mask)
def get_costs_arr(reg_parameters, ref_surf, ref_ice_mask, ref_inner_mask,
def create_cost_func(gdir,
data_logger=None,
surface_noise=None,
bed_measurements=None):
"""
Creates a cost function based on the glacier directory.
Parameters
----------
gdir: NonRGIGlacierDirectory
GlacierDirectory containing precomputed spinup surface and
"observed" surface for final state. Further on contains inversion
settings
data_logger: DataLogger
optionally logs data
Returns
-------
tuple of (cost, grad) with cost as float and grad being a ndarray
with same shape as b
"""
# precompute known data to avoid recomputation during each call of
# cost_fucntion
conv_filter = torch.ones((1, 1, 3, 3), requires_grad=False)
# TODO: think about whether cross is better suited (in forward model no diagonal transport
#conv_filter = torch.tensor([[[[0, 1, 0], [1, 1, 1], [0, 1, 0]]]],
# dtype=torch.float, requires_grad=True)
spinup_surf = salem.GeoTiff(gdir.get_filepath('spinup_dem')).get_vardata()
ref_surf = salem.GeoTiff(gdir.get_filepath('ref_dem')).get_vardata()
if surface_noise is not None:
spinup_surf += surface_noise
ref_surf += surface_noise
# TODO: allow for independent surface perturbations
gpr = None
if bed_measurements is not None:
# PyTorch is a bit messy with masks.
# Instead we use full tensors and multiply by a mask.
gpr_data = torch.tensor(np.ma.filled(bed_measurements, -9999),
dtype=torch.float,
requires_grad=False)
gpr_mask = torch.tensor(1 - bed_measurements.mask,
dtype=torch.float,
requires_grad=False)
gpr = (gpr_data, gpr_mask)
spinup_surf = torch.tensor(spinup_surf,
dtype=torch.float,
requires_grad=False)
ref_surf = torch.tensor(ref_surf, dtype=torch.float, requires_grad=False)
ref_ice_mask = np.load(gdir.get_filepath('ref_ice_mask'))
ref_ice_mask = torch.tensor(ref_ice_mask.astype(np.int),
dtype=torch.float,
requires_grad=False)
ref_inner_mask = torch.zeros(ref_ice_mask.shape)
ref_inner_mask[1:-1, 1:-1] = torch.conv2d(
ref_ice_mask.unsqueeze(0).unsqueeze(0), conv_filter) == 9
inv_settings = gdir.inversion_settings
reg_parameters = inv_settings['reg_parameters']
lambs1 = np.ones(300) * 0.25
lambs1[:120] = np.logspace(-4, -1, 120)
yrs_to_run = inv_settings['yrs_forward_run']
case = inv_settings['case']
mb = inv_settings['mb_forward_run']
if mb is None:
mb = case.get_mb_model()
def c_fun(b):
"""
Wrapper for cost_function. First step for easy exchangeability
afterwards and to get a cost_function signature exhibiting all true
input arguments.
Parameters
----------
b: ndarray
bed heights for which the costs should be calculated. (unit: [m])
Returns
-------
tuple of (cost, grad) with cost as float and grad being a ndarray
with same shape as b
"""
return cost_function(b, reg_parameters, ref_surf, ref_ice_mask,
ref_inner_mask, spinup_surf, conv_filter,
yrs_to_run, case.dx, mb, gpr, data_logger, lambs1)
return c_fun
def velocity_to_gdir(gdir):
"""Reproject the its_live files to the given glacier directory.
Variables are added to the gridded_data nc file.
Reprojecting velocities from one map proj to another is done
reprojecting the vector distances. In this process, absolute velocities
might change as well because map projections do not always preserve
distances -> we scale them back to the original velocities as per the
ITS_LIVE documentation that states that velocities are given in
ground units, i.e. absolute velocities.
We use bilinear interpolation to reproject the velocities to the local
glacier map.
Parameters
----------
gdir : :py:class:`oggm.GlacierDirectory`
where to write the data
"""
reg = find_region(gdir)
if reg is None:
raise InvalidWorkflowError('There does not seem to be its_live data '
'available for this glacier')
if not gdir.has_file('gridded_data'):
raise InvalidWorkflowError('Please run `glacier_masks` before running '
'this task')
with utils.get_lock():
fx = utils.file_downloader(region_files[reg]['vx'])
fy = utils.file_downloader(region_files[reg]['vy'])
# Open the files
dsx = salem.GeoTiff(fx)
dsy = salem.GeoTiff(fy)
# subset them to our map
grid_gla = gdir.grid.center_grid
proj_vel = dsx.grid.proj
x0, x1, y0, y1 = grid_gla.extent_in_crs(proj_vel)
dsx.set_subset(corners=((x0, y0), (x1, y1)), crs=proj_vel, margin=4)
dsy.set_subset(corners=((x0, y0), (x1, y1)), crs=proj_vel, margin=4)
grid_vel = dsx.grid.center_grid
# TODO: this should be taken care of by salem
# https://github.com/fmaussion/salem/issues/171
with rasterio.Env():
with rasterio.open(fx) as src:
nodata = getattr(src, 'nodata', -32767.0)
# Get the coords at t0
xx0, yy0 = grid_vel.center_grid.xy_coordinates
# Compute coords at t1
xx1 = dsx.get_vardata()
yy1 = dsy.get_vardata()
non_valid = (xx1 == nodata) | (yy1 == nodata)
xx1[non_valid] = np.NaN
yy1[non_valid] = np.NaN
orig_vel = np.sqrt(xx1**2 + yy1**2)
xx1 += xx0
yy1 += yy0
# Transform both to glacier proj
xx0, yy0 = salem.transform_proj(proj_vel, grid_gla.proj, xx0, yy0)
xx1, yy1 = salem.transform_proj(proj_vel, grid_gla.proj, xx1, yy1)
# Correct no data after proj as well (inf)
xx1[non_valid] = np.NaN
yy1[non_valid] = np.NaN
# Compute velocities from there
vx = xx1 - xx0
vy = yy1 - yy0
# Scale back velocities - https://github.com/OGGM/oggm/issues/1014
new_vel = np.sqrt(vx**2 + vy**2)
p_ok = new_vel > 0.1 # avoid div by zero
vx[p_ok] = vx[p_ok] * orig_vel[p_ok] / new_vel[p_ok]
vy[p_ok] = vy[p_ok] * orig_vel[p_ok] / new_vel[p_ok]
# And transform to local map
vx = grid_gla.map_gridded_data(vx, grid=grid_vel, interp='linear')
vy = grid_gla.map_gridded_data(vy, grid=grid_vel, interp='linear')
# Write
with utils.ncDataset(gdir.get_filepath('gridded_data'), 'a') as nc:
vn = 'obs_icevel_x'
if vn in nc.variables:
v = nc.variables[vn]
else:
v = nc.createVariable(vn, 'f4', (
'y',
'x',
), zlib=True)
v.units = 'm yr-1'
v.long_name = 'ITS LIVE velocity data in x map direction'
v[:] = vx
vn = 'obs_icevel_y'
if vn in nc.variables:
v = nc.variables[vn]
else:
v = nc.createVariable(vn, 'f4', (
'y',
'x',
), zlib=True)
v.units = 'm yr-1'
v.long_name = 'ITS LIVE velocity data in xy map direction'
v[:] = vy
def create_cost_func(gdir,
data_logger=None,
use_AD=False,
torch_backward_param=None):
"""
Creates a cost function based on the glacier directory.
Parameters
----------
gdir: NonRGIGlacierDirectory
GlacierDirectory containing precomputed spinup surface and
"observed" surface for final state. Further on contains inversion
settings
data_logger: DataLogger
optionally logs data
use_AD: bool
whether or not to use PyTorch AD for gradient calculation.
Determines whether created cost function gives [cost,] or
[cost, gradient]
torch_backward_param: scalar
Exploit 'special' parameter of torch if not None. Only has effect if
use_AD = True
Returns
-------
tuple of (cost, grad) with cost as float and grad being a ndarray
with same shape as b
"""
# precompute known data to avoid recomputation during each call of
# cost_fucntion
conv_filter = torch.ones((1, 1, 3, 3), requires_grad=False)
spinup_surf = salem.GeoTiff(gdir.get_filepath('spinup_dem')).get_vardata()
spinup_surf = torch.tensor(spinup_surf,
dtype=torch.float,
requires_grad=False)
ref_surf = salem.GeoTiff(gdir.get_filepath('ref_dem')).get_vardata()
ref_surf = torch.tensor(ref_surf, dtype=torch.float, requires_grad=False)
ref_ice_mask = np.load(gdir.get_filepath('ref_ice_mask'))
ref_ice_mask = torch.tensor(ref_ice_mask.astype(np.int),
dtype=torch.float,
requires_grad=False)
ref_inner_mask = torch.zeros(ref_ice_mask.shape)
ref_inner_mask[1:-1, 1:-1] = torch.conv2d(
ref_ice_mask.unsqueeze(0).unsqueeze(0), conv_filter) == 9
inv_settings = gdir.inversion_settings
reg_parameters = inv_settings['reg_parameters']
yrs_to_run = inv_settings['yrs_forward_run']
case = inv_settings['case']
mb = inv_settings['mb_forward_run']
if mb is None:
mb = case.get_mb_model()
def c_fun(b):
"""
Wrapper for cost_function. First step for easy exchangeability
afterwards and to get a cost_function signature exhibiting all true
input arguments.
Parameters
----------
b: ndarray
bed heights for which the costs should be calculated. (unit: [m])
Returns
-------
tuple of (cost, grad) with cost as float and grad being a ndarray
with same shape as b
"""
return cost_function(b, reg_parameters, ref_surf, ref_ice_mask,
ref_inner_mask, spinup_surf, conv_filter,
yrs_to_run, case.dx, mb, data_logger, use_AD,
torch_backward_param)
return c_fun
err_calving_front = []
rel_tol_fls = []
rel_tol_calving_front = []
length_fls = []
files_no_data = []
# We need the components of vel
fx = os.path.join(MAIN_PATH, config['vel_x_path'])
fy = os.path.join(MAIN_PATH, config['vel_y_path'])
# And Error
ex = os.path.join(MAIN_PATH, config['err_x_path'])
ey = os.path.join(MAIN_PATH, config['err_y_path'])
# Open the files with salem
dsx = salem.GeoTiff(fx)
dsy = salem.GeoTiff(fy)
dex = salem.GeoTiff(ex)
dey = salem.GeoTiff(ey)
for gdir in gdirs:
# first we compute the centerlines as shapefile to crop the satellite
# data
misc.write_flowlines_to_shape(gdir, path=gdir.dir)
shp_path = os.path.join(gdir.dir, 'RGI60-05.shp')
shp = gpd.read_file(shp_path)
utils_vel.its_live_to_gdir(gdir, dsx=dsx, dsy=dsy, dex=dex, dey=dey, fx=fx)
Returns
-------
"""
with rasterio.open(gdir.get_filepath('dem')) as src:
bed = src.read(1)
profile = src.profile
spinup_surf = run_forward(gdir, case, yr_spinup_end, bed, mb=mb)
profile['dtype'] = 'float32'
with rasterio.open(gdir.get_filepath('spinup_dem'), 'w', **profile) as dst:
dst.write(spinup_surf, 1)
spinup_surf = salem.GeoTiff(gdir.get_filepath('spinup_dem')).get_vardata()
spinup_it = spinup_surf - bed
np.save(gdir.get_filepath('spinup_ice_thickness'), spinup_it)
return spinup_surf, spinup_it
@torch.no_grad()
def run_forward(gdir, case, yrs, bed, mb=None, init_ice_thick=None):
"""
Wrapper for run_forward_core. Can derive mass-balance from case, if not
given, accepts strings for bed_files as well as ndarrays and tensors
Parameters
----------
from combine2d.core import gis, test_cases
from combine2d.core.utils import NonRGIGlacierDirectory
from combine2d.core.inversion import InversionDirectory
from combine2d.core.data_logging import load_pickle
from oggm import cfg
cfg.initialize()
basedir = '/data/philipp/thesis_test/Giluwe/perfect'
case = test_cases.Giluwe
gdir = NonRGIGlacierDirectory(case, basedir)
idir = InversionDirectory(gdir)
idir.inv_settings['inversion_counter'] = 1001
true_bed = salem.GeoTiff(gdir.get_filepath('dem')).get_vardata()
data_logger = load_pickle(idir.get_subdir_filepath('data_logger'))
plt.figure()
plt.imshow(data_logger.beds[-1] - true_bed)
subsampled_beds = data_logger.beds[-3:]
for i in [3, 4, 5, 6, 7, 8, 9, 10]:
yfrom = 6
yend = 8
y = i
xfrom = 0
xend = -1
