def plot_iterative_behaviour(gdir,
subdir,
figsize=(4.5, 3),
file_extension='png',
reset=False):
fig = plt.figure(figsize=figsize)
case = gdir.case
ref_surf = salem.GeoTiff(gdir.get_filepath('ref_dem')).get_vardata()
inv_settings = load_pickle(
get_subdir_filepath(gdir, subdir, 'inversion_settings'))
noise = 0.
if os.path.exists(get_subdir_filepath(gdir, subdir, 'dem_noise')):
noise = np.load(get_subdir_filepath(gdir, subdir, 'dem_noise'))
noisy_ref_surf = ref_surf + noise
ref_ice_mask = np.load(gdir.get_filepath('ref_ice_mask'))
ref_inner_mask = compute_inner_mask(ref_ice_mask, full_array=True)
dl = load_pickle(get_subdir_filepath(gdir, subdir, 'data_logger'))
reg_parameters = inv_settings['reg_parameters']
interesting_costs = []
cost_names = []
for l, lamb in enumerate(reg_parameters):
if lamb != 0:
interesting_costs.append(l)
cost_names.append('J{:d}'.format(l))
interesting_costs.append(-1)
cost_names.append('surf_misfit')
# make sure all directories exist:
plot_dir = os.path.join(gdir.dir, subdir, 'plot')
if reset:
if os.path.exists(plot_dir):
shutil.rmtree(plot_dir)
#if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
os.makedirs(os.path.join(plot_dir, 'bed_error'))
os.makedirs(os.path.join(plot_dir, 'surf_error'))
os.makedirs(os.path.join(plot_dir, 'summed_cost'))
os.makedirs(os.path.join(plot_dir, 'gradient'))
for c_name in cost_names:
os.makedirs(os.path.join(plot_dir, c_name))
dl.plot_rmses(plot_dir)
dl.plot_c_terms(plot_dir)
for i in dl.step_indices:
plot_iterative_step(dl,
i,
interesting_costs,
cost_names,
plot_dir,
case,
ref_ice_mask,
ref_inner_mask,
noisy_ref_surf,
reg_parameters,
file_extension,
existing_fig=fig)
plt.close(fig)
def eval_identical_twin(idir):
header = 'case,run,icevolerr,rmsebed,rmsesurf,biasbed,biassurf,' \
'corr,rmsefg,biasfg,iterations,' \
'maxbeddiffglacier,maxbeddiffdomain,' \
'minbeddiffglacier,minbeddiffdomain,' \
'voloutsidebounds\n'
row = '{case:s},{run:s},{dV:.2f},{rmsebed:.1f},{rmsesurf:.1f},' \
'{biasbed:.1f},{biassurf:.1f},{corr:.3f},{rmsefg:.1f},' \
'{biasfg:.1f},{iterations:d},' \
'{maxbeddiffglacier:.1f},{maxbeddiffdomain:.1f},' \
'{minbeddiffglacier:.1f},{minbeddiffdomain:.1f},' \
'{voloutsidebounds:.9f}'
# TODO: max_bed_diff?
dl = load_pickle(idir.get_subdir_filepath('data_logger'))
vals = {}
vals['case'] = dl.case.name
vals['run'] = idir.inv_settings['inversion_subdir']
ref_it = np.load(idir.gdir.get_filepath('ref_ice_thickness'))
mod_it = (dl.surfs[-1] - dl.beds[-1])
ref_vol = ref_it.sum()
mod_vol = mod_it.sum()
vals['dV'] = (mod_vol - ref_vol) / ref_vol * 1e2
ref_ice_mask = np.load(idir.gdir.get_filepath('ref_ice_mask'))
vals['rmsebed'] = RMSE(dl.true_bed, dl.beds[-1], ref_ice_mask)
vals['rmsesurf'] = RMSE(dl.ref_surf, dl.surfs[-1], ref_ice_mask)
vals['rmsefg'] = RMSE(dl.true_bed, dl.first_guessed_bed, ref_ice_mask)
vals['biasbed'] = mean_BIAS(dl.beds[-1], dl.true_bed, ref_ice_mask)
vals['biassurf'] = mean_BIAS(dl.surfs[-1], dl.ref_surf, ref_ice_mask)
vals['biasfg'] = mean_BIAS(dl.first_guessed_bed, dl.true_bed, ref_ice_mask)
masked_true_it = np.ma.masked_array(dl.ref_surf - dl.true_bed,
mask=np.logical_not(ref_ice_mask))
masked_mod_it = np.ma.masked_array(dl.surfs[-1] - dl.beds[-1],
mask=np.logical_not(ref_ice_mask))
# TODO: ice thickness
vals['corr'] = np.ma.corrcoef(masked_true_it.flatten(),
masked_mod_it.flatten())[0, 1]
vals['iterations'] = len(dl.step_indices)
#vals['maxbeddiff'] = np.max((dl.beds[-1] - dl.true_bed) * ref_ice_mask)
vals['maxbeddiffglacier'] = np.max(
(dl.beds[-1] - dl.true_bed) * ref_ice_mask)
vals['maxbeddiffdomain'] = np.max(dl.beds[-1] - dl.true_bed)
#vals['minbeddiff'] = np.min((dl.beds[-1] - dl.true_bed) * ref_ice_mask)
vals['minbeddiffglacier'] = np.min(
(dl.beds[-1] - dl.true_bed) * ref_ice_mask)
vals['minbeddiffdomain'] = np.min(dl.beds[-1] - dl.true_bed)
vals['voloutsidebounds'] = np.sum(
(dl.surfs[-1] - dl.beds[-1]) * (1 - ref_ice_mask)) * 1e-9
data_row = row.format(**vals)
with open(idir.get_subdir_filepath('results'), 'w') as f:
f.writelines([header, data_row])
return [header, data_row]
def inversion_settings(self):
"""Dictionary with settings for the inversion"""
return load_pickle(self.get_filepath('inversion_settings'))
from scipy.signal import convolve2d
from combine2d.core.arithmetics import compute_inner_mask
cfg.initialize()
basedir = '/home/philipp/HR_01/'
file_extension = 'png'
case = test_cases.BordenHR
dx = case.dx
gdir = NonRGIGlacierDirectory(case, basedir)
experiment = 'identical-twin 0'
output_dir = os.path.join(gdir.dir, experiment, 'plots')
figsize = (4.5, 3)
dl = load_pickle(os.path.join(gdir.dir, experiment, 'data_logger.pkl'))
ref_surf = salem.GeoTiff(gdir.get_filepath('ref_dem')).get_vardata()
ref_ice_mask = np.load(gdir.get_filepath('ref_ice_mask'))
#true_bed = salem.GeoTiff(gdir.get_filepath('dem')).get_vardata()
ref_inner_mask = compute_inner_mask(ref_ice_mask, full_array=True)
#plt.figure()
#plt.imshow(ref_ice_mask)
#plt.imshow(ref_inner_mask, cmap='RdBu')
#plt.show()
inversion_settings = load_pickle(
os.path.join(gdir.dir, experiment, 'inversion_settings.pkl'))
reg_parameters = inversion_settings['reg_parameters']
margin = np.logical_xor(ref_ice_mask, ref_inner_mask)
cfg.initialize()
output_dir = '/home/philipp/final/plots/surf_diff'
basedir = '/home/philipp/final/'
file_extension = 'pdf'
figsize = (4.5, 3)
for case in [test_cases.Giluwe, test_cases.Borden]:
filepaths = glob.glob(
os.path.join(basedir, '{:s}/*/data_logger.pkl'.format(case.name)))
filepaths = sorted(filepaths)
for path in filepaths:
idir, temp = os.path.split(path)
gdir, exp = os.path.split(idir)
dl = load_pickle(path)
exp_name = experiment_naming_engine.get_experiment_name2(exp)
if exp_name is not None and len(dl.step_indices) > 0:
print(exp_name + ' ' + case.name)
ice_mask = np.load(os.path.join(gdir, 'ref_ice_mask.npy'))
diff_optimized_surf = dl.surfs[-1] - dl.ref_surf
cbar_min = diff_optimized_surf.min()
cbar_max = diff_optimized_surf.max()
surf_noise = None
if os.path.exists(os.path.join(idir, 'dem_noise.npy')):
surf_noise = np.load(os.path.join(idir, 'dem_noise.npy'))
cbar_min = min(surf_noise.min(), cbar_min)
cbar_max = max(surf_noise.max(), cbar_max)
columns = [
'experiment', 'experimentgroup', 'experimentsubgroup', 'subgroupindex',
'optimizedbed', 'optimizedsurf', 'optimizedicethick', 'firstguess', 'beds',
'surfs', 'costs', 'cterms', 'optimizedbederror', 'optimizedsurferror',
'optimizedbedrmse', 'optimizedsurfrmse', 'optimizedbedbias',
'optimizedsurfbias', 'firstguessrmse', 'firstguessbias',
'firstguess_5_percentile', 'firstguess_25_percentile',
'firstguess_75_percentile', 'firstguess_95_percentile', 'surfacenoise',
'surfacenoisermse', 'surfacenoisebias', 'surfacenoise_5_percentile',
'surfacenoise_25_percentile', 'surfacenoise_75_percentile',
'surfacenoise_95_percentile', 'bedmeasurements', 'bedmeasurementsrmse',
'bedmeasurementsbias', 'iterations', 'R', 'dV', 'warning', 'dir_path'
]
df = pd.DataFrame(columns=columns)
for path in filepaths:
dl = load_pickle(path)
inv_subdir = os.path.split(path)[0]
inv_settings = load_pickle(
os.path.join(gdir.dir, inv_subdir, 'inversion_settings.pkl'))
experiment = inv_settings['inversion_subdir']
surface_noise = np.zeros(true_bed.shape)
if os.path.exists(os.path.join(gdir.dir, inv_subdir, 'dem_noise.npy')):
surface_noise = np.load(
os.path.join(gdir.dir, inv_subdir, 'dem_noise.npy'))
bed_measurements = np.ma.masked_all(true_bed.shape)
if os.path.exists(
os.path.join(gdir.dir, inv_subdir, 'bed_measurements.pkl')):
bed_measurements = np.load(
os.path.join(gdir.dir, inv_subdir, 'bed_measurements.pkl'))
warning_found = False
# first_guessed_bed_noise = np.load(os.path.join(gdir.dir, inv_subdir,
from combine2d.core.inversion import InversionDirectory
from combine2d.core.data_logging import load_pickle
from oggm import cfg
cfg.initialize()
basedir = '/data/philipp/thesis_test/Giluwe/perfect'
case = test_cases.Giluwe
gdir = NonRGIGlacierDirectory(case, basedir)
idir = InversionDirectory(gdir)
idir.inv_settings['inversion_counter'] = 1001
true_bed = salem.GeoTiff(gdir.get_filepath('dem')).get_vardata()
data_logger = load_pickle(idir.get_subdir_filepath('data_logger'))
plt.figure()
plt.imshow(data_logger.beds[-1] - true_bed)
subsampled_beds = data_logger.beds[-3:]
for i in [3, 4, 5, 6, 7, 8, 9, 10]:
yfrom = 6
yend = 8
y = i
xfrom = 0
xend = -1
plt.figure()
for bed in subsampled_beds:
def run_minimize(self):
"""
Here the actual minimization of the cost_function is done via
scipy.optimize.minimize.
First, data from the glacier directory is read and optionally a
DataLogger is created. The inversion settings used for this
particular inversion are saved in this subdirectory. Bounds for the
minimization are derived. Then the cost function is created and the
minimization of this cost function started. In the end, the result is
written to disk and optionally, further information is written to disk.
The whole process is dominated by the set inversion settings
Returns
-------
Result of minimization as scipy.optimize.minimize returns (res.x
gives flattened ndarray with bed, needs to be reshaped)
"""
# Copy first_guessed_bed to inversion directory
if self.inv_settings['log_minimize_steps']:
# TODO: really useful? -> respect reset argument in gdir?
self.clear_dir(self.get_current_basedir())
with rasterio.open(self.gdir.get_filepath('first_guessed_bed')) as src:
profile = src.profile
data = src.read(1)
with rasterio.open(self.get_subdir_filepath('first_guessed_bed'),
'w', **profile) as dst:
dst.write(data, 1)
if os.path.exists(self.gdir.get_filepath('first_guessed_bed_noise')):
shutil.copy(self.gdir.get_filepath('first_guessed_bed_noise'),
self.get_subdir_filepath('first_guessed_bed_noise'))
write_pickle(self.inv_settings,
self.get_subdir_filepath('inversion_settings'))
# Write out reg_parameters to check easier later on
self.write_string_to_file(self.get_subdir_filepath('reg_parameters'),
str(self.inv_settings['reg_parameters']))
self.inv_settings = load_pickle(
self.get_subdir_filepath('inversion_settings'))
self._read_all_data()
self.minimize_log = ''
self.data_logger = None
callback = None
if self.inv_settings['log_minimize_steps']:
dl = DataLogger(self)
self.data_logger = dl
callback = self.iteration_info_callback
# ----------------------------------------------------------------------
# Core: things are happening here:
bounds = self.get_bounds()
self.cost_func = create_cost_func(self.gdir, self.data_logger,
self.surf_noise,
self.bed_measurements)
res = minimize(fun=self.cost_func,
x0=self.first_guessed_bed.astype(np.float64).flatten(),
method=self.inv_settings['solver'], jac=True,
bounds=bounds,
options=self.inv_settings['minimize_options'],
callback=callback)
inverted_bed = res.x.reshape(self.first_guessed_bed.shape)
# ----------------------------------------------------------------------
profile['dtype'] = 'float64'
with rasterio.open(self.get_subdir_filepath('inverted_bed'),
'w', **profile) as dst:
dst.write(inverted_bed, 1)
if self.inv_settings['log_minimize_steps']:
self.write_string_to_file('log.txt', self.minimize_log)
dir = self.get_current_basedir()
dl.filter_data_from_optimization() # Optional, if we want to
data_logging.write_pickle(dl,
self.get_subdir_filepath('data_logger'))
dl.plot_all(dir)
plt.close('all')
return res
figsize = (4.5, 3)
cbar_min = -37.5
cbar_max = 37.5
cbar_min_max = max(abs(cbar_min), abs(cbar_max))
cbar_steps = 16
for case in [test_cases.Giluwe, test_cases.Borden]:
filepaths = glob.glob(
os.path.join(basedir, '{:s}/*/dem_noise.npy'.format(case.name)))
filepaths = sorted(filepaths)
for path in filepaths:
idir, temp = os.path.split(path)
gdir, exp = os.path.split(idir)
dl = load_pickle(os.path.join(idir, 'data_logger.pkl'))
exp_name = experiment_naming_engine.get_experiment_name2(exp)
dem_noise = np.load(path)
if exp_name is not None and len(dl.step_indices) > 0:
print(exp_name + ' ' + case.name)
print(dem_noise.min())
print(dem_noise.max())
ice_mask = np.load(os.path.join(gdir, 'ref_ice_mask.npy'))
norm = MidpointNormalize(midpoint=0.,
vmin=-cbar_min_max,
vmax=cbar_min_max)
# my_cmap = sns.diverging_palette(240, 15, l=40, s=99, as_cmap=True)
my_cmap = plt.get_cmap('PuOr_r')
plotpath = os.path.join(
