def iteration_info_callback(self, x0):
i = len(self.data_logger.costs) - 1
if i >= 0:
dl = self.data_logger
dl.step_indices.append(i)
b = self.true_bed
log_entry = '''
----------------------------------------------
Function Call: {func_call:d}
Iteration: {iteration:d}
Cost: {cost:g}
Bed RMSE: {bed_rmse:g}
Bed Bias: {bed_bias:g}
Bed Max_diff: {bed_maxdiff:g}
Surface RMSE: {surf_rmse:g}
Surface Max_diff: {surf_maxdiff:g}
'''
myargs = {
'func_call': i,
'iteration': len(dl.step_indices),
'cost': dl.costs[i],
'bed_rmse': RMSE(dl.beds[i], b),
'bed_bias': mean_BIAS(dl.beds[i], b, np.sum(self.ice_mask)),
'bed_maxdiff': np.max(np.abs(dl.beds[i] - b)),
'surf_rmse': RMSE(dl.surfs[i], self.ref_surf),
'surf_maxdiff': np.max(np.abs(dl.surfs[i] - self.ref_surf))
}
if self.surf_noise is not None:
log_entry += 'RMSE to perturbed surf: {:g}\n'.format(
RMSE(dl.surfs[i], self.ref_surf + self.surf_noise))
log_entry = log_entry.format(**myargs)
print(log_entry)
self.minimize_log += log_entry
def eval_identical_twin(idir):
header = 'case,run,icevolerr,rmsebed,rmsesurf,biasbed,biassurf,' \
'corr,rmsefg,biasfg,iterations,' \
'maxbeddiffglacier,maxbeddiffdomain,' \
'minbeddiffglacier,minbeddiffdomain,' \
'voloutsidebounds\n'
row = '{case:s},{run:s},{dV:.2f},{rmsebed:.1f},{rmsesurf:.1f},' \
'{biasbed:.1f},{biassurf:.1f},{corr:.3f},{rmsefg:.1f},' \
'{biasfg:.1f},{iterations:d},' \
'{maxbeddiffglacier:.1f},{maxbeddiffdomain:.1f},' \
'{minbeddiffglacier:.1f},{minbeddiffdomain:.1f},' \
'{voloutsidebounds:.9f}'
# TODO: max_bed_diff?
dl = load_pickle(idir.get_subdir_filepath('data_logger'))
vals = {}
vals['case'] = dl.case.name
vals['run'] = idir.inv_settings['inversion_subdir']
ref_it = np.load(idir.gdir.get_filepath('ref_ice_thickness'))
mod_it = (dl.surfs[-1] - dl.beds[-1])
ref_vol = ref_it.sum()
mod_vol = mod_it.sum()
vals['dV'] = (mod_vol - ref_vol) / ref_vol * 1e2
ref_ice_mask = np.load(idir.gdir.get_filepath('ref_ice_mask'))
vals['rmsebed'] = RMSE(dl.true_bed, dl.beds[-1], ref_ice_mask)
vals['rmsesurf'] = RMSE(dl.ref_surf, dl.surfs[-1], ref_ice_mask)
vals['rmsefg'] = RMSE(dl.true_bed, dl.first_guessed_bed, ref_ice_mask)
vals['biasbed'] = mean_BIAS(dl.beds[-1], dl.true_bed, ref_ice_mask)
vals['biassurf'] = mean_BIAS(dl.surfs[-1], dl.ref_surf, ref_ice_mask)
vals['biasfg'] = mean_BIAS(dl.first_guessed_bed, dl.true_bed, ref_ice_mask)
masked_true_it = np.ma.masked_array(dl.ref_surf - dl.true_bed,
mask=np.logical_not(ref_ice_mask))
masked_mod_it = np.ma.masked_array(dl.surfs[-1] - dl.beds[-1],
mask=np.logical_not(ref_ice_mask))
# TODO: ice thickness
vals['corr'] = np.ma.corrcoef(masked_true_it.flatten(),
masked_mod_it.flatten())[0, 1]
vals['iterations'] = len(dl.step_indices)
#vals['maxbeddiff'] = np.max((dl.beds[-1] - dl.true_bed) * ref_ice_mask)
vals['maxbeddiffglacier'] = np.max(
(dl.beds[-1] - dl.true_bed) * ref_ice_mask)
vals['maxbeddiffdomain'] = np.max(dl.beds[-1] - dl.true_bed)
#vals['minbeddiff'] = np.min((dl.beds[-1] - dl.true_bed) * ref_ice_mask)
vals['minbeddiffglacier'] = np.min(
(dl.beds[-1] - dl.true_bed) * ref_ice_mask)
vals['minbeddiffdomain'] = np.min(dl.beds[-1] - dl.true_bed)
vals['voloutsidebounds'] = np.sum(
(dl.surfs[-1] - dl.beds[-1]) * (1 - ref_ice_mask)) * 1e-9
data_row = row.format(**vals)
with open(idir.get_subdir_filepath('results'), 'w') as f:
f.writelines([header, data_row])
return [header, data_row]
def add_noise_to_first_guess(gdir, noise, cut_noise=True, min_ice_thick=5):
"""
Adds noise to the first guess. Saves this to the file with the first guess
and also saves the applied noise.
Parameters
----------
gdir: NonRGIGlacierDirectory
GlacierDirectory containing the first guess
noise: ndarray
noise to apply to the first guess
cut_noise: bool
whether or not the noise should be cut to not penetrate the surface
and require a minimum ice thickness if applied to first guess
min_ice_thick: float
minimum ice thickness, only applied if noise is cut
"""
fg_filepath = gdir.get_filepath('first_guessed_bed')
with rasterio.open(fg_filepath) as src:
first_guessed_bed = src.read(1)
profile = src.profile
if cut_noise:
ref_ice_mask = np.load(gdir.get_filepath('ref_ice_mask'))
desired_rmse = RMSE(noise, 0, ref_ice_mask)
ref_surf = salem.GeoTiff(gdir.get_filepath('ref_dem')).get_vardata()
penetrating = (first_guessed_bed + noise - min_ice_thick > ref_surf)
penetrating *= ref_ice_mask
noise = np.where(penetrating,
ref_surf - first_guessed_bed - min_ice_thick, noise)
# TODO: will result in problems -> iteratively?
rmse = RMSE(noise, 0, ref_ice_mask)
print('desired rmse: {:g}\\rmse after cutting: {:g}'.format(
desired_rmse, rmse))
# noise *= desired_rmse / rmse # rescale to desired RMSE
# if np.any(first_guessed_bed + noise > ref_surf):
# raise ValueError('First guess is found to penetrate ice surface; '
# 'Aborting')
first_guessed_bed = first_guessed_bed + noise
profile['dtype'] = 'float64'
with rasterio.open(fg_filepath, 'w', **profile) as dst:
dst.write(first_guessed_bed, 1)
np.save(gdir.get_filepath('first_guessed_bed_noise'), noise)
def create_perlin_noise(gdir,
desired_rmse=5.,
octaves=1,
base=1.,
freq=8.0,
glacier_only=True,
use=True):
"""
TODO: Documentation
Parameters
----------
gdir
desired_rmse
octaves
base
freq
glacier_only
Returns
-------
"""
ref_ice_mask = np.load(gdir.get_filepath('ref_ice_mask'))
max_y, max_x = ref_ice_mask.shape
noise = np.zeros((max_y, max_x))
for y in range(max_y):
for x in range(max_x):
# use pnoise here, but snoise would be a reasonable choice as well
noise[y, x] = pnoise2(x / freq,
y / freq,
octaves=octaves,
base=base)
if glacier_only:
noise = noise * ref_ice_mask
rmse = RMSE(noise, 0, ref_ice_mask)
else:
rmse = RMSE(noise, 0)
noise *= desired_rmse / rmse
return noise
def generate_bed_measurements(gdir, bed_measurements_mask, std=0):
true_bed = salem.GeoTiff(gdir.get_filepath('dem')).get_vardata()
noise = std * np.random.randn(*true_bed.shape)
bed_measurements = (true_bed + noise) * bed_measurements_mask
bed_measurements[np.logical_not(bed_measurements_mask)] = -np.inf
bed_measurements = np.ma.masked_array(
bed_measurements, mask=np.logical_not(bed_measurements_mask))
print('Actual RMSE of bed measurements: {:g}'.format(
RMSE(bed_measurements, true_bed)))
# TODO: apply std scaling after masking ...?
# TODO: investigate deviations of RMSE from numpys std
return bed_measurements
bed_measurements = np.ma.masked_all(true_bed.shape)
if os.path.exists(
os.path.join(gdir.dir, inv_subdir, 'bed_measurements.pkl')):
bed_measurements = np.load(
os.path.join(gdir.dir, inv_subdir, 'bed_measurements.pkl'))
warning_found = False
# first_guessed_bed_noise = np.load(os.path.join(gdir.dir, inv_subdir,
# 'first_guessed_bed_noise.npy'))
if os.path.exists(os.path.join(gdir.dir, inv_subdir, 'warning.txt')):
warning_found = True
if len(dl.step_indices) > 0:
final_bed = dl.beds[-1]
final_surf = dl.surfs[-1]
final_it = dl.surfs[-1] - dl.beds[-1]
bed_rmse = RMSE(dl.beds[-1], true_bed, ref_ice_mask)
bed_bias = mean_BIAS(dl.beds[-1], true_bed, ref_ice_mask)
bed_error = final_bed - true_bed
surf_rmse = RMSE(dl.surfs[-1], true_surf, ref_ice_mask)
surf_bias = mean_BIAS(dl.surfs[-1], true_surf, ref_ice_mask)
surf_error = final_surf - true_surf
dV = (((dl.surfs[-1] - dl.beds[-1]).sum()) -
(true_surf - true_bed).sum()) / (true_surf - true_bed).sum()
else:
final_bed = np.ma.masked_all(true_bed.shape)
final_surf = np.ma.masked_all(true_bed.shape)
final_it = np.ma.masked_all(true_bed.shape)
bed_error = np.ma.masked_all(true_bed.shape)
bed_rmse = np.nan
bed_bias = np.nan
surf_error = np.ma.masked_all(true_bed.shape)
if exp_name.startswith('bed measurements'):
ice_mask = np.load(os.path.join(gdir, 'ref_ice_mask.npy'))
bed_measurements = np.load(os.path.join(idir,
'bed_measurements.pkl'))
measurement_noise = bed_measurements - dl.true_bed
np.ma.set_fill_value(measurement_noise, np.nan)
cbar_min = measurement_noise.min()
cbar_max = measurement_noise.max()
cbar_min = np.floor(cbar_min / 5) * 5
cbar_max = np.ceil(cbar_max / 5) * 5
cbar_min_max = max(abs(cbar_min), abs(cbar_max))
norm = MidpointNormalize(midpoint=0., vmin=-cbar_min_max,
vmax=cbar_min_max,
mask=measurement_noise.mask)
#my_cmap = sns.diverging_palette(240, 15, l=40, s=99, as_cmap=True)
my_cmap = plt.get_cmap('PRGn')
measurement_RMSE = RMSE(bed_measurements, dl.true_bed)
text = 'measurement RMSE: {:.1f} m'.format(measurement_RMSE)
plotpath = os.path.join(output_dir,
'{:s}_{:s}_measurement_noise.{:s}'.format(
case.name,
exp_name,
file_extension))
plot_bed_measurement(measurement_noise, plotpath, case,
ice_mask=ice_mask,
cbar_min=cbar_min, cbar_max=cbar_max,
show_cbar=True, norm=norm, cmap=my_cmap,
text=text, n=int((2 * cbar_min_max) / 10))
#'Bed errors case {:s}\n ''experiment {:s}'.format(case,exp_name),
#exit()
max = np.max([pytorch_grad, fin_diff_grad])
ax.plot([min, max], [min, max], color='k', linestyle='dotted', label='$y = x$')
ax.legend()
ax.set_xlabel('Finite difference derivative (m$^{-1}$)')
ax.set_ylabel('PyTorch derivative (m$^{-1}$)')
# plt.axis('equal')
ax.yaxis.set_ticks_position('right')
ax.yaxis.set_label_position('right')
plt.tight_layout()
# plt.show()
fname = '{:s}_scatter_plot_grad.{:s}'.format(case.name, file_extension)
plt.savefig(os.path.join(output_dir, fname))
plt.close(fig)
print('Correlation coefficient:')
print(np.corrcoef(x_data, y_data))
print('Bias:')
print(mean_BIAS(x_data, y_data))
print('RMSE:')
print(RMSE(x_data, y_data))
print('Abs mean:')
print(np.abs(x_data).mean())
print(np.abs(y_data).mean())
y_data2 = 1 / poly[1] * y_data
print('Bias2:')
print(mean_BIAS(x_data, y_data2))
print('RMSE2:')
print(RMSE(x_data, y_data2))
print('end')