def fill_gaps(xres, tags, smooth=False, grid=None):
res = xres.copy()
if grid is None:
grid = LDAS_io().grid
lons, lats = np.meshgrid(grid.ease_lons, grid.ease_lats)
ind_lat = res['row'].values.astype('int')
ind_lon = res['col'].values.astype('int')
imp = IterativeImputer(max_iter=10, random_state=0)
for tag in np.atleast_1d(tags):
img = np.full(lons.shape, np.nan)
img[ind_lat, ind_lon] = res[tag]
# find all non-zero values
idx = np.where(~np.isnan(img))
vmin, vmax = np.percentile(img[idx], [2.5, 97.5])
img[img < vmin] = vmin
img[img > vmax] = vmax
# calculate fitting parameters
imp.set_params(min_value=vmin, max_value=vmax)
imp.fit(img)
# Define an anchor pixel to infer fitted image dimensions
tmp_img = img.copy()
tmp_img[idx[0][20], idx[1][20]] = 1000000
# transform image with and without anchor pixel
tmp_img_fitted = imp.transform(tmp_img)
img_fitted = imp.transform(img)
# # Get indices of fitted image
idx_anchor = np.where(tmp_img_fitted == 1000000)[1][0]
start = idx[1][20] - idx_anchor
end = start + img_fitted.shape[1]
# write output
img[:, start:end] = img_fitted
if smooth:
img = gaussian_filter(img, sigma=0.6, truncate=1)
res.loc[:, tag] = img[ind_lat, ind_lon]
res.loc[:, tag] = res.loc[:, tag].replace(np.nan,
res.loc[:, tag].median())
return res
def smooth_pert_corr():
res = pd.read_csv('/work/GLEAM/perturbation_correction_v2/result.csv', index_col=0)
gpis_valid = get_valid_gpis(latmin=24., latmax=51., lonmin=-128., lonmax=-64.)
ind_valid = np.unravel_index(gpis_valid, (720, 1440))
imp = IterativeImputer(max_iter=10, random_state=0)
ind = np.unravel_index(res.index.values, (720, 1440))
for tag in ['a1', 'b1', 'c1','a2', 'b2', 'c2']:
img = np.full((720, 1440), np.nan)
img[ind] = res[tag]
# find all non-zero values
idx = np.where(~np.isnan(img))
vmin, vmax = np.percentile(img[idx], [2.5, 97.5])
img[img < vmin] = vmin
img[img > vmax] = vmax
# calculate fitting parameters
imp.set_params(min_value=vmin, max_value=vmax)
imp.fit(img)
# Define an anchor pixel to infer fitted image dimensions
tmp_img = img.copy()
tmp_img[idx[0][100], idx[1][100]] = 1000000
# transform image with and without anchor pixel
tmp_img_fitted = imp.transform(tmp_img)
img_fitted = imp.transform(img)
# # Get indices of fitted image
idx_anchor = np.where(tmp_img_fitted == 1000000)[1][0]
start = idx[1][100] - idx_anchor
end = start + img_fitted.shape[1]
# write output
img[:, start:end] = img_fitted
img = gaussian_filter(img, sigma=0.6, truncate=1)
res.loc[:, tag + '_s'] = img[ind_valid]
res.to_csv('/work/GLEAM/perturbation_correction_v2/result_smoothed.csv', float_format='%.8f')
def fill_error_gaps():
res = pd.read_csv('/work/GLEAM/errors/result.csv', index_col=0)
gpis_valid = get_valid_gpis(latmin=24., latmax=51., lonmin=-128., lonmax=-64.)
ind_valid = np.unravel_index(gpis_valid, (720,1440))
res_gapfilled = pd.DataFrame(index=gpis_valid)
r_min = 0.0
ind = (res['R_GLEAM_ASCAT'] <= r_min) | \
(res['R_GLEAM_AMSR2'] <= r_min) | \
(res['R_ASCAT_AMSR2'] <= r_min)
res.loc[ind, ['TC1_R2_GLEAM', 'TC1_R2_ASCAT', 'TC1_R2_AMSR2']] = np.nan
res.loc[ind, ['TC1_RMSE_GLEAM', 'TC1_RMSE_ASCAT', 'TC1_RMSE_AMSR2']] = np.nan
# ind = (res['p_GLEAM_ASCAT'] >= 0.05) | \
# (res['p_GLEAM_AMSR2'] >= 0.05) | \
# (res['p_ASCAT_AMSR2'] >= 0.05)
#
# res.loc[ind, ['TC1_R2_GLEAM', 'TC1_R2_ASCAT', 'TC1_R2_AMSR2']] = np.nan
# res.loc[ind, ['TC1_RMSE_GLEAM', 'TC1_RMSE_ASCAT', 'TC1_RMSE_AMSR2']] = np.nan
ind = (res['R_GLEAM_ASCAT'] <= r_min) | \
(res['R_GLEAM_SMAP'] <= r_min) | \
(res['R_ASCAT_SMAP'] <= r_min)
res.loc[ind, ['TC2_R2_GLEAM', 'TC2_R2_ASCAT', 'TC2_R2_SMAP']] = np.nan
res.loc[ind, ['TC2_RMSE_GLEAM', 'TC2_RMSE_ASCAT', 'TC2_RMSE_SMAP']] = np.nan
# ind = (res['p_GLEAM_ASCAT'] >= 0.05) | \
# (res['p_GLEAM_SMAP'] >= 0.05) | \
# (res['p_ASCAT_SMAP'] >= 0.05)
#
# res.loc[ind, ['TC2_R2_GLEAM', 'TC2_R2_ASCAT', 'TC2_R2_SMAP']] = np.nan
# res.loc[ind, ['TC2_RMSE_GLEAM', 'TC2_RMSE_ASCAT', 'TC2_RMSE_SMAP']] = np.nan
# ---------------------
# tags = ['TC1_R2_GLEAM',]
tags = ['TC1_R2_GLEAM', 'TC1_R2_ASCAT', 'TC1_R2_AMSR2',
'TC2_R2_GLEAM', 'TC2_R2_ASCAT', 'TC2_R2_SMAP',
'TC1_RMSE_GLEAM', 'TC1_RMSE_ASCAT', 'TC1_RMSE_AMSR2',
'TC2_RMSE_GLEAM', 'TC2_RMSE_ASCAT', 'TC2_RMSE_SMAP']
imp = IterativeImputer(max_iter=10, random_state=0)
ind = np.unravel_index(res.index.values, (720,1440))
for tag in tags:
img = np.full((720,1440), np.nan)
img[ind] = res[tag]
# find all non-zero values
idx = np.where(~np.isnan(img))
vmin, vmax = np.percentile(img[idx], [2.5, 97.5])
img[img<vmin] = vmin
img[img>vmax] = vmax
# calculate fitting parameters
imp.set_params(min_value=vmin, max_value=vmax)
imp.fit(img)
# Define an anchor pixel to infer fitted image dimensions
tmp_img = img.copy()
tmp_img[idx[0][100],idx[1][100]] = 1000000
# transform image with and without anchor pixel
tmp_img_fitted = imp.transform(tmp_img)
img_fitted = imp.transform(img)
# # Get indices of fitted image
idx_anchor = np.where(tmp_img_fitted == 1000000)[1][0]
start = idx[1][100] - idx_anchor
end = start + img_fitted.shape[1]
# write output
img[:,start:end] = img_fitted
img = gaussian_filter(img, sigma=0.7, truncate=1)
res_gapfilled.loc[:, tag] = img[ind_valid]
# np.save('/work/GLEAM/test', img)
print(tag, 'finished.')
res_gapfilled.to_csv('/work/GLEAM/errors/result_gapfilled_sig07.csv')
