def fill_gaps(xres, tags, smooth=False, grid=None): res = xres.copy() if grid is None: grid = LDAS_io().grid lons, lats = np.meshgrid(grid.ease_lons, grid.ease_lats) ind_lat = res['row'].values.astype('int') ind_lon = res['col'].values.astype('int') imp = IterativeImputer(max_iter=10, random_state=0) for tag in np.atleast_1d(tags): img = np.full(lons.shape, np.nan) img[ind_lat, ind_lon] = res[tag] # find all non-zero values idx = np.where(~np.isnan(img)) vmin, vmax = np.percentile(img[idx], [2.5, 97.5]) img[img < vmin] = vmin img[img > vmax] = vmax # calculate fitting parameters imp.set_params(min_value=vmin, max_value=vmax) imp.fit(img) # Define an anchor pixel to infer fitted image dimensions tmp_img = img.copy() tmp_img[idx[0][20], idx[1][20]] = 1000000 # transform image with and without anchor pixel tmp_img_fitted = imp.transform(tmp_img) img_fitted = imp.transform(img) # # Get indices of fitted image idx_anchor = np.where(tmp_img_fitted == 1000000)[1][0] start = idx[1][20] - idx_anchor end = start + img_fitted.shape[1] # write output img[:, start:end] = img_fitted if smooth: img = gaussian_filter(img, sigma=0.6, truncate=1) res.loc[:, tag] = img[ind_lat, ind_lon] res.loc[:, tag] = res.loc[:, tag].replace(np.nan, res.loc[:, tag].median()) return res
def smooth_pert_corr(): res = pd.read_csv('/work/GLEAM/perturbation_correction_v2/result.csv', index_col=0) gpis_valid = get_valid_gpis(latmin=24., latmax=51., lonmin=-128., lonmax=-64.) ind_valid = np.unravel_index(gpis_valid, (720, 1440)) imp = IterativeImputer(max_iter=10, random_state=0) ind = np.unravel_index(res.index.values, (720, 1440)) for tag in ['a1', 'b1', 'c1','a2', 'b2', 'c2']: img = np.full((720, 1440), np.nan) img[ind] = res[tag] # find all non-zero values idx = np.where(~np.isnan(img)) vmin, vmax = np.percentile(img[idx], [2.5, 97.5]) img[img < vmin] = vmin img[img > vmax] = vmax # calculate fitting parameters imp.set_params(min_value=vmin, max_value=vmax) imp.fit(img) # Define an anchor pixel to infer fitted image dimensions tmp_img = img.copy() tmp_img[idx[0][100], idx[1][100]] = 1000000 # transform image with and without anchor pixel tmp_img_fitted = imp.transform(tmp_img) img_fitted = imp.transform(img) # # Get indices of fitted image idx_anchor = np.where(tmp_img_fitted == 1000000)[1][0] start = idx[1][100] - idx_anchor end = start + img_fitted.shape[1] # write output img[:, start:end] = img_fitted img = gaussian_filter(img, sigma=0.6, truncate=1) res.loc[:, tag + '_s'] = img[ind_valid] res.to_csv('/work/GLEAM/perturbation_correction_v2/result_smoothed.csv', float_format='%.8f')
def fill_error_gaps(): res = pd.read_csv('/work/GLEAM/errors/result.csv', index_col=0) gpis_valid = get_valid_gpis(latmin=24., latmax=51., lonmin=-128., lonmax=-64.) ind_valid = np.unravel_index(gpis_valid, (720,1440)) res_gapfilled = pd.DataFrame(index=gpis_valid) r_min = 0.0 ind = (res['R_GLEAM_ASCAT'] <= r_min) | \ (res['R_GLEAM_AMSR2'] <= r_min) | \ (res['R_ASCAT_AMSR2'] <= r_min) res.loc[ind, ['TC1_R2_GLEAM', 'TC1_R2_ASCAT', 'TC1_R2_AMSR2']] = np.nan res.loc[ind, ['TC1_RMSE_GLEAM', 'TC1_RMSE_ASCAT', 'TC1_RMSE_AMSR2']] = np.nan # ind = (res['p_GLEAM_ASCAT'] >= 0.05) | \ # (res['p_GLEAM_AMSR2'] >= 0.05) | \ # (res['p_ASCAT_AMSR2'] >= 0.05) # # res.loc[ind, ['TC1_R2_GLEAM', 'TC1_R2_ASCAT', 'TC1_R2_AMSR2']] = np.nan # res.loc[ind, ['TC1_RMSE_GLEAM', 'TC1_RMSE_ASCAT', 'TC1_RMSE_AMSR2']] = np.nan ind = (res['R_GLEAM_ASCAT'] <= r_min) | \ (res['R_GLEAM_SMAP'] <= r_min) | \ (res['R_ASCAT_SMAP'] <= r_min) res.loc[ind, ['TC2_R2_GLEAM', 'TC2_R2_ASCAT', 'TC2_R2_SMAP']] = np.nan res.loc[ind, ['TC2_RMSE_GLEAM', 'TC2_RMSE_ASCAT', 'TC2_RMSE_SMAP']] = np.nan # ind = (res['p_GLEAM_ASCAT'] >= 0.05) | \ # (res['p_GLEAM_SMAP'] >= 0.05) | \ # (res['p_ASCAT_SMAP'] >= 0.05) # # res.loc[ind, ['TC2_R2_GLEAM', 'TC2_R2_ASCAT', 'TC2_R2_SMAP']] = np.nan # res.loc[ind, ['TC2_RMSE_GLEAM', 'TC2_RMSE_ASCAT', 'TC2_RMSE_SMAP']] = np.nan # --------------------- # tags = ['TC1_R2_GLEAM',] tags = ['TC1_R2_GLEAM', 'TC1_R2_ASCAT', 'TC1_R2_AMSR2', 'TC2_R2_GLEAM', 'TC2_R2_ASCAT', 'TC2_R2_SMAP', 'TC1_RMSE_GLEAM', 'TC1_RMSE_ASCAT', 'TC1_RMSE_AMSR2', 'TC2_RMSE_GLEAM', 'TC2_RMSE_ASCAT', 'TC2_RMSE_SMAP'] imp = IterativeImputer(max_iter=10, random_state=0) ind = np.unravel_index(res.index.values, (720,1440)) for tag in tags: img = np.full((720,1440), np.nan) img[ind] = res[tag] # find all non-zero values idx = np.where(~np.isnan(img)) vmin, vmax = np.percentile(img[idx], [2.5, 97.5]) img[img<vmin] = vmin img[img>vmax] = vmax # calculate fitting parameters imp.set_params(min_value=vmin, max_value=vmax) imp.fit(img) # Define an anchor pixel to infer fitted image dimensions tmp_img = img.copy() tmp_img[idx[0][100],idx[1][100]] = 1000000 # transform image with and without anchor pixel tmp_img_fitted = imp.transform(tmp_img) img_fitted = imp.transform(img) # # Get indices of fitted image idx_anchor = np.where(tmp_img_fitted == 1000000)[1][0] start = idx[1][100] - idx_anchor end = start + img_fitted.shape[1] # write output img[:,start:end] = img_fitted img = gaussian_filter(img, sigma=0.7, truncate=1) res_gapfilled.loc[:, tag] = img[ind_valid] # np.save('/work/GLEAM/test', img) print(tag, 'finished.') res_gapfilled.to_csv('/work/GLEAM/errors/result_gapfilled_sig07.csv')