def plot_ensvar_ratio(dir_out):
dtype, hdr, length = template_error_Tb40()
f = plt.figure(figsize=(25, 12))
fontsize = 14
cb = [-10, 10]
cmap = cc.cm.bjy
modes = ['4K', 'abs', 'anom_lst', 'anom_lt', 'anom_st']
titles = [
'4K benchmark', 'Total signal', 'Anomalies', 'LF signal', 'HF signal'
]
labels = [
'(V-pol, Asc.)', '(V-pol, Dsc.)', '(H-pol, Asc.)', '(H-pol, Dsc.)'
]
ios = [
GEOSldas_io('ObsFcstAna',
exp=f'NLv4_M36_US_DA_SMAP_Pcorr_{mode}').timeseries
for mode in modes
]
io_ol = GEOSldas_io('ObsFcstAna', exp=f'NLv4_M36_US_OL_Pcorr').timeseries
grid = GEOSldas_io().grid
for i, (io_da, tit) in enumerate(zip(ios, titles)):
for spc, label in zip(range(4), labels):
tmp1 = io_da['obs_obsvar'][:, spc, :, :].values
tmp2 = io_ol['obs_fcstvar'][:, spc, :, :].values
avg = np.nanmean(tmp1 / tmp2, axis=0)
# ratio = io_da['obs_obsvar'] / io_ol['obs_fcstvar']
# ratio = io['obs_obsvar']
# avg = ratio.mean(dim='time', skipna=True)
plt.subplot(4, 5, spc * 5 + i + 1)
img = plot_latlon_img(10 * np.log10(avg),
io_da.lon.values,
io_da.lat.values,
fontsize=fontsize,
cbrange=cb,
cmap=cmap,
plot_cb=False)
if spc == 0:
plt.title(tit, fontsize=fontsize)
if i == 0:
plt.ylabel(label, fontsize=fontsize)
plot_centered_cbar(f, img, 5, fontsize=fontsize - 2)
fout = dir_out / 'ensvar_ratio.png'
f.savefig(fout, dpi=300, bbox_inches='tight')
plt.close()
def plot_ObsFcstAna_image(species=8):
io = GEOSldas_io('ObsFcstAna')
img = io.read_image(2011, 7, 10, 0, 0)
img = img[img['obs_species']==species]
tag = 'innov'
img[tag] = img['obs_obs']-img['obs_fcst']
img.index = img['obs_tilenum'].values
plot_ease_img(img, tag)
def plot_model_image():
io = GEOSldas_io('xhourly')
img = io.read_image(2011, 4, 20, 10, 30)
# tag = 'precipitation_total_surface_flux'
tag = 'snow_mass'
# cbrange = (0,0.0001)
# cbrange = (0,0.6)
cbrange = (0,100)
plot_ease_img(img, tag, cbrange=cbrange)
def plot_fcst_uncertainties():
io = GEOSldas_io('ObsFcstAna',exp='US_M36_SMOS40_noDA_cal_scaled')
lons = io.images['lon'].values
lats = io.images['lat'].values
figsize=(18,9)
f = plt.figure(num=None, figsize=figsize, dpi=90, facecolor='w', edgecolor='k')
ax = f.add_subplot(2,2,1)
obserr = io.images.sel(species=1)['obs_fcstvar'].mean('time').values
plot_xarr_img(obserr,lons,lats,cbrange=[0,50])
ax.set_title('Asc / H-pol', fontsize=16)
ax = f.add_subplot(2,2,2)
obserr = io.images.sel(species=2)['obs_fcstvar'].mean('time').values
plot_xarr_img(obserr,lons,lats,cbrange=[0,50])
ax.set_title('Dsc / H-pol', fontsize=16)
ax = f.add_subplot(2,2,3)
obserr = io.images.sel(species=3)['obs_fcstvar'].mean('time').values
plot_xarr_img(obserr,lons,lats,cbrange=[0,50])
ax.set_title('Asc / V-pol', fontsize=16)
ax = f.add_subplot(2,2,4)
obserr = io.images.sel(species=4)['obs_fcstvar'].mean('time').values
plot_xarr_img(obserr,lons,lats,cbrange=[0,50])
ax.set_title('Dsc / V-pol', fontsize=16)
plt.tight_layout()
plt.show()
def plot_perturbations():
root = Path('/Users/u0116961/Documents/work/MadKF/CLSM/SM_err_ratio/GEOSldas/')
pc = 'noPcorr'
io = GEOSldas_io('ObsFcstAna')
lut = pd.read_csv(Paths().lut, index_col=0)
ind = np.vectorize(io.grid.colrow2tilenum)(lut.ease2_col, lut.ease2_row, local=False)
for mode in ['abs', 'anom_lst', 'anom_lt', 'anom_st']:
# for mode in ['abs']:
fA = root / 'observation_perturbations' / f'{pc}' / f'{mode}' / 'SMOS_fit_Tb_A.bin'
fD = root / 'observation_perturbations' / f'{pc}' / f'{mode}' / 'SMOS_fit_Tb_D.bin'
dir_out = root / 'plots' / 'obs_pert' / f'{pc}'
if not dir_out.exists():
Path.mkdir(dir_out, parents=True)
dtype, hdr, length = template_error_Tb40()
imgA = io.read_fortran_binary(fA, dtype, hdr=hdr, length=length)
imgD = io.read_fortran_binary(fD, dtype, hdr=hdr, length=length)
imgA.index += 1
imgD.index += 1
cbrange = [0,15]
plt.figure(figsize=(19, 11))
plt.subplot(221)
plot_ease_img2(imgA.reindex(ind),'err_Tbh', cbrange=cbrange, title='H-pol (Asc.)', io=io)
plt.subplot(222)
plot_ease_img2(imgA.reindex(ind),'err_Tbv', cbrange=cbrange, title='V-pol (Asc.)', io=io)
plt.subplot(223)
plot_ease_img2(imgD.reindex(ind),'err_Tbh', cbrange=cbrange, title='H-pol (Dsc.)', io=io)
plt.subplot(224)
plot_ease_img2(imgD.reindex(ind),'err_Tbv', cbrange=cbrange, title='V-pol (Dsc.)', io=io)
plt.savefig(dir_out / f'{mode}.png', dpi=300, bbox_inches='tight')
plt.close()
def calc_ens_var(root):
resdir = root / 'ens_vars' / 'Pcorr'
if not resdir.exists():
Path.mkdir(resdir, parents=True)
exp_ol = 'NLv4_M36_US_OL_Pcorr'
# exp_da = 'NLv4_M36_US_OL_Pcorr'
# param = 'ObsFcstAnaEns'
param = 'ObsFcstAna'
io_ol = GEOSldas_io(param, exp_ol)
# io_da = GEOSldas_io(param, exp_da)
res = pd.DataFrame(index=io_ol.grid.tilecoord.index.values,
columns=['col', 'row'] + [f'obs_var_spc{spc}' for spc in [1,2,3,4]] \
+ [f'fcst_var_spc{spc}' for spc in [1,2,3,4]] \
+ [f'ana_var_spc{spc}' for spc in [1,2,3,4]])
ids = io_ol.grid.tilecoord['tile_id'].values
res.loc[:, 'col'], res.loc[:, 'row'] = np.vectorize(
io_ol.grid.tileid2colrow)(ids, local_cs=False)
for cnt, (idx, val) in enumerate(io_ol.grid.tilecoord.iterrows()):
print('%i / %i' % (cnt, len(res)))
col, row = io_ol.grid.tileid2colrow(val.tile_id)
for spc in [1, 2, 3, 4]:
# ts_fcst = io_ol.read_ts('obs_fcst', col, row, species=spc, lonlat=False).dropna()
# ts_obs = io_da.read_ts('obs_obs', col, row, species=spc, lonlat=False).dropna()
# ts_ana = io_da.read_ts('obs_ana', col, row, species=spc, lonlat=False).dropna()
# if len(ts_ana) == 0:
# continue
# ts_fcst = ts_fcst.reindex(ts_ana.index)
# ts_obs = ts_obs.reindex(ts_ana.index)
# res.loc[idx,f'obs_var_spc{spc}'] = ts_obs.var(axis='columns').mean()
# res.loc[idx,f'fcst_var_spc{spc}'] = ts_fcst.var(axis='columns').mean()
# res.loc[idx,f'ana_var_spc{spc}'] = ts_ana.var(axis='columns').mean()
# res.loc[idx,f'obs_var_spc{spc}'] = np.nanmean(io_da.timeseries['obs_obsvar'][:,spc-1,row,col].values)
# res.loc[idx,f'fcst_var_spc{spc}'] = np.nanmean(io_ol.timeseries['obs_fcstvar'][:,spc-1,row,col].values)
# res.loc[idx,f'ana_var_spc{spc}'] = np.nanmean(io_da.timeseries['obs_anavar'][:,spc-1,row,col].values)
res.loc[idx, f'obs_var_spc{spc}'] = np.nanmean(
io_ol.timeseries['obs_obsvar'][:, spc - 1, row, col].values)
res.loc[idx, f'fcst_var_spc{spc}'] = np.nanmean(
io_ol.timeseries['obs_fcstvar'][:, spc - 1, row, col].values)
res.loc[idx, f'ana_var_spc{spc}'] = np.nanmean(
io_ol.timeseries['obs_fcstvar'][:, spc - 1, row, col].values)
fname = resdir / 'ens_var.csv'
res.to_csv(fname, float_format='%0.8f')
def plot_ease_img(data,tag,
llcrnrlat=24,
urcrnrlat=51,
llcrnrlon=-128,
urcrnrlon=-64,
figsize=(20,10),
cbrange=(-20,20),
cmap='jet',
plot_cb=True,
title='',
fontsize=20):
io = GEOSldas_io()
tc = io.grid.tilecoord
lons,lats = np.meshgrid(io.grid.ease_lons, io.grid.ease_lats)
img = np.empty(lons.shape, dtype='float32')
img.fill(None)
ind_lat = tc.loc[data.index.values,'j_indg']
ind_lon = tc.loc[data.index.values,'i_indg']
img[ind_lat,ind_lon] = data[tag]
img_masked = np.ma.masked_invalid(img)
# plt.figure(num=None, figsize=figsize, dpi=90, facecolor='w', edgecolor='k')
m = Basemap(projection='mill',
llcrnrlat=llcrnrlat,
urcrnrlat=urcrnrlat,
llcrnrlon=llcrnrlon,
urcrnrlon=urcrnrlon,
resolution='c')
m.drawcoastlines()
m.drawcountries()
m.drawstates()
im = m.pcolormesh(lons, lats, img_masked, cmap=cmap, latlon=True)
im.set_clim(vmin=cbrange[0], vmax=cbrange[1])
if plot_cb is True:
cb = m.colorbar(im, "bottom", size="7%", pad=0.05)
for t in cb.ax.get_xticklabels():
t.set_fontsize(fontsize)
for t in cb.ax.get_yticklabels():
t.set_fontsize(fontsize)
plt.title(title,fontsize=fontsize)
def plot_freq_components(dir_out):
ds = GEOSldas_io('ObsFcstAna',
exp='NLv4_M36_US_DA_SMAP_Pcorr_LTST').timeseries
ts = ds['obs_fcst'][:, 0, 50, 120].to_pandas().dropna()
clim = calc_climatology(ts)
anom_lt = calc_anom(ts, mode='longterm')
anom_st = calc_anom(ts, mode='shortterm')
clim = calc_anom(ts, return_clim=True)
anom_lst = anom_lt + anom_st
anom_lst.name = 'Anomalies (HF + LF)'
f, axes = plt.subplots(figsize=(18, 6), nrows=3, ncols=1, sharex=True)
fontsize = 11
df = pd.concat((ts, clim), axis=1)
df.columns = ['$T_b$ signal', 'Climatology']
df.plot(ax=axes[0],
color=['darkorange', 'blue'],
linewidth=1.7,
fontsize=fontsize).legend(loc='upper right', fontsize=fontsize)
df = pd.DataFrame(anom_lst)
df.plot(ax=axes[1],
color='crimson',
ylim=[-18, 18],
linewidth=1.7,
fontsize=fontsize).legend(loc='upper right', fontsize=fontsize)
axes[1].axhline(color='black', linestyle='--', linewidth=0.8)
df = pd.concat((anom_st, anom_lt), axis=1)
df.columns = ['HF signal', 'LF signal']
df.plot(ax=axes[2],
color=['goldenrod', 'teal'],
ylim=[-15, 15],
linewidth=1.7,
fontsize=fontsize).legend(loc='upper right', fontsize=fontsize)
axes[2].axhline(color='black', linestyle='--', linewidth=0.8)
plt.xlabel('')
plt.minorticks_off()
plt.xlim('2015-04', '2021,04')
plt.xticks(fontsize=fontsize)
f.savefig(dir_out / f'frequency_components.png',
dpi=300,
bbox_inches='tight')
plt.close()
def plot_innov(spc=8, row=35, col=65):
ts_scl = GEOSldas_io('ObsFcstAna',exp='NLv4_M36_US_OL_Pcorr_scl_SMAP').timeseries
ts_usc = GEOSldas_io('ObsFcstAna',exp='NLv4_M36_US_OL_Pcorr_SMAP').timeseries
plt.figure(figsize=(18,11))
ax1 = plt.subplot(311)
df = pd.DataFrame(index=ts_scl.time)
df['obs'] = ts_scl['obs_obs'].isel(species=spc-1, lat=row, lon=col).values
df['fcst'] = ts_scl['obs_fcst'].isel(species=spc-1, lat=row, lon=col).values
df.dropna().plot(ax=ax1)
ax2 = plt.subplot(312)
df = pd.DataFrame(index=ts_usc.time)
df['obs'] = ts_usc['obs_obs'].isel(species=spc-1, lat=row, lon=col).values
df['fcst'] = ts_usc['obs_fcst'].isel(species=spc-1, lat=row, lon=col).values
df.dropna().plot(ax=ax2)
ax3 = plt.subplot(313)
df = pd.DataFrame(index=ts_usc.time)
# df['innov'] = ts_scl['obs_obs'].isel(lat=row, lon=col).mean(dim='species').values - ts_scl['obs_fcst'].isel(lat=row, lon=col).mean(dim='species').values
df['innov'] = ts_scl['obs_obs'].isel(species=spc-1, lat=row, lon=col).values - ts_scl['obs_fcst'].isel(species=spc-1, lat=row, lon=col).values
# df['innov_unscl'] = ts_scl['obs_obs'].isel(species=spc-1, lat=row, lon=col).values - ts_usc['obs_fcst'].isel(species=spc-1, lat=row, lon=col).values
df.dropna().plot(ax=ax3)
plt.axhline(color='black', linestyle='--')
print(df.mean())
# print(len(ts_scl['obs_obs'][spc,row,col].dropna('time')))
# print(len(ts_scl['obs_fcst'][spc,row,col].dropna('time')))
# print(len(ts_usc['obs_obs'][spc,row,col].dropna('time')))
# print(len(ts_usc['obs_fcst'][spc,row,col].dropna('time')))
plt.tight_layout()
plt.show()
ts_scl.close()
ts_usc.close()
def generate_station_list(self):
tmplist = self.io.metadata[[('network', 'val'), ('station', 'val'),
('latitude', 'val'), ('longitude', 'val')]]
tmplist.columns = tmplist.columns.droplevel('key')
tmplist.columns.name = None
tmplist = tmplist.iloc[np.unique(tmplist.network + tmplist.station,
return_index=True)[1]]
tmplist.index = np.arange(len(tmplist))
grid = GEOSldas_io().grid
vfindcolrow = np.vectorize(grid.lonlat2colrow)
col, row = vfindcolrow(tmplist.longitude.values,
tmplist.latitude.values)
tmplist['ease_col'] = col
tmplist['ease_row'] = row
tmplist['ease_col'] -= self.col_offs
tmplist['ease_row'] -= self.row_offs
tmplist.to_csv(self.list_file)
self.list = tmplist
def plot_grid_coord_indices():
io = GEOSldas_io('ObsFcstAna', exp='US_M36_SMOS40_noDA_cal_scaled')
lats = io.images.lat.values
lons = io.images.lon.values
plt.figure(figsize=(10, 5))
llcrnrlat = 24
urcrnrlat = 51
llcrnrlon = -128
urcrnrlon = -64
m = Basemap(projection='mill', llcrnrlat=llcrnrlat, urcrnrlat=urcrnrlat, llcrnrlon=llcrnrlon,
urcrnrlon=urcrnrlon,
resolution='c')
m.drawcoastlines(linewidth=0.5)
m.drawcountries(linewidth=0.5)
m.drawstates(linewidth=0.1)
lats = lats[np.arange(0, len(lats), 15)]
lons = lons[np.arange(0, len(lons), 15)]
m.drawparallels(lats, labels=[False, False, False, False], linestyle='--', linewidth=1, color='red')
m.drawmeridians(lons, labels=[False, False, False, False], linestyle='--', linewidth=1, color='red')
x = np.zeros(len(lons))
for i, lon in enumerate(lons):
x[i], tmp = m(lon, lats[0])
y = np.zeros(len(lats))
for i, lat in enumerate(lats):
tmp, y[i] = m(lons[-1], lat)
plt.xticks(x[0:-1], np.arange(0, len(lons) - 1) * 15)
plt.yticks(y[1::], np.arange(1, len(lats)) * 15)
plt.show()
def plot_statistics(res_path, dir_out):
res = pd.read_csv(res_path / 'ascat_eval.csv', index_col=0)
res_ismn = pd.read_csv(res_path / 'insitu_TCA.csv', index_col=0)
networks = ['SCAN', 'USCRN']
res_ismn = res_ismn.loc[res_ismn.network.isin(networks), :]
res_tc = pd.read_csv(
'/Users/u0116961/Documents/work/MadKF/CLSM/SM_err_ratio/GEOSldas/sm_validation/Pcorr/result.csv',
index_col=0)
tg = GEOSldas_io().grid.tilegrids
ind_ismn = []
for col, row in zip(res_ismn.ease_col.values + tg.loc['domain', 'i_offg'],
res_ismn.ease_row.values + tg.loc['domain', 'j_offg']):
try:
ind_ismn += [
res[(res.row == row) & (res.col == col)].index.values[0]
]
except:
continue
runs = [
f'Pcorr_{err}' for err in ['anom_lst', 'anom_lt_ScDY', 'anom_st_ScYH']
]
modes = ['anom_lst', 'anom_lt', 'anom_st']
titles = [f'Anomaly skill', f'LF skill', f'HF skill']
f = plt.figure(figsize=(24, 15))
output = 'ascat' # 'ascat' or 'ascat_ismn' or 'ismn'
met = 'r_corr' # 'R_model_insitu'# or 'R2_model' / 'r' or 'r_corr'
var = 'sm_surface'
if 'ubRMSD' in met:
lim = [-0.015, 0.015]
xloc = -0.0135
bins = 15
else:
if 'ascat' not in output:
lim = [-0.2, 0.3]
xloc = -0.18
bins = 15
else:
lim = [-0.25, 0.25]
xloc = -0.22
bins = 20
f.suptitle(f'{output} / {met} / {var}')
for i, (run, mode, title) in enumerate(zip(runs, modes, titles)):
if 'ascat' in output:
col_ol = f'ana_{met}_Pcorr_OL_{mode}'
col_4k = f'ana_{met}_Pcorr_4K_{mode}'
col_lst = f'ana_{met}_Pcorr_anom_lst_{mode}'
col_da2 = f'ana_{met}_Pcorr_LTST_{mode}'
col_da1 = f'ana_{met}_{run}_{mode}'
if 'ismn' in output:
res = res.reindex(ind_ismn)
else:
r_asc_smap = res_tc[f'r_grid_{mode}_p_ASCAT_SMAP']
r_asc_clsm = res_tc[f'r_grid_{mode}_p_ASCAT_CLSM']
r_smap_clsm = res_tc[f'r_grid_{mode}_p_SMAP_CLSM']
thres = 0.2
ind_valid = res_tc[(r_asc_smap > thres) & (r_asc_smap > thres)
& (r_asc_smap > thres)].index
res = res.reindex(ind_valid)
else:
res = res_ismn
col_ol = f'{met}_Pcorr_OL_{mode}_{var}'
col_4k = f'{met}_Pcorr_4K_{mode}_{var}'
col_lst = f'{met}_Pcorr_anom_lst_{mode}_{var}'
col_da2 = f'{met}_Pcorr_LTST_{mode}_{var}'
col_da1 = f'{met}_{run}_{mode}_{var}'
if 'R2' in met:
res[col_ol] **= 0.5
res[col_4k] **= 0.5
res[col_lst] **= 0.5
res[col_da2] **= 0.5
res[col_da1] **= 0.5
if not ((output == 'ismn') and ('R2' not in met)):
print('filtered')
res[col_ol][res[col_ol] <= 0] = np.nan
res[col_4k][res[col_4k] <= 0] = np.nan
res[col_da1][res[col_da1] <= 0] = np.nan
res[col_da2][res[col_da2] <= 0] = np.nan
res[col_lst][res[col_lst] <= 0] = np.nan
res[col_ol][res[col_ol] >= 1] = np.nan
res[col_4k][res[col_4k] >= 1] = np.nan
res[col_da1][res[col_da1] >= 1] = np.nan
res[col_da2][res[col_da2] >= 1] = np.nan
res[col_lst][res[col_lst] >= 1] = np.nan
# res['single1'] = res[col_da1] - res[col_ol]
# res['single2'] = res[col_da1] - res[col_4k]
# res['joint1'] = res[col_da2] - res[col_ol]
# res['joint2'] = res[col_da2] - res[col_4k]
res['single'] = res[col_da1] - res[col_ol]
res['joint'] = res[col_da2] - res[col_ol]
res['anom'] = res[col_lst] - res[col_ol]
res['4k'] = res[col_4k] - res[col_ol]
ax = plt.subplot(3, 3, i + 1)
# plt.scatter(x=res[col_ol].reindex(ind_valid),y=res[col_da1].reindex(ind_valid))
p1 = res['4k'].hist(bins=bins, grid=False, ax=ax, range=lim, alpha=0.8)
p2 = res['anom'].hist(bins=bins,
grid=False,
ax=ax,
range=lim,
alpha=0.6)
plt.title(title)
plt.yticks(color='w', fontsize=5)
plt.xticks(color='w', fontsize=1)
if i == 0:
plt.ylabel('Anomaly assimilation')
if i == 2:
# plt.legend(labels=['R$_{TC}$ - R$_{4K}$', 'R$_{TC}$ - R$_{OL}$'], loc='upper right', fontsize=14)
plt.legend(labels=['R$_{4K}$ - R$_{OL}$', 'R$_{TC}$ - R$_{OL}$'],
loc='upper right',
fontsize=14)
plt.axvline(color='black', linestyle='--', linewidth=1)
plt.xlim(lim)
ylim = ax.get_ylim()
yloc1 = ylim[1] - (ylim[1] - ylim[0]) / 10
yloc2 = ylim[1] - 1.6 * (ylim[1] - ylim[0]) / 10
plt.text(xloc,
yloc1,
'mean = %.2f' % res['4k'].mean(),
color='#1f77b4')
plt.text(xloc,
yloc2,
'mean = %.2f' % res['anom'].mean(),
color='#ff7f0e')
ax = plt.subplot(3, 3, i + 4)
res['4k'].hist(bins=bins, grid=False, ax=ax, range=lim, alpha=0.8)
res['single'].hist(bins=bins, grid=False, ax=ax, range=lim, alpha=0.6)
plt.yticks(color='w', fontsize=5)
plt.xticks(color='w', fontsize=1)
if i == 0:
plt.ylabel('Individual assimilation')
plt.xlim(lim)
plt.axvline(color='black', linestyle='--', linewidth=1)
ylim = ax.get_ylim()
yloc1 = ylim[1] - (ylim[1] - ylim[0]) / 10
yloc2 = ylim[1] - 1.6 * (ylim[1] - ylim[0]) / 10
plt.text(xloc,
yloc1,
'mean = %.2f' % res['4k'].mean(),
color='#1f77b4')
plt.text(xloc,
yloc2,
'mean = %.2f' % res['single'].mean(),
color='#ff7f0e')
ax = plt.subplot(3, 3, i + 7)
res['4k'].hist(bins=bins, grid=False, ax=ax, range=lim, alpha=0.8)
res['joint'].hist(bins=bins, grid=False, ax=ax, range=lim, alpha=0.6)
plt.yticks(color='w', fontsize=5)
if i == 0:
plt.ylabel('joint assimilation')
plt.xlim(lim)
plt.axvline(color='black', linestyle='--', linewidth=1)
ylim = ax.get_ylim()
yloc1 = ylim[1] - (ylim[1] - ylim[0]) / 10
yloc2 = ylim[1] - 1.6 * (ylim[1] - ylim[0]) / 10
plt.text(xloc,
yloc1,
'mean = %.2f' % res['4k'].mean(),
color='#1f77b4')
plt.text(xloc,
yloc2,
'mean = %.2f' % res['joint'].mean(),
color='#ff7f0e')
if i == 0:
plt.title(title)
f.savefig(dir_out / f'stats_{output}_{met}_{var}.png',
dpi=300,
bbox_inches='tight')
plt.close()
def run_ascat_eval_part(part, parts, ref='ascat'):
import numpy as np
import pandas as pd
from pathlib import Path
from scipy.stats import pearsonr
from pyldas.interface import GEOSldas_io
from myprojects.readers.ascat import HSAF_io
from myprojects.timeseries import calc_anom
from validation_good_practice.ancillary.paths import Paths
res_path = Path(
'~/Documents/work/MadKF/CLSM/SM_err_ratio/GEOSldas/validation_all'
).expanduser()
if not res_path.exists():
Path.mkdir(res_path, parents=True)
result_file = res_path / ('ascat_eval_part%i.csv' % part)
tc_res_pc = pd.read_csv(
'/Users/u0116961/Documents/work/MadKF/CLSM/SM_err_ratio/GEOSldas/sm_validation/Pcorr/result.csv',
index_col=0)
tc_res_nopc = pd.read_csv(
'/Users/u0116961/Documents/work/MadKF/CLSM/SM_err_ratio/GEOSldas/sm_validation/noPcorr/result.csv',
index_col=0)
lut = pd.read_csv(Paths().lut, index_col=0)
# Split grid cell list for parallelization
subs = (np.arange(parts + 1) * len(lut) / parts).astype('int')
subs[-1] = len(lut)
start = subs[part - 1]
end = subs[part]
# Look-up table that contains the grid cells to iterate over
lut = lut.iloc[start:end, :]
root = Path('/Users/u0116961/data_sets/GEOSldas_runs')
runs = [run.name for run in root.glob('*_DA_SMAP_*')]
names = [run[20::] for run in runs]
runs += ['NLv4_M36_US_OL_Pcorr', 'NLv4_M36_US_OL_noPcorr']
names += ['Pcorr_OL', 'noPcorr_OL']
# names = ['OL_Pcorr', 'OL_noPcorr'] + \
# [f'DA_{pc}_{err}' for pc in ['Pcorr','noPcorr'] for err in ['4K','abs','anom_lt','anom_lst','anom_st']]
# runs = ['NLv4_M36_US_OL_Pcorr', 'NLv4_M36_US_OL_noPcorr' ] + \
# [f'NLv4_M36_US_DA_SMAP_{pc}_{err}' for pc in ['Pcorr','noPcorr'] for err in ['4K','abs','anom_lt','anom_lst','anom_st']]
# names = ['OL_Pcorr', 'DA_Pcorr_LTST'] + \
# [f'DA_{pc}_{err}{mode}' for pc in ['Pcorr'] for err in ['4K','anom_lt', 'anom_lt_ScYH', 'anom_lst','anom_st'] for mode in ['', '_ScDY', '_ScYH']]
#
# runs = ['NLv4_M36_US_OL_Pcorr', 'NLv4_M36_US_DA_Pcorr_LTST'] + \
# [f'NLv4_M36_US_DA_SMAP_{pc}_{err}{mode}' for pc in ['Pcorr'] for err in ['4K','abs','anom_lt','anom_lst','anom_st'] for mode in ['', '_ScDY', '_ScYH']]
dss = [
GEOSldas_io('tavg3_1d_lnr_Nt', run).timeseries
if 'DA' in run else GEOSldas_io('SMAP_L4_SM_gph', run).timeseries
for run in runs
]
grid = GEOSldas_io('ObsFcstAna', runs[0]).grid
ds_full = GEOSldas_io('SMAP_L4_SM_gph', 'NLv4_M36_US_OL_Pcorr').timeseries
ds_full = ds_full.assign_coords(
{'time': ds_full['time'].values + pd.to_timedelta('2 hours')})
ds_obs_smap = GEOSldas_io(
'ObsFcstAna', 'NLv4_M36_US_DA_SMAP_Pcorr_4K').timeseries['obs_obs']
modes = ['abs', 'anom_lt', 'anom_st', 'anom_lst']
ascat = HSAF_io()
for cnt, (gpi, data) in enumerate(lut.iterrows()):
print('%i / %i, gpi: %i' % (cnt, len(lut), gpi))
col = int(data.ease2_col - grid.tilegrids.loc['domain', 'i_offg'])
row = int(data.ease2_row - grid.tilegrids.loc['domain', 'j_offg'])
res = pd.DataFrame(index=(gpi, ))
res['col'] = int(data.ease2_col)
res['row'] = int(data.ease2_row)
res['lcol'] = col
res['lrow'] = row
try:
ts_ascat = ascat.read(
data['ascat_gpi']).resample('1d').mean().dropna()
ts_ascat = ts_ascat[~ts_ascat.index.duplicated(keep='first')]
ts_ascat.name = 'ASCAT'
except:
continue
try:
t_df_smap = ds_obs_smap.sel(species=[1, 2]).isel(
lat=row, lon=col).to_pandas()
t_ana = t_df_smap[~np.isnan(t_df_smap[1])
| ~np.isnan(t_df_smap[2])].index
t_ana = pd.Series(1,
index=t_ana).resample('1d').mean().dropna().index
except:
t_ana = pd.DatetimeIndex([])
var = 'sm_surface'
for mode in modes:
if mode == 'anom_lst':
ts_ref = calc_anom(ts_ascat.copy(),
mode='climatological').dropna()
elif mode == 'anom_st':
ts_ref = calc_anom(ts_ascat.copy(), mode='shortterm').dropna()
elif mode == 'anom_lt':
ts_ref = calc_anom(ts_ascat.copy(), mode='longterm').dropna()
else:
ts_ref = ts_ascat.dropna()
for run, ts_model in zip(names, dss):
try:
if 'noPcorr' in run:
r_asc = np.sqrt(tc_res_nopc.loc[
gpi, f'r2_grid_{mode}_m_ASCAT_tc_ASCAT_SMAP_CLSM'])
r_mod = np.sqrt(tc_res_nopc.loc[
gpi, f'r2_grid_{mode}_m_CLSM_tc_ASCAT_SMAP_CLSM'])
else:
r_asc = np.sqrt(tc_res_pc.loc[
gpi, f'r2_grid_{mode}_m_ASCAT_tc_ASCAT_SMAP_CLSM'])
r_mod = np.sqrt(tc_res_pc.loc[
gpi, f'r2_grid_{mode}_m_CLSM_tc_ASCAT_SMAP_CLSM'])
except:
r_asc = np.nan
r_mod = np.nan
ind_valid = ds_full.time.values[
(ds_full['snow_depth'][:, row, col].values == 0) &
(ds_full['soil_temp_layer1'][:, row, col].values > 277.15)]
ts_mod = ts_model[var][:, row, col].to_series()
ts_mod.index += pd.to_timedelta('2 hours')
ts_mod = ts_mod.reindex(ind_valid)
if mode == 'anom_lst':
ts_mod = calc_anom(ts_mod.copy(),
mode='climatological').dropna()
elif mode == 'anom_st':
ts_mod = calc_anom(ts_mod.copy(),
mode='shortterm').dropna()
elif mode == 'anom_lt':
ts_mod = calc_anom(ts_mod.copy(), mode='longterm').dropna()
else:
ts_mod = ts_mod.dropna()
ts_mod = ts_mod.resample('1d').mean()
if 'OL_' in run:
res[f'r_tca_{run}_{mode}'] = r_mod
tmp = pd.DataFrame({1: ts_ref, 2: ts_mod}).dropna()
res[f'len_{run}_{mode}'] = len(tmp)
r, p = pearsonr(tmp[1], tmp[2]) if len(tmp) > 10 else (np.nan,
np.nan)
res[f'r_{run}_{mode}'] = r
res[f'p_{run}_{mode}'] = p
res[f'r_corr_{run}_{mode}'] = min(r / r_asc, 1)
tmp = pd.DataFrame({
1: ts_ref,
2: ts_mod
}).reindex(t_ana).dropna()
res[f'ana_len_{run}_{mode}'] = len(tmp)
r, p = pearsonr(tmp[1], tmp[2]) if len(tmp) > 10 else (np.nan,
np.nan)
res[f'ana_r_{run}_{mode}'] = r
res[f'ana_p_{run}_{mode}'] = p
res[f'ana_r_corr_{run}_{mode}'] = min(r / r_asc, 1)
if not result_file.exists():
res.to_csv(result_file, float_format='%0.3f')
else:
res.to_csv(result_file,
float_format='%0.3f',
mode='a',
header=False)
def run(args, scale_target='SMAP', mode='longterm', use_pc=False):
'''
:param args: summarizes the following three for multiprocessing purposes:
sensor: 'SMOS' or 'SMAP' or 'SMOSSMAP'
date_from: 'yyyy-mm-dd'
date_to: 'yyyy-mm-dd'
:param scale_target: 'SMOS' or 'SMAP'
:param mode: 'longterm' or "shortterm'
:param use_pc: If true, the first principal component of SMOS/SMAP Tb will be used
'''
sensor, date_from, date_to, pc = args
exp_smap = f'NLv4_M36_US_OL_{pc}'
exp_smos = f'NLv4_M36_US_OL_{pc}_SMOS'
if mode == 'shortterm':
ext = '_yearly'
elif mode == 'longterm':
ext = '_daily'
else:
ext = ''
froot = Path(f'~/data_sets/GEOSldas_runs/_scaling_files_{pc}{ext}').expanduser()
if not froot.exists():
Path.mkdir(froot, parents=True)
ios = []
if 'SMAP' in sensor:
ios += [GEOSldas_io('ObsFcstAna', exp=exp_smap)]
if 'SMOS' in sensor:
ios += [GEOSldas_io('ObsFcstAna', exp=exp_smos)]
if not date_from:
date_from = pd.to_datetime(np.min([io.timeseries['time'].values[0] for io in ios]))
else:
date_from = pd.to_datetime(date_from)
if not date_to:
date_to = pd.to_datetime(np.max([io.timeseries['time'].values[-1] for io in ios]))
else:
date_to = pd.to_datetime(date_to)
pent_from = int(np.floor((date_from.dayofyear - 1) / 5.) + 1)
pent_to = int(np.floor((date_to.dayofyear - 1) / 5.) + 1)
fbase = f'Thvf_TbSM_001_src_{sensor}_trg_{scale_target}_{date_from.year}_p{pent_from:02}_{date_to.year}_p{pent_to:02}_W_9p_Nmin_20'
dtype, _, _ = template_scaling(sensor='SMAP')
tiles = ios[0].grid.tilecoord['tile_id'].values.astype('int32')
angles = np.array([40,], 'int')
pols = ['H','V']
orbits = [['A', 'D'],['D', 'A']] # To match SMOS and SMAP species!
template = pd.DataFrame(columns=dtype.names, index=tiles).astype('float32')
template['lon'] = ios[0].grid.tilecoord['com_lon'].values.astype('float32')
template['lat'] = ios[0].grid.tilecoord['com_lat'].values.astype('float32')
template['tile_id'] = tiles.astype('int32')
pentads = np.arange(73)+1
if mode == 'longterm':
years = np.arange(date_from.year, date_to.year + 1)
doys = np.arange(1,367)
data_obs = np.full([len(tiles), len(doys), len(years), len(pols), len(orbits[0])], -9999.)
data_mod = data_obs.copy()
# dummy = np.full([len(tiles), len(doys), len(years), len(angles), len(pols), len(orbits[0])], -9999)
# coords = {'tile_id': tiles,
# 'doy': doys,
# 'year': years,
# 'angle': angles,
# 'pol': pols,
# 'orbit': orbits[0]}
# darr = xr.DataArray(dummy, coords=coords, dims=['tile_id', 'doy', 'year', 'angle', 'pol', 'orbit'])
elif mode == 'shortterm':
years = np.arange(date_from.year, date_to.year+1)
data_obs = np.full([len(tiles), len(pentads), len(years), len(pols), len(orbits[0])], -9999.)
data_mod = data_obs.copy()
n_data = np.full([len(tiles), len(pentads), len(years), len(pols), len(orbits[0])], -9999)
# dummy = np.full([len(tiles), len(pentads), len(years), len(angles), len(pols), len(orbits[0])], -9999)
# coords = {'tile_id': tiles,
# 'pentad': pentads,
# 'year': years,
# 'angle': angles,
# 'pol': pols,
# 'orbit': orbits[0]}
# darr = xr.DataArray(dummy, coords=coords, dims=['tile_id', 'pentad', 'year', 'angle', 'pol', 'orbit'])
else:
# TODO: Currently doesn't work anymore because of modification for lt and st
dummy = np.full([len(tiles),len(pentads),len(angles),len(pols),len(orbits[0])],-9999)
coords = {'tile_id': tiles,
'pentad': pentads,
'angle': angles,
'pol': pols,
'orbit': orbits[0]}
darr = xr.DataArray(dummy, coords=coords, dims=['tile_id','pentad','angle','pol','orbit'])
# ----- calculate mean and reshuffle -----
for i_til, til in enumerate(tiles):
logging.info(f'{i_til} / {len(tiles)}')
for i_pol, pol in enumerate(pols):
# for i_ang, ang in enumerate(angles):
ang = angles[0]
for i_orb, (orb1, orb2) in enumerate(zip(orbits[0], orbits[1])):
col, row = ios[0].grid.tileid2colrow(til)
if sensor.upper() == 'SMOSSMAP':
spcs = [io.get_species(pol=pol, ang=ang, orbit=orb) for io, orb in zip(ios,[orb1, orb2])]
# orb = orb2 if scale_target == 'SMAP' else orb1 # POSSIBLY WRONG!!!!
orb = orb1 if scale_target == 'SMAP' else orb2
else:
spcs = [ios[0].get_species(pol=pol, ang=ang, orbit=orb1)]
if sensor.upper() == 'SMAP':
orb = orb1 if scale_target == 'SMAP' else orb2
else:
orb = orb2 if scale_target == 'SMAP' else orb1
if use_pc and (sensor == 'SMOSSMAP'):
dss = [io.timeseries['obs_obs'][:, spc-1, row, col].to_series() for io, spc in zip(ios,spcs)]
obs = PCA(*dss, window=1.5)['PC-1']
dss = [io.timeseries['obs_fcst'][:, spc-1, row, col].to_series() for io, spc in zip(ios,spcs)]
mod = PCA(*dss, window=1.5)['PC-1']
else:
obs = pd.concat([io.timeseries['obs_obs'][:, spc-1, row, col].to_series() for io, spc in zip(ios,spcs)]).sort_index()
mod = pd.concat([io.timeseries['obs_fcst'][:, spc-1, row, col].to_series() for io, spc in zip(ios,spcs)]).sort_index()
if (len(obs) == 0) | (len(mod) == 0):
continue
if mode == 'longterm':
obs_clim = calc_anom(obs, return_clim=True)
mod_clim = calc_anom(mod, return_clim=True)
obs_anom = calc_anom(obs, mode='shortterm')
mod_anom = calc_anom(mod, mode='shortterm')
m_obs = (obs_clim + obs_anom).resample('1D').mean()
m_mod = (mod_clim + mod_anom).resample('1D').mean()
i_yr = m_obs.index.year.values - years.min()
i_doy = m_obs.index.dayofyear.values - 1
data_obs[i_til, i_doy, i_yr, i_pol, i_orb] = m_obs.replace(np.nan, -9999.).values
data_mod[i_til, i_doy, i_yr, i_pol, i_orb] = m_mod.replace(np.nan, -9999.).values
elif mode == 'shortterm':
for i_yr, yr in enumerate(years):
data_obs[i_til, :, i_yr, i_pol, i_orb] = calc_clim_p(obs[obs.index.year==yr][date_from:date_to])[0].replace(np.nan, -9999.).values
data_mod[i_til, :, i_yr, i_pol, i_orb] = calc_clim_p(mod[mod.index.year==yr][date_from:date_to])[0].replace(np.nan, -9999.).values
n_data[i_til, :, i_yr, i_pol, i_orb] = len(obs[obs.index.year==yr][date_from:date_to].dropna())
else:
# TODO: Doesn't work currently!
data['m_obs'].sel(tile_id=til, pol=pol, angle=ang, orbit=orb)[:],\
data['s_obs'].sel(tile_id=til, pol=pol, angle=ang, orbit=orb)[:] = calc_clim_p(obs[date_from:date_to])
data['m_mod'].sel(tile_id=til, pol=pol, angle=ang, orbit=orb)[:],\
data['s_mod'].sel(tile_id=til, pol=pol, angle=ang, orbit=orb)[:] = calc_clim_p(mod[date_from:date_to])
data['N_data'].sel(tile_id=til, pol=pol, angle=ang, orbit=orb)[:] = len(obs[date_from:date_to].dropna())
modes = np.array([0, 0])
sdate = np.array([date_from.year, date_from.month, date_from.day, 0, 0])
edate = np.array([date_to.year, date_to.month, date_to.day, 0, 0])
lengths = np.array([len(tiles), len(angles), 1]) # tiles, incidence angles, whatever
np.save('/Users/u0116961/data_sets/data_mod', data_mod)
np.save('/Users/u0116961/data_sets/data_obs', data_obs)
# ----- write output files -----
if mode == 'longterm':
for i_orb, orb in enumerate(orbits[0]):
# !!! inconsistent with the definition in the obs_paramfile (species) !!!
modes[0] = 1 if orb == 'A' else 0
for i_yr, yr in enumerate(years):
for i_doy, doy in enumerate(doys):
res = template.copy()
# for i_ang, ang in enumerate(angles):
ang = angles[0]
for i_pol, pol in enumerate(pols):
res.loc[:, f'm_obs_{pol}_{ang}'] = data_obs[:, i_doy, i_yr, i_pol, i_orb].astype('float32')
res.loc[:, f's_obs_{pol}_{ang}'] = data_obs[:, i_doy, i_yr, i_pol, i_orb].astype('float32')
res.loc[:, f'm_mod_{pol}_{ang}'] = data_mod[:, i_doy, i_yr, i_pol, i_orb].astype('float32')
res.loc[:, f's_mod_{pol}_{ang}'] = data_mod[:, i_doy, i_yr, i_pol, i_orb].astype('float32')
res.loc[:, f'N_data_{pol}_{ang}'] = 999
res.replace(np.nan, -9999, inplace=True)
fdir = froot / f'y{yr:04}'
if not fdir.exists():
Path.mkdir(fdir, parents=True)
fname = fdir / f'{fbase}_{orb}_d{doy:03}.bin'
fid = open(fname, 'wb')
ios[0].write_fortran_block(fid, modes)
ios[0].write_fortran_block(fid, sdate)
ios[0].write_fortran_block(fid, edate)
ios[0].write_fortran_block(fid, lengths)
ios[0].write_fortran_block(fid, angles.astype('float')) # required by LDASsa!!
for f in res.columns.values:
ios[0].write_fortran_block(fid, res[f].values)
fid.close()
else:
for i_pent, pent in enumerate(pentads):
for i_orb, orb in enumerate(orbits[0]):
# !!! inconsistent with the definition in the obs_paramfile (species) !!!
modes[0] = 1 if orb == 'A' else 0
if mode == 'shortterm':
for i_yr, yr in enumerate(years):
res = template.copy()
for ang in angles:
for i_pol, pol in enumerate(pols):
res.loc[:, f'm_obs_{pol}_{ang}'] = data_obs[:, i_pent, i_yr, i_pol, i_orb].astype('float32')
res.loc[:, f's_obs_{pol}_{ang}'] = data_obs[:, i_pent, i_yr, i_pol, i_orb].astype('float32')
res.loc[:, f'm_mod_{pol}_{ang}'] = data_mod[:, i_pent, i_yr, i_pol, i_orb].astype('float32')
res.loc[:, f's_mod_{pol}_{ang}'] = data_mod[:, i_pent, i_yr, i_pol, i_orb].astype('float32')
res.loc[:, f'N_data_{pol}_{ang}'] = n_data[:, i_pent, i_yr, i_pol, i_orb].astype('int32')
res.replace(np.nan, -9999, inplace=True)
fname = froot / f'{fbase}_{orb}_p{pent:02}_y{yr:04}.bin'
fid = open(fname, 'wb')
ios[0].write_fortran_block(fid, modes)
ios[0].write_fortran_block(fid, sdate)
ios[0].write_fortran_block(fid, edate)
ios[0].write_fortran_block(fid, lengths)
ios[0].write_fortran_block(fid, angles.astype('float')) # required by LDASsa!!
for f in res.columns.values:
ios[0].write_fortran_block(fid, res[f].values)
fid.close()
else:
res = template.copy()
for ang in angles:
for pol in pols:
res.loc[:, f'm_obs_{pol}_{ang}'] = data['m_obs'].sel(pol=pol, angle=ang, orbit=orb, pentad=pent).to_series()
res.loc[:, f's_obs_{pol}_{ang}'] = data['s_obs'].sel(pol=pol, angle=ang, orbit=orb, pentad=pent).to_series()
res.loc[:, f'm_mod_{pol}_{ang}'] = data['m_mod'].sel(pol=pol, angle=ang, orbit=orb, pentad=pent).to_series()
res.loc[:, f's_mod_{pol}_{ang}'] = data['s_mod'].sel(pol=pol, angle=ang, orbit=orb, pentad=pent).to_series()
res.loc[:, f'N_data_{pol}_{ang}'] = data['N_data'].sel(pol=pol, angle=ang, orbit=orb, pentad=pent).to_series()
res.replace(np.nan, -9999, inplace=True)
fname = froot / f'{fbase}_{orb}_p{pent:02}.bin'
fid = open(fname, 'wb')
ios[0].write_fortran_block(fid, modes)
ios[0].write_fortran_block(fid, sdate)
ios[0].write_fortran_block(fid, edate)
ios[0].write_fortran_block(fid, lengths)
ios[0].write_fortran_block(fid, angles.astype('float')) # required by LDASsa!!
for f in res.columns.values:
ios[0].write_fortran_block(fid, res[f].values)
fid.close()
def create_observation_perturbations():
froot = Path('/Users/u0116961/Documents/work/MadKF/CLSM/SM_err_ratio/GEOSldas')
fbase = 'SMOS_fit_Tb_'
pc = 'Pcorr'
io = GEOSldas_io()
ensvar = pd.read_csv(froot / 'ens_vars' / pc / 'ens_var.csv', index_col=0)
obs_err = ensvar[['col', 'row']]
obs_err.loc[:, 'tile_id'] = io.grid.tilecoord.loc[obs_err.index, 'tile_id'].values
tc_res = pd.read_csv(froot / 'sm_validation' / pc / 'result.csv', index_col=0)
tc_res.index = np.vectorize(io.grid.colrow2tilenum)(tc_res.col.values.astype('int'), tc_res.row.values.astype('int'), local=False)
tc_res = tc_res.reindex(obs_err.index)
ks = {'abs': [(1,0)],
'anom_lt': [(1,0)],
'anom_lst': [(1,0)],
'anom_st':[(1,0)]}
# ks = {'abs': [(1,0)]}
for mode, k in ks.items():
tag_r = f'ubrmse_grid_{mode}_m_SMAP_tc_ASCAT_SMAP_CLSM'
tag_p = f'ubrmse_grid_{mode}_m_CLSM_tc_ASCAT_SMAP_CLSM'
dir_out = froot / 'observation_perturbations' / pc / mode
if not dir_out.exists():
Path.mkdir(dir_out, parents=True)
for i, (k_m, k_a) in enumerate(k):
scl = (tc_res[tag_r] * k_m + k_a)**2 / tc_res[tag_p]**2
# scl = tc_res[tag_r]**2 / tc_res[tag_p]**2
# tmp = (1 + scl) ** (-1)
# tmp[tmp < 0.5] **= 1.75
# scl = tmp ** (-1.75) - 1
for spc in np.arange(1, 5):
obs_err.loc[:, 'obs_var_spc%i' % spc] = (ensvar['fcst_var_spc%i' % spc] * scl)**0.5
obs_err.loc[(obs_err['obs_var_spc%i' % spc] < 0.1), 'obs_var_spc%i' % spc] = 0.1
# obs_err.loc[(obs_err['obs_var_spc%i' % spc] > 1600), 'obs_var_spc%i' % spc] = 1600
obs_err.loc[:, 'obs_var_spc%i' % spc] = fill_gaps(obs_err, 'obs_var_spc%i' % spc, smooth=False, grid=io.grid)['obs_var_spc%i' % spc]
dtype = template_error_Tb40()[0]
angles = np.array([40., ])
orbits = ['A', 'D']
template = pd.DataFrame(columns=dtype.names).astype('float32')
template['lon'] = io.grid.tilecoord['com_lon'].values.astype('float32')
template['lat'] = io.grid.tilecoord['com_lat'].values.astype('float32')
template.index += 1
modes = np.array([0, 0])
sdate = np.array([2015, 4, 1, 0, 0])
edate = np.array([2021, 4, 1, 0, 0])
lengths = np.array([len(template), len(angles)]) # tiles, incidence angles, whatever
# ----- write output files -----
for orb in orbits:
# !!! inconsistent with the definition in the obs_paramfile (species) !!!
modes[0] = 1 if orb == 'A' else 0
res = template.copy()
spc = 0 if orb == 'A' else 1
res.loc[:, 'err_Tbh'] = obs_err.loc[res.index, 'obs_var_spc%i' % (spc + 1)].values
spc = 2 if orb == 'A' else 3
res.loc[:, 'err_Tbv'] = obs_err.loc[res.index, 'obs_var_spc%i' % (spc + 1)].values
fname = dir_out / (fbase + orb + '.bin')
with open(fname, 'wb') as fid:
io.write_fortran_block(fid, modes)
io.write_fortran_block(fid, sdate)
io.write_fortran_block(fid, edate)
io.write_fortran_block(fid, lengths)
io.write_fortran_block(fid, angles)
for f in res.columns.values:
io.write_fortran_block(fid, res[f].values)
def plot_rtm_parameters():
root = paths().plots / 'RTM_parameters'
experiments = ['US_M36_SMOS_DA_calibrated_scaled', 'US_M36_SMOS_DA_nocal_scaled_harmonic']
tc = GEOSldas_io().grid.tilecoord
tg = GEOSldas_io().grid.tilegrids
tc.i_indg -= tg.loc['domain','i_offg'] # col / lon
tc.j_indg -= tg.loc['domain','j_offg'] # row / lat
lons = np.unique(tc.com_lon.values)
lats = np.unique(tc.com_lat.values)[::-1]
lons, lats = np.meshgrid(lons, lats)
llcrnrlat = 24
urcrnrlat = 51
llcrnrlon = -128
urcrnrlon = -64
figsize = (20, 10)
# cbrange = (-20, 20)
cmap = 'jet'
fontsize = 20
for exp in experiments:
outpath = root / exp
if not outpath.exists:
outpath.mkdir(exists=True)
params = GEOSldas_io(exp=exp).read_params('RTMparam')
for param in params:
fname = outpath / (param + '.png')
img = np.full(lons.shape, np.nan)
img[tc.j_indg.values, tc.i_indg.values] = params[param].values
img_masked = np.ma.masked_invalid(img)
f = plt.figure(num=None, figsize=figsize, dpi=90, facecolor='w', edgecolor='k')
m = Basemap(projection='mill',
llcrnrlat=llcrnrlat,
urcrnrlat=urcrnrlat,
llcrnrlon=llcrnrlon,
urcrnrlon=urcrnrlon,
resolution='c')
m.drawcoastlines()
m.drawcountries()
m.drawstates()
im = m.pcolormesh(lons, lats, img_masked, cmap=cmap, latlon=True)
# im.set_clim(vmin=cbrange[0], vmax=cbrange[1])
cb = m.colorbar(im, "bottom", size="7%", pad="8%")
for t in cb.ax.get_xticklabels():
t.set_fontsize(fontsize)
for t in cb.ax.get_yticklabels():
t.set_fontsize(fontsize)
plt.title(param)
plt.savefig(fname, dpi=f.dpi)
plt.close()
def plot_perturbations(dir_out):
root = Path(
'/Users/u0116961/Documents/work/MadKF/CLSM/SM_err_ratio/GEOSldas/observation_perturbations/Pcorr'
)
res_tc = pd.read_csv(
'/Users/u0116961/Documents/work/MadKF/CLSM/SM_err_ratio/GEOSldas/sm_validation/Pcorr/result.csv',
index_col=0)
pc = 'Pcorr'
io = GEOSldas_io('ObsFcstAna')
io2 = LDASsa_io('ObsFcstAna')
lut = pd.read_csv(Paths().lut, index_col=0)
ind = np.vectorize(io.grid.colrow2tilenum)(lut.ease2_col,
lut.ease2_row,
local=False)
dtype, hdr, length = template_error_Tb40()
f = plt.figure(figsize=(22, 8))
fontsize = 14
cbrange = [0, 8]
cmap = cc.cm.bjy_r
# cmap='viridis'
modes = ['anom_lst', 'anom_lt', 'anom_st']
titles = ['Anomalies', 'LF signal', 'HF signal']
for i, (mode, title) in enumerate(zip(modes, titles)):
fA = root / f'{mode}' / 'SMOS_fit_Tb_A.bin'
fD = root / f'{mode}' / 'SMOS_fit_Tb_D.bin'
imgA = io2.read_fortran_binary(fA, dtype, hdr=hdr, length=length)
imgD = io2.read_fortran_binary(fD, dtype, hdr=hdr, length=length)
imgA.index += 1
imgD.index += 1
r_asc_smap = res_tc[f'r_grid_{mode}_p_ASCAT_SMAP']
r_asc_clsm = res_tc[f'r_grid_{mode}_p_ASCAT_CLSM']
r_smap_clsm = res_tc[f'r_grid_{mode}_p_SMAP_CLSM']
thres = 0.2
ind_valid = res_tc[(r_asc_smap > thres) & (r_asc_smap > thres) &
(r_asc_smap > thres)].index
ind_valid = np.vectorize(io.grid.colrow2tilenum)(
res_tc.loc[ind_valid, 'col'].values,
res_tc.loc[ind_valid, 'row'].values,
local=False)
plt.subplot(3, 4, i * 4 + 1)
im = plot_ease_img2(imgA.reindex(ind).reindex(ind_valid),
'err_Tbv',
cbrange=cbrange,
cmap=cmap,
io=io,
plot_cmap=False)
if i == 0:
plt.title('$\hat{R}$ (V-pol, Asc.)', fontsize=fontsize)
plt.ylabel(title, fontsize=fontsize)
plt.subplot(3, 4, i * 4 + 2)
plot_ease_img2(imgD.reindex(ind).reindex(ind_valid),
'err_Tbv',
cbrange=cbrange,
cmap=cmap,
io=io,
plot_cmap=False)
if i == 0:
plt.title('$\hat{R}$ (V-pol, Dsc.)', fontsize=fontsize)
plt.subplot(3, 4, i * 4 + 3)
plot_ease_img2(imgA.reindex(ind).reindex(ind_valid),
'err_Tbh',
cbrange=cbrange,
cmap=cmap,
io=io,
plot_cmap=False)
if i == 0:
plt.title('$\hat{R}$ (H-pol, Asc.)', fontsize=fontsize)
plt.subplot(3, 4, i * 4 + 4)
plot_ease_img2(imgD.reindex(ind).reindex(ind_valid),
'err_Tbh',
cbrange=cbrange,
cmap=cmap,
io=io,
plot_cmap=False)
if i == 0:
plt.title('$\hat{R}$ (H-pol, Dsc.)', fontsize=fontsize)
plot_centered_cbar(f, im, 4, fontsize=fontsize - 4, bottom=0.07)
plt.savefig(dir_out / f'perturbations.png', dpi=300, bbox_inches='tight')
plt.close()
def plot_ismn_statistics(res_path, dir_out):
res = pd.read_csv(res_path / 'ascat_eval.csv', index_col=0)
res_ismn = pd.read_csv(res_path / 'insitu_TCA.csv', index_col=0)
networks = ['SCAN', 'USCRN']
res_ismn = res_ismn.loc[res_ismn.network.isin(networks), :]
res_tc = pd.read_csv(
'/Users/u0116961/Documents/work/MadKF/CLSM/SM_err_ratio/GEOSldas/sm_validation/Pcorr/result.csv',
index_col=0)
tg = GEOSldas_io().grid.tilegrids
ind_ismn = []
for col, row in zip(res_ismn.ease_col.values + tg.loc['domain', 'i_offg'],
res_ismn.ease_row.values + tg.loc['domain', 'j_offg']):
try:
ind_ismn += [
res[(res.row == row) & (res.col == col)].index.values[0]
]
except:
continue
refs = ['Pcorr_OL', 'Pcorr_4K']
runs = [['Pcorr_anom_lst', 'Pcorr_anom_lt_ScDY', 'Pcorr_anom_st_ScYH'],
'Pcorr_LTST']
titles = [
'Individual assim. (ASCAT, SSM)',
'Joint assim. (ASCAT, SSM)',
'Individual assim. (ISMN, SSM)',
'Joint assim. (ISMN, SSM)',
'Individual assim. (ISMN, RZSM)',
'Joint assim. (ISMN, RZSM)',
]
modes = ['anom_lst', 'anom_lt', 'anom_st']
labels = [f'Anomaly skill', f'LF skill', f'HF skill']
f = plt.figure(figsize=(26, 14))
fontsize = 16
# output = 'ascat' # 'ascat' or 'ascat_ismn' or 'ismn'
# met = 'r_corr' # 'R_model_insitu'# or 'R2_model' / 'r' or 'r_corr'
outputs = ['ascat_ismn', 'ismn', 'ismn']
mets = ['r_corr', 'R_model_insitu', 'R_model_insitu']
variables = ['sm_surface', 'sm_surface', 'sm_rootzone']
xres = res.copy()
xres_ismn = res_ismn.copy()
for i, (mode, label) in enumerate(zip(modes, labels)):
for j, (output, met, var) in enumerate(zip(outputs, mets, variables)):
res = xres.copy()
res_ismn = xres_ismn.copy()
if 'ubRMSD' in met:
lim = [-0.015, 0.015]
xloc = -0.0135
bins = 15
else:
if 'ascat' not in output:
lim = [-0.2, 0.3]
xloc = -0.18
bins = 15
else:
lim = [-0.25, 0.25]
xloc = -0.22
bins = 20
for k, run in enumerate(runs):
r = run if k == 1 else run[i]
title = titles[j * 2 + k] if i == 0 else ''
if 'ascat' in output:
col_ol = f'ana_{met}_Pcorr_OL_{mode}'
col_4k = f'ana_{met}_Pcorr_4K_{mode}'
col_da = f'ana_{met}_{r}_{mode}'
if 'ismn' in output:
res = res.reindex(ind_ismn)
else:
r_asc_smap = res_tc[f'r_grid_{mode}_p_ASCAT_SMAP']
r_asc_clsm = res_tc[f'r_grid_{mode}_p_ASCAT_CLSM']
r_smap_clsm = res_tc[f'r_grid_{mode}_p_SMAP_CLSM']
thres = 0.2
ind_valid = res_tc[(r_asc_smap > thres)
& (r_asc_smap > thres) &
(r_asc_smap > thres)].index
res = res.reindex(ind_valid)
else:
res = res_ismn
col_ol = f'{met}_Pcorr_OL_{mode}_{var}'
col_4k = f'{met}_Pcorr_4K_{mode}_{var}'
col_da = f'{met}_{r}_{mode}_{var}'
if 'R2' in met:
res[col_ol] **= 0.5
res[col_4k] **= 0.5
res[col_da] **= 0.5
if not ((output == 'ismn') and ('R2' not in met)):
print('filtered')
res[col_ol][res[col_ol] <= 0] = np.nan
res[col_4k][res[col_4k] <= 0] = np.nan
res[col_da][res[col_da] <= 0] = np.nan
res[col_ol][res[col_ol] >= 1] = np.nan
res[col_4k][res[col_4k] >= 1] = np.nan
res[col_da][res[col_da] >= 1] = np.nan
res['da'] = res[col_da] - res[col_ol]
res['4k'] = res[col_4k] - res[col_ol]
ax = plt.subplot(3, 6, i * 6 + j * 2 + k + 1)
p1 = res['4k'].hist(bins=bins,
grid=False,
ax=ax,
range=lim,
alpha=0.8)
p2 = res['da'].hist(bins=bins,
grid=False,
ax=ax,
range=lim,
alpha=0.6)
plt.yticks(color='w', fontsize=5)
if i < 2:
plt.xticks(color='w', fontsize=1)
if i == 0:
plt.title(title, fontsize=fontsize - 3)
if (j == 0) & (k == 0):
plt.ylabel(label, fontsize=fontsize)
if (i == 2) & (j == 2) & (k == 1):
plt.legend(labels=[
'R$_{4K, ref}$ - R$_{OL, ref}$',
'R$_{TC, ref}$ - R$_{OL, ref}$'
],
loc='lower right',
fontsize=fontsize - 4)
plt.axvline(color='black', linestyle='--', linewidth=1)
plt.xlim(lim)
ylim = ax.get_ylim()
yloc1 = ylim[1] - (ylim[1] - ylim[0]) / 10
yloc2 = ylim[1] - 1.6 * (ylim[1] - ylim[0]) / 10
plt.text(xloc,
yloc1,
'mean = %.2f' % res['4k'].mean(),
color='#1f77b4',
fontsize=fontsize - 3)
plt.text(xloc,
yloc2,
'mean = %.2f' % res['da'].mean(),
color='#ff7f0e',
fontsize=fontsize - 3)
f.savefig(dir_out / f'stats.png', dpi=300, bbox_inches='tight')
plt.close()
def plot_filter_diagnostics(res_path, dir_out):
fname = res_path / 'filter_diagnostics.nc'
res = pd.read_csv(
'/Users/u0116961/Documents/work/MadKF/CLSM/SM_err_ratio/GEOSldas/sm_validation/Pcorr/result.csv',
index_col=0)
res_tc = pd.read_csv(
'/Users/u0116961/Documents/work/MadKF/CLSM/SM_err_ratio/GEOSldas/sm_validation/Pcorr/result.csv',
index_col=0)
tg = GEOSldas_io().grid.tilegrids
res_cols = res.col.values - tg.loc['domain', 'i_offg']
res_rows = res.row.values - tg.loc['domain', 'j_offg']
r_asc_smap = res_tc[f'r_grid_anom_lst_p_ASCAT_SMAP']
r_asc_clsm = res_tc[f'r_grid_anom_lst_p_ASCAT_CLSM']
r_smap_clsm = res_tc[f'r_grid_anom_lst_p_SMAP_CLSM']
thres = 0.2
ind_valid = res_tc[(r_asc_smap > thres) & (r_asc_smap > thres) &
(r_asc_smap > thres)].index
fontsize = 14
root = Path('/Users/u0116961/data_sets/GEOSldas_runs')
runs = [run.name for run in root.glob('*_DA_SMAP_*')]
runs += ['NLv4_M36_US_OL_Pcorr', 'NLv4_M36_US_OL_noPcorr']
tags = ['OL_Pcorr', 'Pcorr_4K', f'Pcorr_anom_lst', 'Pcorr_LTST']
iters = [np.where([tag in run for run in runs])[0][0] for tag in tags]
titles = ['Open-loop', '4K constant', 'Anomalies']
labels = ['H pol. / Asc.', 'V pol. / Asc.']
with Dataset(fname) as ds:
lons = ds.variables['lon'][:]
lats = ds.variables['lat'][:]
lons, lats = np.meshgrid(lons, lats)
var = 'innov_autocorr'
cbrange = [0, 0.7]
step = 0.2
cmap = 'viridis'
f = plt.figure(figsize=(19, 6))
for j, (spc, label) in enumerate(zip([0, 2], labels)):
for i, (it_tit, it) in enumerate(zip(titles, iters)):
title = it_tit if j == 0 else ''
plt.subplot(2, 3, j * 3 + i + 1)
data = ds.variables[var][:, :, it, spc]
res['tmp'] = data[res_rows, res_cols]
im = plot_ease_img(res.reindex(ind_valid),
'tmp',
fontsize=fontsize,
cbrange=cbrange,
cmap=cmap,
title=title,
plot_cb=False,
print_meanstd=True)
if i == 0:
plt.ylabel(label, fontsize=fontsize)
plot_centered_cbar(f, im, 3, fontsize=fontsize - 2, bottom=0.07)
fout = dir_out / f'{var}.png'
f.savefig(fout, dpi=300, bbox_inches='tight')
plt.close()
ax.set_title('Asc / V-pol', fontsize=16)
ax = f.add_subplot(2,2,4)
obserr = io.images.sel(species=4)['obs_fcstvar'].mean('time').values
plot_xarr_img(obserr,lons,lats,cbrange=[0,50])
ax.set_title('Dsc / V-pol', fontsize=16)
plt.tight_layout()
plt.show()
if __name__=='__main__':
lat, lon = 41.83347716648588, -98.47471538470059
grid = GEOSldas_io().grid
col, row = grid.lonlat2colrow(lon, lat)
plot_innov(spc=1, row=row, col=col)
# plot_catparams()
# llcrnrlat = -58.,
# urcrnrlat = 78.,
# llcrnrlon = -172.,
# urcrnrlon = 180.,
# clipped global
# llcrnrlat=-58.,
# urcrnrlat=60.,
def plot_uncertainty_ratios(dir_out):
sensors = ['ASCAT', 'SMAP', 'CLSM']
res = pd.read_csv(
'/Users/u0116961/Documents/work/MadKF/CLSM/SM_err_ratio/GEOSldas/sm_validation/Pcorr/result.csv',
index_col=0)
res_tc = pd.read_csv(
'/Users/u0116961/Documents/work/MadKF/CLSM/SM_err_ratio/GEOSldas/sm_validation/Pcorr/result.csv',
index_col=0)
tg = GEOSldas_io().grid.tilegrids
res_cols = res.col.values - tg.loc['domain', 'i_offg']
res_rows = res.row.values - tg.loc['domain', 'j_offg']
figsize = (16, 4)
fontsize = 10
cb = [-10, 10]
cmap = cc.cm.bjy
modes = ['4K', 'anom_lst', 'anom_lt', 'anom_st']
titles = ['4K benchmark', 'Anomalies', 'LF signal', 'HF signal']
ios = [
GEOSldas_io('ObsFcstAna',
exp=f'NLv4_M36_US_DA_SMAP_Pcorr_{mode}').timeseries
for mode in modes
]
io_ol = GEOSldas_io('ObsFcstAna', exp=f'NLv4_M36_US_OL_Pcorr').timeseries
grid = GEOSldas_io().grid
f = plt.figure(figsize=figsize)
for n, (mode, title, io_da) in enumerate(zip(modes, titles, ios)):
if n > 0:
plt.subplot(2, 4, n + 1)
tagP = 'ubrmse_grid_' + mode + '_m_CLSM_tc_ASCAT_SMAP_CLSM'
tagR = 'ubrmse_grid_' + mode + '_m_SMAP_tc_ASCAT_SMAP_CLSM'
res['tmp'] = 10 * np.log10(res[tagP]**2 / res[tagR]**2)
r_asc_smap = res_tc[f'r_grid_{mode}_p_ASCAT_SMAP']
r_asc_clsm = res_tc[f'r_grid_{mode}_p_ASCAT_CLSM']
r_smap_clsm = res_tc[f'r_grid_{mode}_p_SMAP_CLSM']
thres = 0.2
ind_valid = res_tc[(r_asc_smap > thres) & (r_asc_smap > thres) &
(r_asc_smap > thres)].index
img = plot_ease_img(res.reindex(ind_valid),
'tmp',
fontsize=fontsize,
cbrange=cb,
cmap=cmap,
plot_cb=False)
plt.title(title, fontsize=fontsize)
if n == 1:
plt.ylabel('TCA unc. ratio', fontsize=fontsize)
for n, (mode, title, io_da) in enumerate(zip(modes, titles, ios)):
if mode != '4K':
r_asc_smap = res_tc[f'r_grid_{mode}_p_ASCAT_SMAP']
r_asc_clsm = res_tc[f'r_grid_{mode}_p_ASCAT_CLSM']
r_smap_clsm = res_tc[f'r_grid_{mode}_p_SMAP_CLSM']
thres = 0.2
ind_valid = res_tc[(r_asc_smap > thres) & (r_asc_smap > thres) &
(r_asc_smap > thres)].index
else:
ind_valid = res.index
avg = np.full(io_da['obs_obsvar'].shape[1::], np.nan)
for spc in range(4):
tmp1 = io_da['obs_obsvar'][:, spc, :, :].values
tmp2 = io_ol['obs_fcstvar'][:, spc, :, :].values
avg[spc, :, :] = np.nanmean(tmp2 / tmp1, axis=0)
avg = np.nanmean(avg, axis=0)
res['avg'] = 10 * np.log10(avg[res_rows, res_cols])
plt.subplot(2, 4, n + 5)
img = plot_ease_img(res.reindex(ind_valid),
'avg',
fontsize=fontsize,
cbrange=cb,
cmap=cmap,
plot_cb=False)
if n == 0:
plt.title(title, fontsize=fontsize)
plt.ylabel('Ens. var. ratio', fontsize=fontsize)
plot_centered_cbar(f, img, 4, fontsize=fontsize, bottom=0.07)
fout = dir_out / 'uncertainty_ratio.png'
f.savefig(fout, dpi=300, bbox_inches='tight')
plt.close()