def run(mode='annual'): # annual / longterm
froot = r"D:\data_sets\LDAS_runs\US_M36_SMOS_noDA_cal_unscaled\obs_scaling_pentadal_annual"
fbase = '7Thv_TbSM_001_SMOS_zscore_stats_2010_p37_2015_p36_hscale_0.00_W_9p_Nmin_20_'
io = LDAS_io('ObsFcstAna', exp='US_M36_SMOS_noDA_cal_unscaled')
dtype = template_scaling()[0]
tiles = io.grid.tilecoord['tile_id'].values.astype('int32')
angles = np.array([30., 35., 40., 45., 50., 55., 60.])
pols = ['V', 'H']
orbits = ['A', 'D']
template = pd.DataFrame(columns=dtype.names, index=tiles).astype('float32')
template['lon'] = io.grid.tilecoord['com_lon'].values.astype('float32')
template['lat'] = io.grid.tilecoord['com_lat'].values.astype('float32')
template['tile_id'] = tiles.astype('int32')
pentads = np.arange(73) + 1
if mode == 'longterm':
dummy = np.full(
[len(tiles),
len(pentads),
len(angles),
len(pols),
len(orbits)], -9999)
coords = {
'tile_id': tiles,
'pentad': pentads,
'angle': angles,
'pol': pols,
'orbit': orbits
}
darr = xr.DataArray(
dummy,
coords=coords,
dims=['tile_id', 'pentad', 'angle', 'pol', 'orbit'])
else:
years = np.arange(2010, 2017)
dummy = np.full([
len(tiles),
len(pentads),
len(years),
len(angles),
len(pols),
len(orbits)
], -9999)
coords = {
'tile_id': tiles,
'pentad': pentads,
'year': years,
'angle': angles,
'pol': pols,
'orbit': orbits
}
darr = xr.DataArray(
dummy,
coords=coords,
dims=['tile_id', 'pentad', 'year', 'angle', 'pol', 'orbit'])
data = xr.Dataset({
'm_obs': darr.astype('float32'),
'm_mod': darr.astype('float32'),
'N_data': darr.astype('int32')
})
# ----- calculate mean and reshuffle -----
for i, til in enumerate(tiles):
logging.info('%i/%i' % (i, len(tiles)))
for pol in pols:
for ang in angles:
for orb in orbits:
spc = io.get_species(pol=pol, ang=ang, orbit=orb)
col, row = io.grid.tileid2colrow(til)
obs = io.timeseries['obs_obs'][spc - 1, row,
col].to_series()
mod = io.timeseries['obs_fcst'][spc - 1, row,
col].to_series()
if mode == 'longterm':
data['m_obs'].sel(
tile_id=til, pol=pol, angle=ang,
orbit=orb)[:] = calc_clim_p(obs).values
data['m_mod'].sel(
tile_id=til, pol=pol, angle=ang,
orbit=orb)[:] = calc_clim_p(mod).values
data['N_data'].sel(tile_id=til,
pol=pol,
angle=ang,
orbit=orb)[:] = len(obs.dropna())
else:
for yr in years:
data['m_obs'].sel(
tile_id=til,
pol=pol,
angle=ang,
orbit=orb,
year=yr)[:] = calc_clim_p(
obs[obs.index.year == yr]).values
data['m_mod'].sel(
tile_id=til,
pol=pol,
angle=ang,
orbit=orb,
year=yr)[:] = calc_clim_p(
mod[obs.index.year == yr]).values
data['N_data'].sel(
tile_id=til,
pol=pol,
angle=ang,
orbit=orb,
year=yr)[:] = len(
obs[obs.index.year == yr].dropna())
modes = np.array([0, 0])
sdate = np.array([2010, 1, 1, 0, 0])
edate = np.array([2016, 12, 31, 0, 0])
lengths = np.array([len(tiles), len(angles),
1]) # tiles, incidence angles, whatever
# ----- write output files -----
for pent in pentads:
for orb in orbits:
# !!! inconsistent with the definition in the obs_paramfile (species) !!!
modes[0] = 1 if orb == 'A' else 0
if mode == 'longterm':
res = template.copy()
for ang in angles:
for pol in pols:
tmp = data['m_obs'].sel(pol=pol,
angle=ang,
orbit=orb,
pentad=pent).to_series()
res.loc[tmp.index, 'm_obs_' + pol + '_%i' % ang] = tmp
res.loc[tmp.index, 's_obs_' + pol + '_%i' % ang] = tmp
tmp = data['m_mod'].sel(pol=pol,
angle=ang,
orbit=orb,
pentad=pent).to_series()
res.loc[tmp.index, 'm_mod_' + pol + '_%i' % ang] = tmp
res.loc[tmp.index, 's_mod_' + pol + '_%i' % ang] = tmp
tmp = data['N_data'].sel(pol=pol,
angle=ang,
orbit=orb,
pentad=pent).to_series()
res.loc[tmp.index, 'N_data_' + pol + '_%i' % ang] = tmp
res.replace(np.nan, -9999, inplace=True)
fname = os.path.join(froot,
fbase + orb + '_p%02i' % pent + '.bin')
fid = open(fname, 'wb')
io.write_fortran_block(fid, modes)
io.write_fortran_block(fid, sdate)
io.write_fortran_block(fid, edate)
io.write_fortran_block(fid, lengths)
io.write_fortran_block(fid, angles)
for f in res.columns.values:
io.write_fortran_block(fid, res[f].values)
fid.close()
else:
for yr in years:
res = template.copy()
for ang in angles:
for pol in pols:
tmp = data['m_obs'].sel(pol=pol,
angle=ang,
orbit=orb,
pentad=pent,
year=yr).to_series()
res.loc[tmp.index,
'm_obs_' + pol + '_%i' % ang] = tmp
res.loc[tmp.index,
's_obs_' + pol + '_%i' % ang] = tmp
tmp = data['m_mod'].sel(pol=pol,
angle=ang,
orbit=orb,
pentad=pent,
year=yr).to_series()
res.loc[tmp.index,
'm_mod_' + pol + '_%i' % ang] = tmp
res.loc[tmp.index,
's_mod_' + pol + '_%i' % ang] = tmp
tmp = data['N_data'].sel(pol=pol,
angle=ang,
orbit=orb,
pentad=pent,
year=yr).to_series()
res.loc[tmp.index,
'N_data_' + pol + '_%i' % ang] = tmp
res.replace(np.nan, -9999, inplace=True)
fname = os.path.join(
froot,
fbase + orb + '_p%02i' % pent + '_y%04i' % yr + '.bin')
fid = open(fname, 'wb')
io.write_fortran_block(fid, modes)
io.write_fortran_block(fid, sdate)
io.write_fortran_block(fid, edate)
io.write_fortran_block(fid, lengths)
io.write_fortran_block(fid, angles)
for f in res.columns.values:
io.write_fortran_block(fid, res[f].values)
fid.close()
def run():
anom = False
longterm = False
fcst_err_corrected = False
exp = 'US_M36_SMAP_TB_MadKF_OL_it11'
io = LDAS_io('ObsFcstAna', exp)
froot = Path(
'/Users/u0116961/Documents/work/MadKF/CLSM/SMAP/rmsd_pert/error_files')
fbase = 'SMOS_fit_Tb_'
dir_out = froot / ((('anom_' +
('lt' if longterm else 'st')) if anom else 'abs') +
('_fcst_corr' if fcst_err_corrected else '_uncorr'))
if not dir_out.exists():
Path.mkdir(dir_out, parents=True)
dtype = template_error_Tb40()[0]
angles = np.array([
40.,
])
orbits = ['A', 'D']
tiles = io.grid.tilecoord['tile_id'].values.astype('int32')
ind_lat = io.grid.tilecoord.loc[:,
'j_indg'].values - io.grid.tilegrids.loc[
'domain', 'j_offg']
ind_lon = io.grid.tilecoord.loc[:,
'i_indg'].values - io.grid.tilegrids.loc[
'domain', 'i_offg']
template = pd.DataFrame(columns=dtype.names, index=tiles).astype('float32')
template['lon'] = io.grid.tilecoord['com_lon'].values.astype('float32')
template['lat'] = io.grid.tilecoord['com_lat'].values.astype('float32')
modes = np.array([0, 0])
sdate = np.array([2015, 4, 1, 0, 0])
edate = np.array([2020, 4, 31, 0, 0])
lengths = np.array([len(tiles),
len(angles)]) # tiles, incidence angles, whatever
dims = io.timeseries['obs_obs'].shape
obs_errstd = np.full(dims[1::], np.nan)
# ----- Calculate anomalies -----
cnt = 0
for spc in np.arange(dims[1]):
for lat in np.arange(dims[2]):
for lon in np.arange(dims[3]):
cnt += 1
logging.info('%i / %i' % (cnt, np.prod(dims[1::])))
try:
if anom:
obs = calc_anom(io.timeseries['obs_obs']
[:, spc, lat,
lon].to_dataframe()['obs_obs'],
longterm=longterm)
fcst = calc_anom(io.timeseries['obs_fcst']
[:, spc, lat,
lon].to_dataframe()['obs_fcst'],
longterm=longterm)
else:
obs = io.timeseries['obs_obs'][:, spc,
lat, lon].to_dataframe(
)['obs_obs']
fcst = io.timeseries['obs_fcst'][:, spc, lat,
lon].to_dataframe(
)['obs_fcst']
fcst_errvar = np.nanmean(
io.timeseries['obs_fcstvar']
[:, spc, lat, lon].values) if fcst_err_corrected else 0
tmp_obs_errstd = (((obs - fcst)**2).mean() -
fcst_errvar)**0.5
if not np.isnan(tmp_obs_errstd):
obs_errstd[spc, lat, lon] = tmp_obs_errstd
except:
pass
np.place(obs_errstd, obs_errstd < 0, 0)
np.place(obs_errstd, obs_errstd > 20, 20)
# ----- write output files -----
for orb in orbits:
# !!! inconsistent with the definition in the obs_paramfile (species) !!!
modes[0] = 1 if orb == 'A' else 0
res = template.copy()
res.index = np.arange(len(res)) + 1
res['row'] = ind_lat
res['col'] = ind_lon
spc = 0 if orb == 'A' else 1
res['err_Tbh'] = obs_errstd[spc, ind_lat, ind_lon]
spc = 2 if orb == 'A' else 3
res['err_Tbv'] = obs_errstd[spc, ind_lat, ind_lon]
res = fill_gaps(res, ['err_Tbh', 'err_Tbv'],
smooth=False,
grid=io.grid)
fname = os.path.join(dir_out, fbase + orb + '.bin')
fid = open(fname, 'wb')
io.write_fortran_block(fid, modes)
io.write_fortran_block(fid, sdate)
io.write_fortran_block(fid, edate)
io.write_fortran_block(fid, lengths)
io.write_fortran_block(fid, angles)
for f in res.drop(['row', 'col'], axis='columns').columns.values:
io.write_fortran_block(fid, res[f].values)
fid.close()
def run():
exp = 'US_M36_SMOS40_noDA_cal_scaled'
io = LDAS_io('ObsFcstAna', exp)
froot = r"D:\data_sets\LDAS_runs" + "\\" + exp + "\\obs_err"
fbase = 'SMOS_fit_Tb_'
dtype = template_error_Tb40()[0]
angles = np.array([
40.,
])
orbits = ['A', 'D']
tiles = io.grid.tilecoord['tile_id'].values.astype('int32')
ind_lat = io.grid.tilecoord.loc[:,
'j_indg'].values - io.grid.tilegrids.loc[
'domain', 'j_offg']
ind_lon = io.grid.tilecoord.loc[:,
'i_indg'].values - io.grid.tilegrids.loc[
'domain', 'i_offg']
template = pd.DataFrame(columns=dtype.names, index=tiles).astype('float32')
template['lon'] = io.grid.tilecoord['com_lon'].values.astype('float32')
template['lat'] = io.grid.tilecoord['com_lat'].values.astype('float32')
modes = np.array([0, 0])
sdate = np.array([2010, 1, 1, 0, 0])
edate = np.array([2016, 12, 31, 0, 0])
lengths = np.array([len(tiles),
len(angles)]) # tiles, incidence angles, whatever
dims = io.timeseries['obs_obs'].shape
obs_errstd = np.full(dims[0:-1], 4.)
# ----- Calculate anomalies -----
cnt = 0
for spc in np.arange(dims[0]):
for lat in np.arange(dims[1]):
for lon in np.arange(dims[2]):
cnt += 1
logging.info('%i / %i' % (cnt, np.prod(dims[0:-1])))
try:
obs = calc_anomaly(io.timeseries['obs_obs'][
spc, lat, lon, :].to_dataframe()['obs_obs'],
method='moving_average',
longterm=True)
fcst = calc_anomaly(io.timeseries['obs_fcst'][
spc, lat, lon, :].to_dataframe()['obs_fcst'],
method='moving_average',
longterm=True)
fcst_errvar = np.nanmean(
io.timeseries['obs_fcstvar'][spc, lat, lon, :].values)
tmp_obs_errstd = (((obs - fcst)**2).mean() -
fcst_errvar)**0.5
if not np.isnan(tmp_obs_errstd):
obs_errstd[spc, lat, lon] = tmp_obs_errstd
except:
pass
np.place(obs_errstd, obs_errstd < 0, 0)
np.place(obs_errstd, obs_errstd > 20, 20)
# ----- write output files -----
for orb in orbits:
# !!! inconsistent with the definition in the obs_paramfile (species) !!!
modes[0] = 1 if orb == 'A' else 0
res = template.copy()
spc = 0 if orb == 'A' else 1
res['err_Tbh'] = obs_errstd[spc, ind_lat, ind_lon]
spc = 2 if orb == 'A' else 3
res['err_Tbv'] = obs_errstd[spc, ind_lat, ind_lon]
fname = os.path.join(froot, fbase + orb + '.bin')
fid = open(fname, 'wb')
io.write_fortran_block(fid, modes)
io.write_fortran_block(fid, sdate)
io.write_fortran_block(fid, edate)
io.write_fortran_block(fid, lengths)
io.write_fortran_block(fid, angles)
for f in res.columns.values:
io.write_fortran_block(fid, res[f].values)
fid.close()