def test_vdr(self, class_case_dir):
# Init
cfg.initialize()
cfg.PARAMS['use_intersects'] = False
cfg.PATHS['working_dir'] = class_case_dir
cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif')
hef_file = get_demo_file('Hintereisferner_RGI5.shp')
gdir = workflow.init_glacier_directories(gpd.read_file(hef_file))[0]
exps = ['ERA5', 'ERA5dr']
files = []
ref_hgts = []
for base in exps:
cfg.PARAMS['baseline_climate'] = base
tasks.process_climate_data(gdir, output_filesuffix=base)
files.append(
gdir.get_filepath('climate_historical', filesuffix=base))
with xr.open_dataset(files[-1]) as ds:
ref_hgts.append(ds.ref_hgt)
assert ds.ref_pix_dis < 10000
with xr.open_dataset(files[0]) as d1, xr.open_dataset(files[1]) as d2:
np.testing.assert_allclose(d1.temp, d2.temp)
np.testing.assert_allclose(d1.prcp, d2.prcp)
# Fake tests, the plots look plausible
np.testing.assert_allclose(d2.gradient.mean(), -0.0058, atol=.001)
np.testing.assert_allclose(d2.temp_std.mean(), 3.35, atol=0.1)
def test_ecmwf_workflow(self, class_case_dir):
# Init
cfg.initialize()
cfg.PARAMS['use_intersects'] = False
cfg.PATHS['working_dir'] = class_case_dir
cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif')
hef_file = get_demo_file('Hintereisferner_RGI5.shp')
gdir = workflow.init_glacier_directories(gpd.read_file(hef_file))[0]
cfg.PARAMS['baseline_climate'] = 'CERA+ERA5L'
tasks.process_climate_data(gdir)
f_ref = gdir.get_filepath('climate_historical')
with xr.open_dataset(f_ref) as his:
# Let's do some basic checks
ci = gdir.get_climate_info()
assert ci['baseline_climate_source'] == 'CERA+ERA5L'
assert ci['baseline_hydro_yr_0'] == 1902
assert ci['baseline_hydro_yr_1'] == 2018
cfg.PARAMS['baseline_climate'] = 'CERA|ERA5'
tasks.process_climate_data(gdir)
f_ref = gdir.get_filepath('climate_historical')
with xr.open_dataset(f_ref) as his:
# Let's do some basic checks
ci = gdir.get_climate_info()
assert ci['baseline_climate_source'] == 'CERA|ERA5'
assert ci['baseline_hydro_yr_0'] == 1902
assert ci['baseline_hydro_yr_1'] == 2010
def test_all_at_once(self, class_case_dir):
# Init
cfg.initialize()
cfg.PARAMS['use_intersects'] = False
cfg.PATHS['working_dir'] = class_case_dir
cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif')
hef_file = get_demo_file('Hintereisferner_RGI5.shp')
gdir = workflow.init_glacier_directories(gpd.read_file(hef_file))[0]
exps = ['CRU', 'HISTALP', 'ERA5', 'ERA5L', 'CERA']
files = []
ref_hgts = []
for base in exps:
cfg.PARAMS['baseline_climate'] = base
tasks.process_climate_data(gdir, output_filesuffix=base)
files.append(
gdir.get_filepath('climate_historical', filesuffix=base))
with xr.open_dataset(files[-1]) as ds:
ref_hgts.append(ds.ref_hgt)
assert ds.ref_pix_dis < 30000
# TEMP
with xr.open_mfdataset(files, concat_dim=exps) as ds:
dft = ds.temp.to_dataframe().unstack().T
dft.index = dft.index.levels[1]
# Common period
dfy = dft.resample('AS').mean().dropna().iloc[1:]
dfm = dft.groupby(dft.index.month).mean()
assert dfy.corr().min().min() > 0.44 # ERA5L and CERA do no correlate
assert dfm.corr().min().min() > 0.97
dfavg = dfy.describe()
# Correct for hgt
ref_h = ref_hgts[0]
for h, d in zip(ref_hgts, exps):
dfy[d] = dfy[d] - 0.0065 * (ref_h - h)
dfm[d] = dfm[d] - 0.0065 * (ref_h - h)
dfavg_cor = dfy.describe()
# After correction less spread
assert dfavg_cor.loc['mean'].std() < 0.8 * dfavg.loc['mean'].std()
assert dfavg_cor.loc['mean'].std() < 2.1
# PRECIP
with xr.open_mfdataset(files, concat_dim=exps) as ds:
dft = ds.prcp.to_dataframe().unstack().T
dft.index = dft.index.levels[1]
# Common period
dfy = dft.resample('AS').mean().dropna().iloc[1:] * 12
dfm = dft.groupby(dft.index.month).mean()
assert dfy.corr().min().min() > 0.5
assert dfm.corr().min().min() > 0.8
dfavg = dfy.describe()
assert dfavg.loc['mean'].std() / dfavg.loc['mean'].mean() < 0.25 # %
def test_hydro_month_changes(self, hef_gdir):
# test for HEF if applying different hydro_months does the right thing
# check if mb of neighbouring hydro_months correlate
# do this for different climate scenarios
# maybe there is already somewhere an overview or a better way to get
# these dates, but I did not find it
base_data_time = {
'CRU': {
'start_year': 1901,
'end_year': 2014
},
'ERA5': {
'start_year': 1979,
'end_year': 2018
},
'ERA5dr': {
'start_year': 1979,
'end_year': 2019
},
'HISTALP': {
'start_year': 1850,
'end_year': 2014
},
'CERA': {
'start_year': 1901,
'end_year': 2010
},
'ERA5L': {
'start_year': 1981,
'end_year': 2018
}
}
gdir = hef_gdir
oggm.core.flowline.init_present_time_glacier(gdir)
mb_mod = oggm.core.massbalance.PastMassBalance(gdir)
h, w = gdir.get_inversion_flowline_hw()
exps = ['ERA5dr', 'CRU', 'HISTALP', 'ERA5', 'ERA5L', 'CERA']
for base in exps:
# this does not need to be the best one,
# just for comparison between different hydro months
mu_opt = 213.54
files = []
ref_hgts = []
dft = []
dfp = []
tot_mbs = []
cfg.PARAMS['baseline_climate'] = base
for m in np.arange(1, 13):
cfg.PARAMS['hydro_month_nh'] = m
fsuff = '_{}_{}'.format(base, m)
tasks.process_climate_data(gdir, output_filesuffix=fsuff)
files.append(
gdir.get_filepath('climate_historical', filesuffix=fsuff))
with xr.open_dataset(files[-1]) as ds:
ref_hgts.append(ds.ref_hgt)
dft.append(ds.temp.to_series())
dfp.append(ds.prcp.to_series())
ci = gdir.get_climate_info(input_filesuffix=fsuff)
# check if the right climate source is used
assert base in ci['baseline_climate_source']
mm = str(m) if m > 9 else str(0) + str(m)
mm_e = str(m - 1) if (m - 1) > 9 else str(0) + str(m - 1)
b_s_y = base_data_time[base]['start_year']
b_e_y = base_data_time[base]['end_year']
stime = '{}-{}-01'.format(b_s_y, mm)
assert ds.time[0] == np.datetime64(stime)
if m == 1:
assert ci['baseline_hydro_yr_0'] == b_s_y
if base == 'ERA5dr':
# do not have full 2019
assert ci['baseline_hydro_yr_1'] == b_e_y - 1
else:
assert ci['baseline_hydro_yr_1'] == b_e_y
elif m < 7 and base == 'ERA5dr':
# have data till 2019-05 for ERA5dr
stime = '{}-{}-01'.format(b_e_y, mm_e)
assert ds.time[-1] == np.datetime64(stime)
assert ci['baseline_hydro_yr_0'] == b_s_y + 1
assert ci['baseline_hydro_yr_1'] == b_e_y
else:
assert ci['baseline_hydro_yr_0'] == b_s_y + 1
if base == 'ERA5dr':
# do not have full 2019
stime = '{}-{}-01'.format(b_e_y - 1, mm_e)
assert ds.time[-1] == np.datetime64(stime)
assert ci['baseline_hydro_yr_1'] == b_e_y - 1
else:
assert ci['baseline_hydro_yr_1'] == b_e_y
stime = '{}-{}-01'.format(b_e_y, mm_e)
assert ds.time[-1] == np.datetime64(stime)
mb_mod = massbalance.PastMassBalance(
gdir,
mu_star=mu_opt,
input_filesuffix=fsuff,
bias=0,
check_calib_params=False)
years = np.arange(ds.hydro_yr_0, ds.hydro_yr_1 + 1)
mb_ts = mb_mod.get_specific_mb(heights=h,
widths=w,
year=years)
tot_mbs.append(pd.Series(mb_ts))
# check if all ref_hgts are equal
# means that we likely compare same glacier and climate dataset
assert len(np.unique(ref_hgts)) == 1
# concatenate temperature and prcp from different hydromonths
dft = pd.concat(dft, axis=1, keys=np.arange(1, 13))
dfp = pd.concat(dfp, axis=1, keys=np.arange(1, 13))
# Common period
dft_na = dft.dropna().iloc[1:]
dfp_na = dfp.dropna().iloc[1:]
# check if the common period of temperature prcp
# series is equal for all starting hydromonth dates
assert np.all(dft_na.eq(dft_na.iloc[:, 0], axis=0).all(1))
assert np.all(dfp_na.eq(dfp_na.iloc[:, 0], axis=0).all(1))
# mass balance of different years
pd_tot_mbs = pd.concat(tot_mbs, axis=1, keys=np.arange(1, 13))
pd_tot_mbs = pd_tot_mbs.dropna()
# compute correlations
corrs = []
for m in np.arange(1, 12):
# check if correlation between time series of hydro_month =1,
# is high to hydro_month = 2 and so on
corrs.append(pd_tot_mbs.corr().loc[m, m + 1])
# would be better if for hydro_month=12,
# correlation is tested to next year
assert np.mean(corrs) > 0.9
def test_all_at_once(self, gdir):
# Init
exps = ['CRU', 'HISTALP', 'ERA5', 'ERA5L', 'CERA',
'ERA5_daily', 'WFDE5_CRU_daily', 'W5E5_daily',
'WFDE5_CRU_monthly', 'W5E5_monthly']
ref_hgts = []
dft = []
dfp = []
for base in exps:
if base not in ['ERA5_daily', 'WFDE5_CRU_daily',
'W5E5_daily',
'WFDE5_CRU_monthly', 'W5E5_monthly']:
cfg.PARAMS['baseline_climate'] = base
tasks.process_climate_data(gdir, output_filesuffix=base)
elif base == 'ERA5_daily':
cfg.PARAMS['baseline_climate'] = base
process_era5_daily_data(gdir, output_filesuffix=base)
elif base == 'WFDE5_CRU_daily' or base == 'W5E5_daily':
process_w5e5_data(gdir, output_filesuffix=base,
temporal_resol='daily')
elif '_monthly' in base:
# wfde5_cru and w5e5
process_w5e5_data(gdir, output_filesuffix=base,
temporal_resol='monthly')
f = gdir.get_filepath('climate_historical',
filesuffix=base)
with xr.open_dataset(f) as ds:
ref_hgts.append(ds.ref_hgt)
assert ds.ref_pix_dis < 30000
dft.append(ds.temp.to_series())
dfp.append(ds.prcp.to_series())
dft = pd.concat(dft, axis=1, keys=exps)
dfp = pd.concat(dfp, axis=1, keys=exps)
# compare daily mean temperatures of ERA5 and WFDE5
assert dft[['ERA5_daily', 'WFDE5_CRU_daily']].corr().min().min() > 0.95
assert dft[['W5E5_daily', 'WFDE5_CRU_daily']].corr().min().min() > 0.95
# want to compare mean monthly temperatures
# (daily resolution datasets have to be resampled)
dft = dft.resample('MS').mean()
print(dft)
# Common period
#dft = dft.resample(time='1M').mean()
dfy = dft.resample('AS').mean().dropna().iloc[1:]
dfm = dft.groupby(dft.index.month).mean()
assert dfy.corr().min().min() > 0.44 # ERA5L and CERA do not correlate
assert dfm.corr().min().min() > 0.97
dfavg = dfy.describe()
# Correct for hgt
ref_h = ref_hgts[0]
for h, d in zip(ref_hgts, exps):
dfy[d] = dfy[d] - 0.0065 * (ref_h - h)
dfm[d] = dfm[d] - 0.0065 * (ref_h - h)
dfavg_cor = dfy.describe()
# After correction less spread
assert dfavg_cor.loc['mean'].std() < 0.8 * dfavg.loc['mean'].std()
print(dfavg_cor)
# with ERA5_daily and WFDE5_daily, smaller std (from <2.1 -> <1.7)
assert dfavg_cor.loc['mean'].std() < 1.7
# PRECIP
# want to compare summed up monthly precipitation
# (daily resolution datasets, so far only wfde5, have to resample)
dfp = dfp.resample('MS').sum(min_count =1)
# Common period
dfy = dfp.resample('AS').mean().dropna().iloc[1:] * 12
dfm = dfp.groupby(dfp.index.month).mean()
assert dfy.corr().min().min() > 0.5
# monthly prcp of WFDE5 is quite different > 0.8 -> > 0.75
assert dfm.corr().min().min() > 0.73 # 0.8
dfavg = dfy.describe()
assert dfavg.loc['mean'].std() / dfavg.loc['mean'].mean() < 0.28 # %
def test_monthly_glacier_massbalance(gdir):
# TODO: problem with that, monthly and annual MB not exactly the same!!!
# I think there is a problem with SEC_IN_MONTH/SEC_IN_YEAR ...
# do this for all model types
# ONLY TEST it for ERA5dr or ERA5_daily!!!
for climate in ['ERA5dr', 'ERA5_daily']:
for mb_type in ['mb_monthly', 'mb_daily', 'mb_real_daily']:
for grad_type in ['cte', 'var_an_cycle']:
if grad_type == 'var_an_cycle':
fail_err_4 = ((mb_type == 'mb_monthly')
and (climate == 'CRU'))
mu_star_opt = mu_star_opt_var
else:
fail_err_4 = False
mu_star_opt = mu_star_opt_cte
if climate == 'ERA5dr':
cfg.PARAMS['baseline_climate'] = 'ERA5dr'
oggm.shop.ecmwf.process_ecmwf_data(gdir, dataset="ERA5dr")
elif climate == 'ERA5_daily':
process_era5_daily_data(gdir)
cfg.PARAMS['baseline_climate'] = 'ERA5_daily'
else:
tasks.process_climate_data(gdir)
pass
# mb_type ='mb_daily'
fail_err_1 = (mb_type == 'mb_daily') and (climate != 'ERA5dr')
fail_err_2 = ((mb_type == 'mb_monthly')
and (climate == 'ERA5_daily'))
fail_err_3 = ((mb_type == 'mb_real_daily')
and (climate != 'ERA5_daily'))
if fail_err_1 or fail_err_2 or fail_err_3 or fail_err_4:
with pytest.raises(InvalidParamsError):
mb_mod = mb_modules(gdir,
mu_star_opt[mb_type],
mb_type=mb_type,
prcp_fac=pf,
t_solid=0,
t_liq=2,
t_melt=0,
default_grad=-0.0065,
bias=0,
grad_type=grad_type)
else:
# but this is just a test for reproducibility!
mb_mod = mb_modules(gdir,
mu_star_opt[mb_type],
mb_type=mb_type,
prcp_fac=pf,
t_solid=0,
t_liq=2,
t_melt=0,
default_grad=-0.0065,
bias=0,
grad_type=grad_type)
hgts, widths = gdir.get_inversion_flowline_hw()
rho = 900 # ice density
yrp = [1980, 2018]
for i, yr in enumerate(np.arange(yrp[0], yrp[1] + 1)):
my_mon_mb_on_h = 0.
dayofyear = 0
for m in np.arange(12):
yrm = utils.date_to_floatyear(yr, m + 1)
_, dayofmonth = monthrange(yr, m + 1)
dayofyear += dayofmonth
tmp = (mb_mod.get_monthly_mb(hgts, yrm) *
dayofmonth * SEC_IN_DAY * rho)
my_mon_mb_on_h += tmp
my_an_mb_on_h = (mb_mod.get_annual_mb(hgts, yr) *
dayofyear * SEC_IN_DAY * rho)
# these large errors might come from the problematic of
# different amount of days in a year?
# or maybe it just does not fit ...
assert_allclose(np.mean(my_an_mb_on_h -
my_mon_mb_on_h),
0,
atol=100)
def test_present_time_glacier_massbalance(gdir):
# check if area of HUSS flowlines corresponds to the rgi area
h, w = gdir.get_inversion_flowline_hw()
fls = gdir.read_pickle('inversion_flowlines')
assert_allclose(gdir.rgi_area_m2, np.sum(w * gdir.grid.dx * fls[0].dx))
# do this for all model types
# ONLY TEST it for ERA5dr or ERA5_daily!!!
for climate in ['ERA5dr', 'ERA5_daily']:
for mb_type in ['mb_monthly', 'mb_daily', 'mb_real_daily']:
for grad_type in ['cte', 'var_an_cycle']:
if grad_type == 'var_an_cycle':
fail_err_4 = ((mb_type == 'mb_monthly')
and (climate == 'CRU'))
mu_star_opt = mu_star_opt_var
else:
fail_err_4 = False
mu_star_opt = mu_star_opt_cte
if climate == 'ERA5dr':
cfg.PARAMS['baseline_climate'] = 'ERA5dr'
oggm.shop.ecmwf.process_ecmwf_data(gdir, dataset="ERA5dr")
elif climate == 'ERA5_daily':
cfg.PARAMS['baseline_climate'] = 'ERA5_daily'
process_era5_daily_data(gdir)
else:
tasks.process_climate_data(gdir)
pass
fail_err_1 = (mb_type == 'mb_daily') and (climate != 'ERA5dr')
fail_err_2 = ((mb_type == 'mb_monthly')
and (climate == 'ERA5_daily'))
fail_err_3 = ((mb_type == 'mb_real_daily')
and (climate != 'ERA5_daily'))
if fail_err_1 or fail_err_2 or fail_err_3 or fail_err_4:
with pytest.raises(InvalidParamsError):
mb_mod = mb_modules(gdir,
mu_star_opt[mb_type],
mb_type=mb_type,
prcp_fac=pf,
t_solid=0,
t_liq=2,
t_melt=0,
default_grad=-0.0065,
bias=0,
grad_type=grad_type)
else:
# this is just a test for reproducibility!
mb_mod = mb_modules(gdir,
mu_star_opt[mb_type],
mb_type=mb_type,
prcp_fac=pf,
t_solid=0,
t_liq=2,
t_melt=0,
default_grad=-0.0065,
bias=0,
grad_type=grad_type)
mbdf = gdir.get_ref_mb_data()
hgts, widths = gdir.get_inversion_flowline_hw()
tot_mb = []
refmb = []
grads = hgts * 0
for yr, mb in mbdf.iterrows():
refmb.append(mb['ANNUAL_BALANCE'])
mbh = (mb_mod.get_annual_mb(hgts, yr) * SEC_IN_YEAR *
cfg.PARAMS['ice_density'])
grads += mbh
tot_mb.append(np.average(mbh, weights=widths))
grads /= len(tot_mb)
# check if calibrated total mass balance similar
# to observe mass balance time series
assert np.abs(utils.md(tot_mb, refmb)) < 50
