def test_vdr(self, class_case_dir): # Init cfg.initialize() cfg.PARAMS['use_intersects'] = False cfg.PATHS['working_dir'] = class_case_dir cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif') hef_file = get_demo_file('Hintereisferner_RGI5.shp') gdir = workflow.init_glacier_directories(gpd.read_file(hef_file))[0] exps = ['ERA5', 'ERA5dr'] files = [] ref_hgts = [] for base in exps: cfg.PARAMS['baseline_climate'] = base tasks.process_climate_data(gdir, output_filesuffix=base) files.append( gdir.get_filepath('climate_historical', filesuffix=base)) with xr.open_dataset(files[-1]) as ds: ref_hgts.append(ds.ref_hgt) assert ds.ref_pix_dis < 10000 with xr.open_dataset(files[0]) as d1, xr.open_dataset(files[1]) as d2: np.testing.assert_allclose(d1.temp, d2.temp) np.testing.assert_allclose(d1.prcp, d2.prcp) # Fake tests, the plots look plausible np.testing.assert_allclose(d2.gradient.mean(), -0.0058, atol=.001) np.testing.assert_allclose(d2.temp_std.mean(), 3.35, atol=0.1)
def test_ecmwf_workflow(self, class_case_dir): # Init cfg.initialize() cfg.PARAMS['use_intersects'] = False cfg.PATHS['working_dir'] = class_case_dir cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif') hef_file = get_demo_file('Hintereisferner_RGI5.shp') gdir = workflow.init_glacier_directories(gpd.read_file(hef_file))[0] cfg.PARAMS['baseline_climate'] = 'CERA+ERA5L' tasks.process_climate_data(gdir) f_ref = gdir.get_filepath('climate_historical') with xr.open_dataset(f_ref) as his: # Let's do some basic checks ci = gdir.get_climate_info() assert ci['baseline_climate_source'] == 'CERA+ERA5L' assert ci['baseline_hydro_yr_0'] == 1902 assert ci['baseline_hydro_yr_1'] == 2018 cfg.PARAMS['baseline_climate'] = 'CERA|ERA5' tasks.process_climate_data(gdir) f_ref = gdir.get_filepath('climate_historical') with xr.open_dataset(f_ref) as his: # Let's do some basic checks ci = gdir.get_climate_info() assert ci['baseline_climate_source'] == 'CERA|ERA5' assert ci['baseline_hydro_yr_0'] == 1902 assert ci['baseline_hydro_yr_1'] == 2010
def test_all_at_once(self, class_case_dir): # Init cfg.initialize() cfg.PARAMS['use_intersects'] = False cfg.PATHS['working_dir'] = class_case_dir cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif') hef_file = get_demo_file('Hintereisferner_RGI5.shp') gdir = workflow.init_glacier_directories(gpd.read_file(hef_file))[0] exps = ['CRU', 'HISTALP', 'ERA5', 'ERA5L', 'CERA'] files = [] ref_hgts = [] for base in exps: cfg.PARAMS['baseline_climate'] = base tasks.process_climate_data(gdir, output_filesuffix=base) files.append( gdir.get_filepath('climate_historical', filesuffix=base)) with xr.open_dataset(files[-1]) as ds: ref_hgts.append(ds.ref_hgt) assert ds.ref_pix_dis < 30000 # TEMP with xr.open_mfdataset(files, concat_dim=exps) as ds: dft = ds.temp.to_dataframe().unstack().T dft.index = dft.index.levels[1] # Common period dfy = dft.resample('AS').mean().dropna().iloc[1:] dfm = dft.groupby(dft.index.month).mean() assert dfy.corr().min().min() > 0.44 # ERA5L and CERA do no correlate assert dfm.corr().min().min() > 0.97 dfavg = dfy.describe() # Correct for hgt ref_h = ref_hgts[0] for h, d in zip(ref_hgts, exps): dfy[d] = dfy[d] - 0.0065 * (ref_h - h) dfm[d] = dfm[d] - 0.0065 * (ref_h - h) dfavg_cor = dfy.describe() # After correction less spread assert dfavg_cor.loc['mean'].std() < 0.8 * dfavg.loc['mean'].std() assert dfavg_cor.loc['mean'].std() < 2.1 # PRECIP with xr.open_mfdataset(files, concat_dim=exps) as ds: dft = ds.prcp.to_dataframe().unstack().T dft.index = dft.index.levels[1] # Common period dfy = dft.resample('AS').mean().dropna().iloc[1:] * 12 dfm = dft.groupby(dft.index.month).mean() assert dfy.corr().min().min() > 0.5 assert dfm.corr().min().min() > 0.8 dfavg = dfy.describe() assert dfavg.loc['mean'].std() / dfavg.loc['mean'].mean() < 0.25 # %
def test_hydro_month_changes(self, hef_gdir): # test for HEF if applying different hydro_months does the right thing # check if mb of neighbouring hydro_months correlate # do this for different climate scenarios # maybe there is already somewhere an overview or a better way to get # these dates, but I did not find it base_data_time = { 'CRU': { 'start_year': 1901, 'end_year': 2014 }, 'ERA5': { 'start_year': 1979, 'end_year': 2018 }, 'ERA5dr': { 'start_year': 1979, 'end_year': 2019 }, 'HISTALP': { 'start_year': 1850, 'end_year': 2014 }, 'CERA': { 'start_year': 1901, 'end_year': 2010 }, 'ERA5L': { 'start_year': 1981, 'end_year': 2018 } } gdir = hef_gdir oggm.core.flowline.init_present_time_glacier(gdir) mb_mod = oggm.core.massbalance.PastMassBalance(gdir) h, w = gdir.get_inversion_flowline_hw() exps = ['ERA5dr', 'CRU', 'HISTALP', 'ERA5', 'ERA5L', 'CERA'] for base in exps: # this does not need to be the best one, # just for comparison between different hydro months mu_opt = 213.54 files = [] ref_hgts = [] dft = [] dfp = [] tot_mbs = [] cfg.PARAMS['baseline_climate'] = base for m in np.arange(1, 13): cfg.PARAMS['hydro_month_nh'] = m fsuff = '_{}_{}'.format(base, m) tasks.process_climate_data(gdir, output_filesuffix=fsuff) files.append( gdir.get_filepath('climate_historical', filesuffix=fsuff)) with xr.open_dataset(files[-1]) as ds: ref_hgts.append(ds.ref_hgt) dft.append(ds.temp.to_series()) dfp.append(ds.prcp.to_series()) ci = gdir.get_climate_info(input_filesuffix=fsuff) # check if the right climate source is used assert base in ci['baseline_climate_source'] mm = str(m) if m > 9 else str(0) + str(m) mm_e = str(m - 1) if (m - 1) > 9 else str(0) + str(m - 1) b_s_y = base_data_time[base]['start_year'] b_e_y = base_data_time[base]['end_year'] stime = '{}-{}-01'.format(b_s_y, mm) assert ds.time[0] == np.datetime64(stime) if m == 1: assert ci['baseline_hydro_yr_0'] == b_s_y if base == 'ERA5dr': # do not have full 2019 assert ci['baseline_hydro_yr_1'] == b_e_y - 1 else: assert ci['baseline_hydro_yr_1'] == b_e_y elif m < 7 and base == 'ERA5dr': # have data till 2019-05 for ERA5dr stime = '{}-{}-01'.format(b_e_y, mm_e) assert ds.time[-1] == np.datetime64(stime) assert ci['baseline_hydro_yr_0'] == b_s_y + 1 assert ci['baseline_hydro_yr_1'] == b_e_y else: assert ci['baseline_hydro_yr_0'] == b_s_y + 1 if base == 'ERA5dr': # do not have full 2019 stime = '{}-{}-01'.format(b_e_y - 1, mm_e) assert ds.time[-1] == np.datetime64(stime) assert ci['baseline_hydro_yr_1'] == b_e_y - 1 else: assert ci['baseline_hydro_yr_1'] == b_e_y stime = '{}-{}-01'.format(b_e_y, mm_e) assert ds.time[-1] == np.datetime64(stime) mb_mod = massbalance.PastMassBalance( gdir, mu_star=mu_opt, input_filesuffix=fsuff, bias=0, check_calib_params=False) years = np.arange(ds.hydro_yr_0, ds.hydro_yr_1 + 1) mb_ts = mb_mod.get_specific_mb(heights=h, widths=w, year=years) tot_mbs.append(pd.Series(mb_ts)) # check if all ref_hgts are equal # means that we likely compare same glacier and climate dataset assert len(np.unique(ref_hgts)) == 1 # concatenate temperature and prcp from different hydromonths dft = pd.concat(dft, axis=1, keys=np.arange(1, 13)) dfp = pd.concat(dfp, axis=1, keys=np.arange(1, 13)) # Common period dft_na = dft.dropna().iloc[1:] dfp_na = dfp.dropna().iloc[1:] # check if the common period of temperature prcp # series is equal for all starting hydromonth dates assert np.all(dft_na.eq(dft_na.iloc[:, 0], axis=0).all(1)) assert np.all(dfp_na.eq(dfp_na.iloc[:, 0], axis=0).all(1)) # mass balance of different years pd_tot_mbs = pd.concat(tot_mbs, axis=1, keys=np.arange(1, 13)) pd_tot_mbs = pd_tot_mbs.dropna() # compute correlations corrs = [] for m in np.arange(1, 12): # check if correlation between time series of hydro_month =1, # is high to hydro_month = 2 and so on corrs.append(pd_tot_mbs.corr().loc[m, m + 1]) # would be better if for hydro_month=12, # correlation is tested to next year assert np.mean(corrs) > 0.9
def test_all_at_once(self, gdir): # Init exps = ['CRU', 'HISTALP', 'ERA5', 'ERA5L', 'CERA', 'ERA5_daily', 'WFDE5_CRU_daily', 'W5E5_daily', 'WFDE5_CRU_monthly', 'W5E5_monthly'] ref_hgts = [] dft = [] dfp = [] for base in exps: if base not in ['ERA5_daily', 'WFDE5_CRU_daily', 'W5E5_daily', 'WFDE5_CRU_monthly', 'W5E5_monthly']: cfg.PARAMS['baseline_climate'] = base tasks.process_climate_data(gdir, output_filesuffix=base) elif base == 'ERA5_daily': cfg.PARAMS['baseline_climate'] = base process_era5_daily_data(gdir, output_filesuffix=base) elif base == 'WFDE5_CRU_daily' or base == 'W5E5_daily': process_w5e5_data(gdir, output_filesuffix=base, temporal_resol='daily') elif '_monthly' in base: # wfde5_cru and w5e5 process_w5e5_data(gdir, output_filesuffix=base, temporal_resol='monthly') f = gdir.get_filepath('climate_historical', filesuffix=base) with xr.open_dataset(f) as ds: ref_hgts.append(ds.ref_hgt) assert ds.ref_pix_dis < 30000 dft.append(ds.temp.to_series()) dfp.append(ds.prcp.to_series()) dft = pd.concat(dft, axis=1, keys=exps) dfp = pd.concat(dfp, axis=1, keys=exps) # compare daily mean temperatures of ERA5 and WFDE5 assert dft[['ERA5_daily', 'WFDE5_CRU_daily']].corr().min().min() > 0.95 assert dft[['W5E5_daily', 'WFDE5_CRU_daily']].corr().min().min() > 0.95 # want to compare mean monthly temperatures # (daily resolution datasets have to be resampled) dft = dft.resample('MS').mean() print(dft) # Common period #dft = dft.resample(time='1M').mean() dfy = dft.resample('AS').mean().dropna().iloc[1:] dfm = dft.groupby(dft.index.month).mean() assert dfy.corr().min().min() > 0.44 # ERA5L and CERA do not correlate assert dfm.corr().min().min() > 0.97 dfavg = dfy.describe() # Correct for hgt ref_h = ref_hgts[0] for h, d in zip(ref_hgts, exps): dfy[d] = dfy[d] - 0.0065 * (ref_h - h) dfm[d] = dfm[d] - 0.0065 * (ref_h - h) dfavg_cor = dfy.describe() # After correction less spread assert dfavg_cor.loc['mean'].std() < 0.8 * dfavg.loc['mean'].std() print(dfavg_cor) # with ERA5_daily and WFDE5_daily, smaller std (from <2.1 -> <1.7) assert dfavg_cor.loc['mean'].std() < 1.7 # PRECIP # want to compare summed up monthly precipitation # (daily resolution datasets, so far only wfde5, have to resample) dfp = dfp.resample('MS').sum(min_count =1) # Common period dfy = dfp.resample('AS').mean().dropna().iloc[1:] * 12 dfm = dfp.groupby(dfp.index.month).mean() assert dfy.corr().min().min() > 0.5 # monthly prcp of WFDE5 is quite different > 0.8 -> > 0.75 assert dfm.corr().min().min() > 0.73 # 0.8 dfavg = dfy.describe() assert dfavg.loc['mean'].std() / dfavg.loc['mean'].mean() < 0.28 # %
def test_monthly_glacier_massbalance(gdir): # TODO: problem with that, monthly and annual MB not exactly the same!!! # I think there is a problem with SEC_IN_MONTH/SEC_IN_YEAR ... # do this for all model types # ONLY TEST it for ERA5dr or ERA5_daily!!! for climate in ['ERA5dr', 'ERA5_daily']: for mb_type in ['mb_monthly', 'mb_daily', 'mb_real_daily']: for grad_type in ['cte', 'var_an_cycle']: if grad_type == 'var_an_cycle': fail_err_4 = ((mb_type == 'mb_monthly') and (climate == 'CRU')) mu_star_opt = mu_star_opt_var else: fail_err_4 = False mu_star_opt = mu_star_opt_cte if climate == 'ERA5dr': cfg.PARAMS['baseline_climate'] = 'ERA5dr' oggm.shop.ecmwf.process_ecmwf_data(gdir, dataset="ERA5dr") elif climate == 'ERA5_daily': process_era5_daily_data(gdir) cfg.PARAMS['baseline_climate'] = 'ERA5_daily' else: tasks.process_climate_data(gdir) pass # mb_type ='mb_daily' fail_err_1 = (mb_type == 'mb_daily') and (climate != 'ERA5dr') fail_err_2 = ((mb_type == 'mb_monthly') and (climate == 'ERA5_daily')) fail_err_3 = ((mb_type == 'mb_real_daily') and (climate != 'ERA5_daily')) if fail_err_1 or fail_err_2 or fail_err_3 or fail_err_4: with pytest.raises(InvalidParamsError): mb_mod = mb_modules(gdir, mu_star_opt[mb_type], mb_type=mb_type, prcp_fac=pf, t_solid=0, t_liq=2, t_melt=0, default_grad=-0.0065, bias=0, grad_type=grad_type) else: # but this is just a test for reproducibility! mb_mod = mb_modules(gdir, mu_star_opt[mb_type], mb_type=mb_type, prcp_fac=pf, t_solid=0, t_liq=2, t_melt=0, default_grad=-0.0065, bias=0, grad_type=grad_type) hgts, widths = gdir.get_inversion_flowline_hw() rho = 900 # ice density yrp = [1980, 2018] for i, yr in enumerate(np.arange(yrp[0], yrp[1] + 1)): my_mon_mb_on_h = 0. dayofyear = 0 for m in np.arange(12): yrm = utils.date_to_floatyear(yr, m + 1) _, dayofmonth = monthrange(yr, m + 1) dayofyear += dayofmonth tmp = (mb_mod.get_monthly_mb(hgts, yrm) * dayofmonth * SEC_IN_DAY * rho) my_mon_mb_on_h += tmp my_an_mb_on_h = (mb_mod.get_annual_mb(hgts, yr) * dayofyear * SEC_IN_DAY * rho) # these large errors might come from the problematic of # different amount of days in a year? # or maybe it just does not fit ... assert_allclose(np.mean(my_an_mb_on_h - my_mon_mb_on_h), 0, atol=100)
def test_present_time_glacier_massbalance(gdir): # check if area of HUSS flowlines corresponds to the rgi area h, w = gdir.get_inversion_flowline_hw() fls = gdir.read_pickle('inversion_flowlines') assert_allclose(gdir.rgi_area_m2, np.sum(w * gdir.grid.dx * fls[0].dx)) # do this for all model types # ONLY TEST it for ERA5dr or ERA5_daily!!! for climate in ['ERA5dr', 'ERA5_daily']: for mb_type in ['mb_monthly', 'mb_daily', 'mb_real_daily']: for grad_type in ['cte', 'var_an_cycle']: if grad_type == 'var_an_cycle': fail_err_4 = ((mb_type == 'mb_monthly') and (climate == 'CRU')) mu_star_opt = mu_star_opt_var else: fail_err_4 = False mu_star_opt = mu_star_opt_cte if climate == 'ERA5dr': cfg.PARAMS['baseline_climate'] = 'ERA5dr' oggm.shop.ecmwf.process_ecmwf_data(gdir, dataset="ERA5dr") elif climate == 'ERA5_daily': cfg.PARAMS['baseline_climate'] = 'ERA5_daily' process_era5_daily_data(gdir) else: tasks.process_climate_data(gdir) pass fail_err_1 = (mb_type == 'mb_daily') and (climate != 'ERA5dr') fail_err_2 = ((mb_type == 'mb_monthly') and (climate == 'ERA5_daily')) fail_err_3 = ((mb_type == 'mb_real_daily') and (climate != 'ERA5_daily')) if fail_err_1 or fail_err_2 or fail_err_3 or fail_err_4: with pytest.raises(InvalidParamsError): mb_mod = mb_modules(gdir, mu_star_opt[mb_type], mb_type=mb_type, prcp_fac=pf, t_solid=0, t_liq=2, t_melt=0, default_grad=-0.0065, bias=0, grad_type=grad_type) else: # this is just a test for reproducibility! mb_mod = mb_modules(gdir, mu_star_opt[mb_type], mb_type=mb_type, prcp_fac=pf, t_solid=0, t_liq=2, t_melt=0, default_grad=-0.0065, bias=0, grad_type=grad_type) mbdf = gdir.get_ref_mb_data() hgts, widths = gdir.get_inversion_flowline_hw() tot_mb = [] refmb = [] grads = hgts * 0 for yr, mb in mbdf.iterrows(): refmb.append(mb['ANNUAL_BALANCE']) mbh = (mb_mod.get_annual_mb(hgts, yr) * SEC_IN_YEAR * cfg.PARAMS['ice_density']) grads += mbh tot_mb.append(np.average(mbh, weights=widths)) grads /= len(tot_mb) # check if calibrated total mass balance similar # to observe mass balance time series assert np.abs(utils.md(tot_mb, refmb)) < 50