def parse_dt_per_dt(gdir, monthly=False):
"""Local climate change signal for this glacier added to the diagnostics
"""
# Use xarray to read the data
lon = gdir.cenlon + 360 if gdir.cenlon < 0 else gdir.cenlon
lat = gdir.cenlat
if monthly:
convert = list
tf = DT_PER_DT_MON_FILE
tp = DP_PER_DT_MON_FILE
else:
convert = float
tf = DT_PER_DT_FILE
tp = DP_PER_DT_FILE
with xr.open_dataset(utils.file_downloader(tf)) as ds:
assert ds.longitude.min() >= 0
ds = ds.sel(longitude=lon, latitude=lat, method='nearest')
gdir.add_to_diagnostics('magicc_dt_per_dt',
convert(ds['cmip_avg'].data))
with xr.open_dataset(utils.file_downloader(tp)) as ds:
assert ds.longitude.min() >= 0
ds = ds.sel(longitude=lon, latitude=lat, method='nearest')
gdir.add_to_diagnostics('magicc_dp_per_dt',
convert(ds['cmip_avg'].data))
def download_ref_tstars(base_url=None):
"""Downloads and copies the reference list of t* to the working directory.
Example url:
https://cluster.klima.uni-bremen.de/~oggm/ref_mb_params/oggm_v1.4/RGIV62/CRU/centerlines/qc3/pcp2.5
Parameters
----------
base_url : str
url of the params file.
"""
shutil.copyfile(utils.file_downloader(base_url + '/ref_tstars.csv'),
os.path.join(cfg.PATHS['working_dir'], 'ref_tstars.csv'))
shutil.copyfile(utils.file_downloader(base_url + '/ref_tstars_params.json'),
os.path.join(cfg.PATHS['working_dir'], 'ref_tstars_params.json'))
def get_geodetic_tile(gdir, source='Hugonnet'):
""" Download geodetic raster file
Parameter
--------
gdir : :oggm.GlacierDirectory
source : str: which geodetic mass balance data will be used
should be in ['Hugonnet', 'Shean'], default is 'Hugonnet'
return
------
n_path : str: the path of downloaded mass balance raster
"""
if source == 'Hugonnet':
url = hugonnet_url
grid = gdir.grid
minlon, maxlon, minlat, maxlat = grid.extent_in_crs(crs=salem.wgs84)
minlon = int(minlon)
maxlon = int(maxlon)
minlat = int(minlat)
maxlat = int(maxlat)
lons = list(set([minlon, maxlon]))
lats = list(set([minlat, maxlat]))
fpaths = []
for lon in lons:
for lat in lats:
ns_hsp = 'N' if lon >= 0 else 'S'
we_hsp = 'E' if lat >= 0 else 'W'
tile_name = "{}{}{}{:0>3d}".format(ns_hsp, abs(lat), we_hsp,
abs(lon))
print(tile_name)
path = utils.file_downloader(os.path.join(url, tile_name))
fpath = os.path.join(
os.path.split(path)[0],
f'{tile_name}_2000-01-01_2020-01-01_dhdt.tif')
if not os.path.exists(fpath):
shutil.copy(path, fpath)
fpaths.append(fpath)
elif source == 'Shean':
url = shean_url
fpath = utils.file_downloader(url)
fpaths = [fpath]
else:
raise ValueError(
f"Meet unexcepted geodetic mass balance source: {source}!")
return fpaths
def gdir_from_prepro(entity,
from_prepro_level=None,
prepro_border=None,
prepro_rgi_version=None,
check_demo_glacier=False):
if prepro_border is None:
prepro_border = int(cfg.PARAMS['border'])
if prepro_rgi_version is None:
prepro_rgi_version = cfg.PARAMS['rgi_version']
try:
rid = entity.RGIId
except AttributeError:
rid = entity
demo_url = False
if check_demo_glacier:
demo_id = utils.demo_glacier_id(rid)
if demo_id is not None:
rid = demo_id
entity = demo_id
demo_url = True
tar_url = utils.prepro_gdir_url(prepro_rgi_version,
rid,
prepro_border,
from_prepro_level,
demo_url=demo_url)
tar_base = utils.file_downloader(tar_url)
if tar_base is None:
raise RuntimeError('Could not find file at ' + tar_url)
from_tar = os.path.join(tar_base.replace('.tar', ''), rid + '.tar.gz')
return oggm.GlacierDirectory(entity, from_tar=from_tar)
def get_ecmwf_file(dataset='ERA5', var=None):
"""Returns a path to the desired ECMWF baseline climate file.
If the file is not present, download it.
Parameters
----------
dataset : str
'ERA5', 'ERA5L', 'CERA'
var : str
'inv' for invariant
'tmp' for temperature
'pre' for precipitation
Returns
-------
str
path to the file
"""
# Be sure input makes sense
if dataset not in BASENAMES.keys():
raise InvalidParamsError('ECMWF dataset {} not '
'in {}'.format(dataset, BASENAMES.keys()))
if var not in BASENAMES[dataset].keys():
raise InvalidParamsError('ECMWF variable {} not '
'in {}'.format(var,
BASENAMES[dataset].keys()))
# File to look for
return utils.file_downloader(ECMWF_SERVER + BASENAMES[dataset][var])
def get_cru_file(var=None):
"""Returns a path to the desired CRU baseline climate file.
If the file is not present, download it.
Parameters
----------
var : str
'tmp' for temperature
'pre' for precipitation
Returns
-------
str
path to the CRU file
"""
# Be sure input makes sense
if var not in ['tmp', 'pre']:
raise InvalidParamsError('CRU variable {} does not exist!'.format(var))
# Download
cru_filename = CRU_BASE.format(var)
cru_url = CRU_SERVER + '{}/'.format(var) + cru_filename + '.gz'
return utils.file_extractor(utils.file_downloader(cru_url))
def get_histalp_file(var=None):
"""Returns a path to the desired HISTALP baseline climate file.
If the file is not present, download it.
Parameters
----------
var : str
'tmp' for temperature
'pre' for precipitation
Returns
-------
str
path to the CRU file
"""
# Be sure input makes sense
if var not in ['tmp', 'pre']:
raise InvalidParamsError('HISTALP variable {} '
'does not exist!'.format(var))
# File to look for
if var == 'tmp':
bname = 'HISTALP_temperature_1780-2014.nc'
else:
bname = 'HISTALP_precipitation_all_abs_1801-2014.nc'
h_url = HISTALP_SERVER + bname + '.bz2'
return utils.file_extractor(utils.file_downloader(h_url))
def add_consensus_thickness(gdir, base_url=None):
"""Add the consensus thickness estimate to the gridded_data file.
varname: consensus_ice_thickness
Parameters
----------
gdir ::py:class:`oggm.GlacierDirectory`
the glacier directory to process
base_url : str
where to find the thickness data. Default is
https://cluster.klima.uni-bremen.de/~fmaussion/icevol/composite
"""
if base_url is None:
base_url = default_base_url
if not base_url.endswith('/'):
base_url += '/'
rgi_str = gdir.rgi_id
rgi_reg_str = rgi_str[:8]
url = base_url + rgi_reg_str + '/' + rgi_str + '_thickness.tif'
input_file = utils.file_downloader(url)
dsb = salem.GeoTiff(input_file)
thick = utils.clip_min(dsb.get_vardata(), 0)
in_volume = thick.sum() * dsb.grid.dx**2
thick = gdir.grid.map_gridded_data(thick, dsb.grid, interp='linear')
# Correct for volume
thick = utils.clip_min(thick.filled(0), 0)
out_volume = thick.sum() * gdir.grid.dx**2
if out_volume > 0:
thick *= in_volume / out_volume
# We mask zero ice as nodata
thick = np.where(thick == 0, np.NaN, thick)
# Write
with utils.ncDataset(gdir.get_filepath('gridded_data'), 'a') as nc:
vn = 'consensus_ice_thickness'
if vn in nc.variables:
v = nc.variables[vn]
else:
v = nc.createVariable(vn, 'f4', (
'y',
'x',
), zlib=True)
v.units = 'm'
ln = 'Ice thickness from the consensus estimate'
v.long_name = ln
v.base_url = base_url
v[:] = thick
def region_grid(reg):
global region_grids
if reg not in region_grids:
with utils.get_lock():
fp = utils.file_downloader(region_files[reg]['vx'])
ds = salem.GeoTiff(fp)
region_grids[reg] = ds.grid
return region_grids[reg]
def test_ensemble_workflow(self, case_dir):
cfg.initialize()
cfg.PARAMS['prcp_scaling_factor'] = 1.6
cfg.PATHS['working_dir'] = case_dir
cfg.PARAMS['use_multiprocessing'] = True
cfg.PARAMS['store_diagnostic_variables'] = ['volume', 'area']
# Go - get the pre-processed glacier directories
rgi_ids = ['RGI60-03.04384']
gdirs = workflow.init_glacier_directories(
rgi_ids,
from_prepro_level=5,
prepro_base_url=prepro_base_url,
prepro_border=80,
prepro_rgi_version='62')
workflow.execute_entity_task(parse_dt_per_dt, gdirs)
exp = 'netzero_py2050_fac1.0_decr0.3'
magicc_file = magicc_dir + exp + '.nc'
with xr.open_dataset(utils.file_downloader(magicc_file),
decode_times=False) as ds:
ds = ds.load()
odf = pd.DataFrame()
for q in ds['quantile'].data:
df = ds.global_mean_temperature.sel(quantile=q).to_series()
workflow.execute_entity_task(tasks.run_with_hydro,
gdirs,
run_task=run_from_magicc_data,
magicc_ts=df,
init_model_filesuffix='_historical',
output_filesuffix='_{:.2f}'.format(q))
ods = utils.compile_run_output(gdirs,
filesuffix='_{:.2f}'.format(q))
odf[q] = ods.volume.isel(rgi_id=0).to_series()
odf = odf / odf.iloc[0, 0]
assert odf.loc[2150].std() > 0.05
assert_allclose(odf.loc[2010].std(), 0, atol=1e-3)
assert_allclose(odf.loc[2300].std(), 0, atol=1e-2)
if DO_PLOT:
odf.plot(title='Ensemble stuff')
plt.show()
def histalp():
# If HISTALP run, only region 11 is valid:
mbcfg.PARAMS['region'] == '11'
url = 'https://cluster.klima.uni-bremen.de/~mdusch/' + \
'histalp_merged_full_1850.nc.tar.gz'
histalptar = utils.file_downloader(url)
import tarfile
with tarfile.open(histalptar, "r") as tar:
tar.extractall(path=os.path.split(histalptar)[0])
mbcfg.PATHS['histalpfile'] = os.path.join(
os.path.split(histalptar)[0],
tar.getmembers()[0].name)
if not os.path.exists(mbcfg.PATHS['histalpfile']):
raise RuntimeError('Histalpfile not found')
def test_varying_bias_workflow(self, case_dir):
cfg.initialize()
cfg.PARAMS['prcp_scaling_factor'] = 1.6
cfg.PATHS['working_dir'] = case_dir
cfg.PARAMS['use_multiprocessing'] = True
# Go - get the pre-processed glacier directories
gdirs = workflow.init_glacier_directories(
['RGI60-11.00897'],
from_prepro_level=5,
prepro_base_url=prepro_base_url,
prepro_border=80,
prepro_rgi_version='62')
workflow.execute_entity_task(parse_dt_per_dt, gdirs)
gdir = gdirs[0]
exp = 'netzero_py2020_fac1.0_decr0.3'
magicc_file = magicc_dir + exp + '.nc'
with xr.open_dataset(utils.file_downloader(magicc_file),
decode_times=False) as ds:
ds = ds.load()
df = ds['ens_avg'].to_dataframe()
run_from_magicc_data(gdir,
magicc_ts=df['ens_avg'],
init_model_filesuffix='_historical',
output_filesuffix='_' + exp)
with xr.open_dataset(
gdir.get_filepath('model_diagnostics',
filesuffix='_' + exp)) as ds:
df['vol'] = ds.volume_m3.to_series()
assert ds.time[0] == 2009
assert ds.time[-1] == 2301
assert ds.volume_m3.isel(time=0) > ds.volume_m3.isel(time=-1)
assert ds.volume_m3.min() < ds.volume_m3.isel(time=-1)
# Pre-download other files which will be needed later
_ = utils.get_cru_file(var='tmp')
_ = utils.get_cru_file(var='pre')
# Copy the precalibrated tstar file
# ---------------------------------
# Note that to be exact, this procedure can only be applied if the run
# parameters don't change between the calibration and the run.
# After testing, it appears that changing the 'border' parameter won't affect
# the results much (expectedly), so that it's ok to change it. All the rest
# (e.g. smoothing, dx, prcp factor...) should imply a re-calibration
mbf = 'https://dl.dropboxusercontent.com/u/20930277/ref_tstars_no_tidewater.csv'
mbf = utils.file_downloader(mbf)
shutil.copyfile(mbf, os.path.join(WORKING_DIR, 'ref_tstars.csv'))
# Copy the RGI file
# -----------------
# Download RGI files
rgi_dir = utils.get_rgi_dir()
rgi_shp = list(glob(os.path.join(rgi_dir, "*", rgi_reg+ '_rgi50_*.shp')))
assert len(rgi_shp) == 1
rgidf = salem.read_shapefile(rgi_shp[0], cached=True)
# Sort for more efficient parallel computing
rgidf = rgidf.sort_values('Area', ascending=False)
def process_lmr_data(gdir,
fpath_temp=None,
fpath_precip=None,
year_range=('1951', '1980'),
filesuffix='',
**kwargs):
"""Read, process and store the Last Millennium Reanalysis (LMR) data for this glacier.
LMR data: https://atmos.washington.edu/~hakim/lmr/LMRv2/
LMR data is annualised in anomaly format relative to 1951-1980. We
create synthetic timeseries from the reference data.
It stores the data in a format that can be used by the OGGM mass balance
model and in the glacier directory.
Parameters
----------
fpath_temp : str
path to the temp file (default: LMR v2.1 from server above)
fpath_precip : str
path to the precip file (default: LMR v2.1 from server above)
year_range : tuple of str
the year range for which you want to compute the anomalies. Default
for LMR is `('1951', '1980')`
filesuffix : str
append a suffix to the filename (useful for ensemble experiments).
**kwargs: any kwarg to be passed to ref:`process_gcm_data`
"""
# Get the path of GCM temperature & precipitation data
base_url = 'https://atmos.washington.edu/%7Ehakim/lmr/LMRv2/'
if fpath_temp is None:
with utils.get_lock():
fpath_temp = utils.file_downloader(
base_url + 'air_MCruns_ensemble_mean_LMRv2.1.nc')
if fpath_precip is None:
with utils.get_lock():
fpath_precip = utils.file_downloader(
base_url + 'prate_MCruns_ensemble_mean_LMRv2.1.nc')
# Glacier location
glon = gdir.cenlon
glat = gdir.cenlat
# Read the GCM files
with xr.open_dataset(fpath_temp, use_cftime=True) as tempds, \
xr.open_dataset(fpath_precip, use_cftime=True) as precipds:
# Check longitude conventions
if tempds.lon.min() >= 0 and glon <= 0:
glon += 360
# Take the closest to the glacier
# Should we consider GCM interpolation?
temp = tempds.air.sel(lat=glat, lon=glon, method='nearest')
precip = precipds.prate.sel(lat=glat, lon=glon, method='nearest')
# Currently we just take the mean of the ensemble, although
# this is probably not advised. The GCM climate will correct
# anyways
temp = temp.mean(dim='MCrun')
precip = precip.mean(dim='MCrun')
# Precip unit is kg/m^2/s we convert to mm month since we apply the anomaly after
precip = precip * 30.5 * (60 * 60 * 24)
# Back to [-180, 180] for OGGM
temp.lon.values = temp.lon if temp.lon <= 180 else temp.lon - 360
precip.lon.values = precip.lon if precip.lon <= 180 else precip.lon - 360
# OK now we have to turn these annual timeseries in monthly data
# We take the ref climate
fpath = gdir.get_filepath('climate_historical')
with xr.open_dataset(fpath) as ds_ref:
ds_ref = ds_ref.sel(time=slice(*year_range))
loc_tmp = ds_ref.temp.groupby('time.month').mean()
loc_pre = ds_ref.prcp.groupby('time.month').mean()
# Make time coord
t = np.cumsum([31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] *
len(temp))
t = cftime.num2date(np.append([0], t[:-1]),
'days since 0000-01-01 00:00:00',
calendar='noleap')
temp = xr.DataArray(
(loc_tmp.data + temp.data[:, np.newaxis]).flatten(),
coords={
'time': t,
'lon': temp.lon,
'lat': temp.lat
},
dims=('time', ))
# For precip the std dev is very small - lets keep it as is for now but
# this is a bit ridiculous. We clip to zero here to be sure
precip = utils.clip_min(
(loc_pre.data + precip.data[:, np.newaxis]).flatten(), 0)
precip = xr.DataArray(precip,
dims=('time', ),
coords={
'time': t,
'lon': temp.lon,
'lat': temp.lat
})
process_gcm_data(gdir,
filesuffix=filesuffix,
prcp=precip,
temp=temp,
year_range=year_range,
calendar='noleap',
source='lmr',
**kwargs)
def test_varying_bias_mb_model(self, case_dir):
cfg.initialize()
cfg.PARAMS['prcp_scaling_factor'] = 1.6
cfg.PATHS['working_dir'] = case_dir
cfg.PARAMS['use_multiprocessing'] = True
# Go - get the pre-processed glacier directories
for rid in ['RGI60-03.04384', 'RGI60-11.00897', 'RGI60-16.02207']:
gdirs = workflow.init_glacier_directories(
[rid],
from_prepro_level=5,
prepro_base_url=prepro_base_url,
prepro_border=80,
prepro_rgi_version='62')
workflow.execute_entity_task(parse_dt_per_dt, gdirs)
gdir = gdirs[0]
exp = 'netzero_py2020_fac1.0_decr0.3'
magicc_file = magicc_dir + exp + '.nc'
with xr.open_dataset(utils.file_downloader(magicc_file),
decode_times=False) as ds:
ds = ds.load()
df = ds['ens_avg'].to_dataframe()
dt_per_dt = gdir.get_diagnostics()['magicc_dt_per_dt']
dp_per_dt = gdir.get_diagnostics()['magicc_dp_per_dt']
fls = gdir.read_pickle('model_flowlines')
y0 = 2014
hs = 5
mbc = massbalance.ConstantMassBalance(gdir, y0=y0, halfsize=hs)
mb_ref = massbalance.PastMassBalance(gdir)
years = np.arange(1980, 2020, dtype=np.float64)
df['mb_ref'] = pd.Series(index=years,
data=mb_ref.get_specific_mb(fls=fls,
year=years))
mb = MagiccMassBalance(gdir,
y0=y0,
halfsize=hs,
magicc_ts=df['ens_avg'],
dt_per_dt=dt_per_dt)
df['mb'] = mb.get_specific_mb(fls=fls, year=df.index)
mb = MagiccMassBalance(gdir,
y0=y0,
halfsize=hs,
magicc_ts=df['ens_avg'],
dt_per_dt=dt_per_dt,
dp_per_dt=dp_per_dt)
df['mb_wp'] = mb.get_specific_mb(fls=fls, year=df.index)
mb = MagiccMassBalance(gdir,
y0=y0,
halfsize=hs,
magicc_ts=df['ens_avg'])
df['mb_nodt'] = mb.get_specific_mb(fls=fls, year=df.index)
assert_allclose(df.loc[y0 - hs:y0 + hs]['mb'].mean(),
mbc.get_specific_mb(fls=fls),
rtol=5e-3)
assert_allclose(df.loc[y0 - hs:y0 + hs]['mb_ref'].mean(),
df.loc[y0 - hs:y0 + hs]['mb'].mean(),
atol=1e-2)
assert_allclose(df.loc[y0 - hs:y0 + hs]['mb_ref'].mean(),
df.loc[y0 - hs:y0 + hs]['mb_wp'].mean(),
atol=1e-2)
assert_allclose(df.loc[y0 - hs:y0 + hs]['mb_ref'].mean(),
df.loc[y0 - hs:y0 + hs]['mb_nodt'].mean(),
atol=1e-2)
assert_allclose(df[['ens_avg', 'mb']].corr().min().min(),
-1,
rtol=1e-2)
assert_allclose(df[['ens_avg', 'mb_nodt']].corr().min().min(),
-1,
rtol=1e-2)
assert_allclose(df[['mb_wp', 'mb']].corr().min().min(),
1,
rtol=1e-2)
assert_allclose(df['mb'], df['mb_wp'], atol=115)
if DO_PLOT:
plt.figure()
df[['mb_ref', 'mb', 'mb_wp',
'mb_nodt']].loc[2000:2025].plot(title=rid)
plt.figure()
df[['mb_ref', 'mb', 'mb_wp',
'mb_nodt']].loc[2000:].plot(title=rid)
plt.show()
def test_scaling_monhly_vs_annual(self, case_dir):
cfg.initialize()
cfg.PARAMS['prcp_scaling_factor'] = 1.6
cfg.PATHS['working_dir'] = case_dir
cfg.PARAMS['use_multiprocessing'] = True
cfg.PARAMS['continue_on_error'] = False
cfg.PARAMS['store_diagnostic_variables'] = ALL_DIAGS
# Go - get the pre-processed glacier directories
rgi_ids = ['RGI60-11.00897', 'RGI60-14.06794']
gdirs = workflow.init_glacier_directories(
rgi_ids,
from_prepro_level=5,
prepro_base_url=prepro_base_url,
prepro_border=80,
prepro_rgi_version='62')
exp = 'netzero_py2050_fac1.0_decr0.3'
magicc_file = magicc_dir + exp + '.nc'
with xr.open_dataset(utils.file_downloader(magicc_file),
decode_times=False) as ds:
ds = ds.load()
df = ds['ens_avg'].to_dataframe()
# Default is annual
workflow.execute_entity_task(parse_dt_per_dt, gdirs)
workflow.execute_entity_task(tasks.run_with_hydro,
gdirs,
store_monthly_hydro=True,
run_task=run_from_magicc_data,
magicc_ts=df['ens_avg'],
init_model_filesuffix='_historical',
output_filesuffix='_annual')
# Monthly
workflow.execute_entity_task(parse_dt_per_dt, gdirs, monthly=True)
workflow.execute_entity_task(tasks.run_with_hydro,
gdirs,
store_monthly_hydro=True,
run_task=run_from_magicc_data,
magicc_ts=df['ens_avg'],
init_model_filesuffix='_historical',
output_filesuffix='_monthly')
for gdir in gdirs:
with xr.open_dataset(
gdir.get_filepath('model_diagnostics',
filesuffix='_annual')) as ds:
sel_vars = [
v for v in ds.variables if 'month_2d' not in ds[v].dims
]
odf_a = ds[sel_vars].to_dataframe()
sel_vars = [
v for v in ds.variables if 'month_2d' in ds[v].dims
]
odf_ma = ds[sel_vars].isel(time=slice(0, 10)).mean(
dim='time').to_dataframe()
odf_ma.columns = [
c.replace('_monthly', '') for c in odf_ma.columns
]
odf_ma_e = ds[sel_vars].isel(time=slice(-10, -1)).mean(
dim='time').to_dataframe()
odf_ma_e.columns = [
c.replace('_monthly', '') for c in odf_ma_e.columns
]
with xr.open_dataset(
gdir.get_filepath('model_diagnostics',
filesuffix='_monthly')) as ds:
sel_vars = [
v for v in ds.variables if 'month_2d' not in ds[v].dims
]
odf_m = ds[sel_vars].to_dataframe()
sel_vars = [
v for v in ds.variables if 'month_2d' in ds[v].dims
]
odf_mm = ds[sel_vars].isel(time=slice(0, 10)).mean(
dim='time').to_dataframe()
odf_mm.columns = [
c.replace('_monthly', '') for c in odf_ma.columns
]
odf_mm_e = ds[sel_vars].isel(time=slice(-10, -1)).mean(
dim='time').to_dataframe()
odf_mm_e.columns = [
c.replace('_monthly', '') for c in odf_ma_e.columns
]
# Check annual and monthly stuff
for _df in [odf_ma, odf_ma_e, odf_mm, odf_mm_e]:
_df['tot_prcp'] = (_df['liq_prcp_off_glacier'] +
_df['liq_prcp_on_glacier'] +
_df['snowfall_off_glacier'] +
_df['snowfall_on_glacier'])
_df['runoff'] = (_df['melt_on_glacier'] +
_df['melt_off_glacier'] +
_df['liq_prcp_on_glacier'] +
_df['liq_prcp_off_glacier'])
# Residual MB should not be crazy large
# frac = odf_ma['residual_mb'] / odf_ma['melt_on_glacier']
# assert_allclose(frac.loc[~frac.isnull()], 0, atol=0.02)
fac_a = odf_ma_e.tot_prcp / odf_ma.tot_prcp
assert_allclose(fac_a, fac_a.iloc[0])
fac_m = odf_mm_e.tot_prcp / odf_mm.tot_prcp
# Here this is not the same!
assert_allclose(fac_m, fac_m.iloc[0], rtol=0.1)
# Check that the number of months with + and - is the same
dp_dt = np.asarray(gdir.get_diagnostics()['magicc_dp_per_dt'])
assert np.sum(fac_m > 1) == np.sum(dp_dt > 0)
if DO_PLOT:
rvars = [
'melt_on_glacier', 'melt_off_glacier',
'liq_prcp_on_glacier', 'liq_prcp_off_glacier'
]
f, (ax1, ax2) = plt.subplots(2, 1, figsize=(8, 8), sharey=True)
odf_mm[rvars].clip(0).plot.area(ax=ax1,
title=gdir.rgi_id + ' begin')
odf_mm_e[rvars].clip(0).plot.area(ax=ax2,
title=gdir.rgi_id + ' end')
plt.tight_layout()
rvars = [
'snowfall_on_glacier', 'snowfall_off_glacier',
'liq_prcp_on_glacier', 'liq_prcp_off_glacier'
]
f, (ax1, ax2) = plt.subplots(2, 1, figsize=(8, 8), sharey=True)
odf_mm[rvars].clip(0).plot.area(ax=ax1,
title=gdir.rgi_id + ' begin')
odf_mm_e[rvars].clip(0).plot.area(ax=ax2,
title=gdir.rgi_id + ' end')
plt.tight_layout()
plt.show()
def test_hydro_monhly_vs_annual(self, case_dir):
cfg.initialize()
cfg.PARAMS['prcp_scaling_factor'] = 1.6
cfg.PATHS['working_dir'] = case_dir
cfg.PARAMS['use_multiprocessing'] = True
cfg.PARAMS['continue_on_error'] = False
cfg.PARAMS['store_diagnostic_variables'] = ALL_DIAGS
# Go - get the pre-processed glacier directories
rgi_ids = ['RGI60-11.00897', 'RGI60-14.06794']
gdirs = workflow.init_glacier_directories(
rgi_ids,
from_prepro_level=5,
prepro_base_url=prepro_base_url,
prepro_border=80,
prepro_rgi_version='62')
workflow.execute_entity_task(parse_dt_per_dt, gdirs)
exp = 'netzero_py2050_fac1.0_decr0.3'
magicc_file = magicc_dir + exp + '.nc'
with xr.open_dataset(utils.file_downloader(magicc_file),
decode_times=False) as ds:
ds = ds.load()
df = ds['ens_avg'].to_dataframe()
workflow.execute_entity_task(tasks.run_with_hydro,
gdirs,
store_monthly_hydro=False,
run_task=run_from_magicc_data,
magicc_ts=df['ens_avg'],
use_dp_per_dt=False,
init_model_filesuffix='_historical',
output_filesuffix='_annual')
workflow.execute_entity_task(tasks.run_with_hydro,
gdirs,
store_monthly_hydro=True,
run_task=run_from_magicc_data,
magicc_ts=df['ens_avg'],
use_dp_per_dt=False,
init_model_filesuffix='_historical',
output_filesuffix='_monthly')
for gdir in gdirs:
with xr.open_dataset(
gdir.get_filepath('model_diagnostics',
filesuffix='_annual')) as ds:
odf_a = ds.to_dataframe()
with xr.open_dataset(
gdir.get_filepath('model_diagnostics',
filesuffix='_monthly')) as ds:
sel_vars = [
v for v in ds.variables if 'month_2d' not in ds[v].dims
]
odf_m = ds[sel_vars].to_dataframe()
sel_vars = [
v for v in ds.variables if 'month_2d' in ds[v].dims
]
odf_ma = ds[sel_vars].isel(time=slice(0, 10)).mean(
dim='time').to_dataframe()
odf_ma.columns = [
c.replace('_monthly', '') for c in odf_ma.columns
]
odf_ma_e = ds[sel_vars].isel(time=slice(-10, -1)).mean(
dim='time').to_dataframe()
odf_ma_e.columns = [
c.replace('_monthly', '') for c in odf_ma_e.columns
]
# Check that yearly equals monthly
np.testing.assert_array_equal(odf_a.columns, odf_m.columns)
for c in odf_a.columns:
rtol = 1e-5
if c == 'melt_off_glacier':
rtol = 0.1
if c in ['snow_bucket']:
continue
# if DO_PLOT:
# f, ax = plt.subplots(1, 1)
# odf_a[c].plot(ax=ax, title=c)
# odf_m[c].plot(ax=ax)
# plt.show()
assert_allclose(odf_a[c], odf_m[c], rtol=rtol)
# Check monthly stuff
odf_ma['tot_prcp'] = (odf_ma['liq_prcp_off_glacier'] +
odf_ma['liq_prcp_on_glacier'] +
odf_ma['snowfall_off_glacier'] +
odf_ma['snowfall_on_glacier'])
odf_ma['runoff'] = (odf_ma['melt_on_glacier'] +
odf_ma['melt_off_glacier'] +
odf_ma['liq_prcp_on_glacier'] +
odf_ma['liq_prcp_off_glacier'])
# Residual MB should not be crazy large
frac = odf_ma['residual_mb'] / odf_ma['melt_on_glacier']
assert_allclose(frac.loc[~frac.isnull()], 0, atol=0.02)
if DO_PLOT:
rvars = [
'melt_on_glacier', 'melt_off_glacier',
'liq_prcp_on_glacier', 'liq_prcp_off_glacier'
]
f, (ax1, ax2) = plt.subplots(2,
1,
figsize=(5, 10),
sharey=True)
odf_ma[rvars].clip(0).plot.area(ax=ax1,
title=gdir.rgi_id + ' begin')
odf_ma_e[rvars].clip(0).plot.area(ax=ax2,
title=gdir.rgi_id + ' end')
plt.show()
def test_hydro_workflow(self, case_dir):
cfg.initialize()
cfg.PARAMS['prcp_scaling_factor'] = 1.6
cfg.PATHS['working_dir'] = case_dir
cfg.PARAMS['use_multiprocessing'] = True
cfg.PARAMS['store_diagnostic_variables'] = ALL_DIAGS
# Go - get the pre-processed glacier directories
rgi_ids = ['RGI60-14.06794', 'RGI60-11.00897']
gdirs = workflow.init_glacier_directories(
rgi_ids,
from_prepro_level=5,
prepro_base_url=prepro_base_url,
prepro_border=80,
prepro_rgi_version='62')
workflow.execute_entity_task(parse_dt_per_dt, gdirs)
exp = 'netzero_py2050_fac1.0_decr0.3'
magicc_file = magicc_dir + exp + '.nc'
with xr.open_dataset(utils.file_downloader(magicc_file),
decode_times=False) as ds:
ds = ds.load()
df = ds['ens_avg'].to_dataframe()
workflow.execute_entity_task(tasks.run_with_hydro,
gdirs,
ref_area_from_y0=True,
run_task=run_from_magicc_data,
magicc_ts=df['ens_avg'],
use_dp_per_dt=False,
init_model_filesuffix='_historical',
output_filesuffix='_' + exp)
ds = utils.compile_run_output(gdirs, filesuffix='_' + exp)
for rgi_id in rgi_ids:
odf = ds.sel(rgi_id=rgi_id).to_dataframe().iloc[:-1]
# Sanity checks
# Tot prcp here is constant (constant climate)
odf['tot_prcp'] = (odf['liq_prcp_off_glacier'] +
odf['liq_prcp_on_glacier'] +
odf['snowfall_off_glacier'] +
odf['snowfall_on_glacier'])
assert_allclose(odf['tot_prcp'],
odf['tot_prcp'].iloc[0],
rtol=1e-4)
# Glacier area is the same (remove on_area?)
assert_allclose(odf['on_area'], odf['area'])
# Our MB is the same as the glacier dyn one
reconstructed_vol = (
odf['model_mb'].cumsum() / cfg.PARAMS['ice_density'] +
odf['volume'].iloc[0])
assert_allclose(odf['volume'].iloc[1:],
reconstructed_vol.iloc[:-1],
atol=1e-2)
# Mass-conservation
odf['runoff'] = (odf['melt_on_glacier'] + odf['melt_off_glacier'] +
odf['liq_prcp_on_glacier'] +
odf['liq_prcp_off_glacier'])
mass_in_glacier_end = odf['volume'].iloc[-1] * cfg.PARAMS[
'ice_density']
mass_in_glacier_start = odf['volume'].iloc[0] * cfg.PARAMS[
'ice_density']
mass_in_snow = odf['snow_bucket'].iloc[-1]
mass_in = odf['tot_prcp'].iloc[:-1].sum()
mass_out = odf['runoff'].iloc[:-1].sum()
assert_allclose(mass_in_glacier_end,
mass_in_glacier_start + mass_in - mass_out -
mass_in_snow,
atol=1e-2) # 0.01 kg is OK as numerical error
# Qualitative assessments
assert odf['melt_on_glacier'].iloc[
-1] < odf['melt_on_glacier'].iloc[0] * 0.7
assert odf['liq_prcp_off_glacier'].iloc[-1] > odf[
'liq_prcp_on_glacier'].iloc[-1]
assert odf['liq_prcp_off_glacier'].iloc[0] < odf[
'liq_prcp_on_glacier'].iloc[0]
# Residual MB should not be crazy large
frac = odf['residual_mb'] / odf['runoff']
assert_allclose(frac, 0, atol=0.13)
if DO_PLOT:
plt.figure()
odf[['volume']].plot(title=rgi_id)
plt.figure()
odf[[
'melt_on_glacier', 'melt_off_glacier',
'liq_prcp_on_glacier', 'liq_prcp_off_glacier'
]].plot.area(title=rgi_id)
plt.figure()
frac.plot(title=rgi_id)
plt.show()
dataset='ERA5dr')
# for gdir in gdirs:
# try:
# # can do workflow.execute_entity_task
# oggm.shop.ecmwf.process_ecmwf_data(gdir, dataset='ERA5dr')
# except:
# print(gdir.rgi_id)
# download the CMIP5 files and bias correct them to the calibration data
try:
bp = 'https://cluster.klima.uni-bremen.de/~oggm/cmip5-ng/pr/pr_mon_CCSM4_{}_r1i1p1_g025.nc'
bt = 'https://cluster.klima.uni-bremen.de/~oggm/cmip5-ng/tas/tas_mon_CCSM4_{}_r1i1p1_g025.nc'
for rcp in ['rcp26', 'rcp45', 'rcp60', 'rcp85']:
# Download the files
ft = utils.file_downloader(bt.format(rcp))
fp = utils.file_downloader(bp.format(rcp))
# bias correct them
workflow.execute_entity_task(
gcm_climate.process_cmip_data,
gdirs,
filesuffix='_CCSM4_{}'.format(rcp),
# recognize the climate file for later
fpath_temp=ft,
# temperature projections
fpath_precip=fp, # precip projections
)
except:
pass
else:
gdirs = workflow.init_glacier_directories(
def match_geodetic_mb_for_selection(gdirs,
period='2000-01-01_2020-01-01',
file_path=None,
fail_safe=False):
"""Shift the mass-balance residual to match geodetic mb observations.
It is similar to match_regional_geodetic_mb but uses the raw, glacier
per glacier tabular data.
This method finds the "best mass-balance residual" to match all glaciers in
gdirs with available OGGM mass balance and available geodetic mass-balance
measurements from Hugonnet 2021 or any other file with the same format.
The default is to use hugonnet_2021_ds_rgi60_pergla_rates_10_20_worldwide_filled.hdf
in https://cluster.klima.uni-bremen.de/~oggm/geodetic_ref_mb/
Parameters
----------
gdirs : the list of gdirs
period : str
One of
'2000-01-01_2020-01-01',
'2000-01-01_2010-01-01',
'2010-01-01_2020-01-01'.
file_path: str
local file path to tabular file containing geodetic measurements, file must
contain the columns:
- 'rgiid': is the RGIId as in the RGI 6.0
- 'period': time intervall of the measurements in the format shown
above
- 'dmdtda': the specific-mass change rate in meters water-equivalent
per year,
- 'area': is the glacier area (same as in RGI 6.0) in meters square
fail_safe : bool
some glaciers in the obs data have been corrected with the regional
average. We don't use these values, unless there is no other choice and
in which case you can set fail_safe to True
"""
# Get the mass-balance OGGM would give out of the box
df = utils.compile_fixed_geometry_mass_balance(gdirs, path=False)
df = df.dropna(axis=0, how='all').dropna(axis=1, how='all')
# And also the Area and calving fluxes
dfs = utils.compile_glacier_statistics(gdirs, path=False)
y0 = int(period.split('_')[0].split('-')[0])
y1 = int(period.split('_')[1].split('-')[0]) - 1
odf = pd.DataFrame(df.loc[y0:y1].mean(), columns=['SMB'])
odf['AREA'] = dfs.rgi_area_km2 * 1e6
# Just take the calving rate and change its units
# Original units: km3 a-1, to change to mm a-1 (units of specific MB)
rho = cfg.PARAMS['ice_density']
if 'calving_flux' in dfs:
odf['CALVING'] = dfs['calving_flux'].fillna(
0) * 1e9 * rho / odf['AREA']
else:
odf['CALVING'] = 0
# We have to drop nans here, which occur when calving glaciers fail to run
odf = odf.dropna()
# save all rgi_ids for which a valid OGGM mb is available
rgi_ids_oggm = odf.index.values
# fetch the file online or read custom file
if file_path is None:
base_url = 'https://cluster.klima.uni-bremen.de/~oggm/geodetic_ref_mb/'
file_name = 'hugonnet_2021_ds_rgi60_pergla_rates_10_20_worldwide_filled.hdf'
df = pd.read_hdf(utils.file_downloader(base_url + file_name))
else:
extension = os.path.splitext(file_path)[1]
if extension == '.csv':
df = pd.read_csv(file_path, index_col='rgiid')
elif extension == '.hdf':
df = pd.read_hdf(file_path, index_col='rgiid')
# get the correct period from the whole dataset
df = df.loc[df['period'] == period]
# get only geodetic measurements for which a valid OGGM mb is available
rdf_all = df.loc[rgi_ids_oggm]
if rdf_all.empty:
raise InvalidWorkflowError('No geodetic MB measurements available for '
'this glacier selection!')
# drop glaciers with no valid geodetic measurements
rdf = rdf_all.loc[~rdf_all['is_cor']]
if rdf.empty:
if not fail_safe:
raise InvalidWorkflowError(
'No gedoetic MB measurements available for '
'this glacier selection! Set '
'fail_safe=True to use the '
'corrected values.')
rdf = rdf_all
# the remaining glaciers now have a OGGM mb and geodetic measurements
rgi_ids = rdf.index.values
msg = ('Applying geodetic MB correction using {} of {} glaciers, with '
'available OGGM MB and available geodetic measurements.')
log.workflow(msg.format(len(rgi_ids), len(gdirs)))
# Total MB OGGM, only using glaciers with OGGM mb and geodetic measurements
odf = odf.loc[rgi_ids]
out_smb = np.average(odf['SMB'], weights=odf['AREA']) # for logging
out_cal = np.average(odf['CALVING'], weights=odf['AREA']) # for logging
smb_oggm = np.average(odf['SMB'] - odf['CALVING'], weights=odf['AREA'])
# Total geodetic MB, no need for indexing
smb_ref = rdf.dmdtda.values * 1000 # m to mm conversion
area_ref = rdf.area.values
smb_ref = np.average(smb_ref, weights=area_ref)
# Diff between the two
residual = smb_ref - smb_oggm
# Let's just shift
log.workflow('Shifting regional MB bias by {}'.format(residual))
log.workflow('Observations give {}'.format(smb_ref))
log.workflow('OGGM SMB gives {}'.format(out_smb))
log.workflow('OGGM frontal ablation gives {}'.format(out_cal))
# This time we shift over all glaciers
for gdir in gdirs:
try:
df = gdir.read_json('local_mustar')
gdir.add_to_diagnostics('mb_bias_before_geodetic_corr', df['bias'])
df['bias'] = df['bias'] - residual
gdir.write_json(df, 'local_mustar')
except FileNotFoundError:
pass
cfg.PARAMS['continue_on_error'] = False
# We use intersects
# Here we use the global file but there are regional files too (faster)
cfg.set_intersects_db(utils.get_rgi_intersects_region_file('00', version='5'))
# Pre-download other files which will be needed later
utils.get_cru_cl_file()
utils.get_cru_file(var='tmp')
utils.get_cru_file(var='pre')
# Download the RGI file for the run
# We us a set of four glaciers here but this could be an entire RGI region,
# or any glacier list you'd like to model
dl = 'https://cluster.klima.uni-bremen.de/~fmaussion/misc/RGI_example_glaciers.zip'
with zipfile.ZipFile(utils.file_downloader(dl)) as zf:
zf.extractall(WORKING_DIR)
rgidf = salem.read_shapefile(
path.join(WORKING_DIR, 'RGI_example_glaciers', 'RGI_example_glaciers.shp'))
# Sort for more efficient parallel computing
rgidf = rgidf.sort_values('Area', ascending=False)
log.info('Starting OGGM run')
log.info('Number of glaciers: {}'.format(len(rgidf)))
# Go - initialize working directories
gdirs = workflow.init_glacier_regions(rgidf)
# Preprocessing tasks
task_list = [
# Pre-download other files which will be needed later
_ = utils.get_cru_file(var='tmp')
_ = utils.get_cru_file(var='pre')
# Copy the precalibrated tstar file
# ---------------------------------
# Note that to be exact, this procedure can only be applied if the run
# parameters don't change between the calibration and the run.
# After testing, it appears that changing the 'border' parameter won't affect
# the results much (expectedly), so that it's ok to change it. All the rest
# (e.g. smoothing, dx, prcp factor...) should imply a re-calibration
mbf = 'https://dl.dropboxusercontent.com/u/20930277/ref_tstars_b60_prcpfac_25_defaults.csv'
mbf = utils.file_downloader(mbf)
shutil.copyfile(mbf, os.path.join(WORKING_DIR, 'ref_tstars.csv'))
# Initialize OGGM and set up the run parameters
# ---------------------------------------------
# Download and read in the RGI file
rgif = 'https://dl.dropboxusercontent.com/u/20930277/RGI_example_glaciers.zip'
rgif = utils.file_downloader(rgif)
with zipfile.ZipFile(rgif) as zf:
zf.extractall(WORKING_DIR)
rgif = os.path.join(WORKING_DIR, 'RGI_example_glaciers.shp')
rgidf = salem.read_shapefile(rgif, cached=True)
# Sort for more efficient parallel computing
rgidf = rgidf.sort_values('Area', ascending=False)
import oggm
from oggm import cfg, tasks, graphics
from oggm.sandbox.gmd_paper import PLOT_DIR
from oggm.utils import file_downloader, nicenumber, mkdir
cfg.initialize()
cfg.PARAMS['border'] = 10
reset = False
base_dir = os.path.join(os.path.expanduser('~/tmp'), 'OGGM_GMD', 'Grindelwald')
cfg.PATHS['working_dir'] = base_dir
mkdir(base_dir, reset=reset)
rgif = 'https://www.dropbox.com/s/zkprlr5ad5voysh/rgiv5_grindelwald.zip?dl=1'
rgif = file_downloader(rgif)
with zipfile.ZipFile(rgif) as zf:
zf.extractall(base_dir)
rgif = os.path.join(base_dir, 'rgiv5_grindelwald.shp')
rgidf = salem.read_shapefile(rgif, cached=True)
entity = rgidf.iloc[0]
gdir = oggm.GlacierDirectory(entity, base_dir=base_dir, reset=True)
tasks.define_glacier_region(gdir, entity=entity)
tasks.glacier_masks(gdir)
tasks.compute_centerlines(gdir)
tasks.initialize_flowlines(gdir)
tasks.compute_downstream_line(gdir)
tasks.catchment_area(gdir)
tasks.catchment_intersections(gdir)
tasks.catchment_width_geom(gdir)
def run_prepro_levels(rgi_version=None,
rgi_reg=None,
border=None,
output_folder='',
working_dir='',
dem_source='',
is_test=False,
test_ids=None,
demo=False,
test_rgidf=None,
test_intersects_file=None,
test_topofile=None,
disable_mp=False,
params_file=None,
elev_bands=False,
match_regional_geodetic_mb=False,
match_geodetic_mb_per_glacier=False,
evolution_model='fl_sia',
centerlines_only=False,
override_params=None,
add_consensus=False,
start_level=None,
start_base_url=None,
max_level=5,
ref_tstars_base_url='',
logging_level='WORKFLOW',
disable_dl_verify=False,
dynamic_spinup=False,
dynamic_spinup_start_year=1979,
continue_on_error=True):
"""Generate the preprocessed OGGM glacier directories for this OGGM version
Parameters
----------
rgi_version : str
the RGI version to use (defaults to cfg.PARAMS)
rgi_reg : str
the RGI region to process
border : int
the number of pixels at the maps border
output_folder : str
path to the output folder (where to put the preprocessed tar files)
dem_source : str
which DEM source to use: default, SOURCE_NAME or ALL
working_dir : str
path to the OGGM working directory
ref_tstars_base_url : str
url where to find the pre-calibrated reference tstar list.
Required as of v1.4.
params_file : str
path to the OGGM parameter file (to override defaults)
is_test : bool
to test on a couple of glaciers only!
test_ids : list
if is_test: list of ids to process
demo : bool
to run the prepro for the list of demo glaciers
test_rgidf : shapefile
for testing purposes only
test_intersects_file : shapefile
for testing purposes only
test_topofile : str
for testing purposes only
test_crudir : str
for testing purposes only
disable_mp : bool
disable multiprocessing
elev_bands : bool
compute all flowlines based on the Huss&Hock 2015 method instead
of the OGGM default, which is a mix of elev_bands and centerlines.
centerlines_only : bool
compute all flowlines based on the OGGM centerline(s) method instead
of the OGGM default, which is a mix of elev_bands and centerlines.
match_regional_geodetic_mb : str
match the regional mass balance estimates at the regional level
('hugonnet': Hugonnet et al., 2020 or 'zemp': Zemp et al., 2019).
match_geodetic_mb_per_glacier : str
match the mass balance estimates at the glacier level
(currently only 'hugonnet': Hugonnet et al., 2020).
evolution_model : str
which geometry evolution model to use: `fl_sia` (default),
or `massredis` (mass redistribution curve).
add_consensus : bool
adds (reprojects) the consensus estimates thickness to the glacier
directories. With elev_bands=True, the data will also be binned.
start_level : int
the pre-processed level to start from (default is to start from
scratch). If set, you'll need to indicate start_base_url as well.
start_base_url : str
the pre-processed base-url to fetch the data from.
max_level : int
the maximum pre-processing level before stopping
logging_level : str
the logging level to use (DEBUG, INFO, WARNING, WORKFLOW)
override_params : dict
a dict of parameters to override.
disable_dl_verify : bool
disable the hash verification of OGGM downloads
dynamic_spinup: str
include a dynamic spinup matching 'area' OR 'volume' at the RGI-date
dynamic_spinup_start_year : int
if dynamic_spinup is set, define the starting year for the simulation.
The default is 1979, unless the climate data starts later.
"""
# Input check
if max_level not in [1, 2, 3, 4, 5]:
raise InvalidParamsError('max_level should be one of [1, 2, 3, 4, 5]')
if start_level is not None:
if start_level not in [0, 1, 2, 3, 4]:
raise InvalidParamsError(
'start_level should be one of [0, 1, 2, 3, 4]')
if start_level > 0 and start_base_url is None:
raise InvalidParamsError('With start_level, please also indicate '
'start_base_url')
else:
start_level = 0
if match_regional_geodetic_mb and match_geodetic_mb_per_glacier:
raise InvalidParamsError(
'match_regional_geodetic_mb incompatible with '
'match_geodetic_mb_per_glacier!')
if match_geodetic_mb_per_glacier and match_geodetic_mb_per_glacier != 'hugonnet':
raise InvalidParamsError('Currently only `hugonnet` is available for '
'match_geodetic_mb_per_glacier.')
if evolution_model not in ['fl_sia', 'massredis']:
raise InvalidParamsError('evolution_model should be one of '
"['fl_sia', 'massredis'].")
if dynamic_spinup and dynamic_spinup not in ['area', 'volume']:
raise InvalidParamsError(f"Dynamic spinup option '{dynamic_spinup}' "
"not supported")
if dynamic_spinup and evolution_model == 'massredis':
raise InvalidParamsError("Dynamic spinup is not working/tested"
"with massredis!")
# Time
start = time.time()
def _time_log():
# Log util
m, s = divmod(time.time() - start, 60)
h, m = divmod(m, 60)
log.workflow('OGGM prepro_levels is done! Time needed: '
'{:02d}:{:02d}:{:02d}'.format(int(h), int(m), int(s)))
# Local paths
if override_params is None:
override_params = {}
utils.mkdir(working_dir)
override_params['working_dir'] = working_dir
# Initialize OGGM and set up the run parameters
cfg.initialize(file=params_file,
params=override_params,
logging_level=logging_level,
future=True)
if match_geodetic_mb_per_glacier and (cfg.PARAMS['hydro_month_nh'] != 1 or
cfg.PARAMS['hydro_month_sh'] != 1):
raise InvalidParamsError('We recommend to set hydro_month_nh and sh '
'to 1 for the geodetic MB calibration per '
'glacier.')
# Use multiprocessing?
cfg.PARAMS['use_multiprocessing'] = not disable_mp
# How many grid points around the glacier?
# Make it large if you expect your glaciers to grow large
cfg.PARAMS['border'] = border
# Set to True for operational runs
cfg.PARAMS['continue_on_error'] = continue_on_error
# Check for the integrity of the files OGGM downloads at run time
# For large files (e.g. using a 1 tif DEM like ALASKA) calculating the hash
# takes a long time, so deactivating this can make sense
cfg.PARAMS['dl_verify'] = not disable_dl_verify
# Other things that make sense
cfg.PARAMS['store_model_geometry'] = True
# Log the parameters
msg = '# OGGM Run parameters:'
for k, v in cfg.PARAMS.items():
if type(v) in [pd.DataFrame, dict]:
continue
msg += '\n {}: {}'.format(k, v)
log.workflow(msg)
if rgi_version is None:
rgi_version = cfg.PARAMS['rgi_version']
output_base_dir = os.path.join(output_folder, 'RGI{}'.format(rgi_version),
'b_{:03d}'.format(border))
# Add a package version file
utils.mkdir(output_base_dir)
opath = os.path.join(output_base_dir, 'package_versions.txt')
with open(opath, 'w') as vfile:
vfile.write(utils.show_versions(logger=log))
if demo:
rgidf = utils.get_rgi_glacier_entities(cfg.DATA['demo_glaciers'].index)
elif test_rgidf is None:
# Get the RGI file
rgidf = gpd.read_file(
utils.get_rgi_region_file(rgi_reg, version=rgi_version))
# We use intersects
rgif = utils.get_rgi_intersects_region_file(rgi_reg,
version=rgi_version)
cfg.set_intersects_db(rgif)
# Some RGI input quality checks - this is based on visual checks
# of large glaciers in the RGI
ids_to_ice_cap = [
'RGI60-05.10315', # huge Greenland ice cap
'RGI60-03.01466', # strange thing next to Devon
'RGI60-09.00918', # Academy of sciences Ice cap
'RGI60-09.00969',
'RGI60-09.00958',
'RGI60-09.00957',
]
rgidf.loc[rgidf.RGIId.isin(ids_to_ice_cap), 'Form'] = '1'
# In AA almost all large ice bodies are actually ice caps
if rgi_reg == '19':
rgidf.loc[rgidf.Area > 100, 'Form'] = '1'
# For greenland we omit connectivity level 2
if rgi_reg == '05':
rgidf = rgidf.loc[rgidf['Connect'] != 2]
else:
rgidf = test_rgidf
cfg.set_intersects_db(test_intersects_file)
if is_test:
if test_ids is not None:
rgidf = rgidf.loc[rgidf.RGIId.isin(test_ids)]
else:
rgidf = rgidf.sample(4)
if max_level > 2:
# Also use ref tstars
utils.apply_test_ref_tstars()
if max_level > 2 and ref_tstars_base_url:
workflow.download_ref_tstars(base_url=ref_tstars_base_url)
log.workflow('Starting prepro run for RGI reg: {} '
'and border: {}'.format(rgi_reg, border))
log.workflow('Number of glaciers: {}'.format(len(rgidf)))
# L0 - go
if start_level == 0:
gdirs = workflow.init_glacier_directories(rgidf,
reset=True,
force=True)
# Glacier stats
sum_dir = os.path.join(output_base_dir, 'L0', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir,
'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# L0 OK - compress all in output directory
log.workflow('L0 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L0')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 0:
_time_log()
return
else:
gdirs = workflow.init_glacier_directories(
rgidf,
reset=True,
force=True,
from_prepro_level=start_level,
prepro_border=border,
prepro_rgi_version=rgi_version,
prepro_base_url=start_base_url)
# L1 - Add dem files
if start_level == 0:
if test_topofile:
cfg.PATHS['dem_file'] = test_topofile
# Which DEM source?
if dem_source.upper() == 'ALL':
# This is the complex one, just do the job and leave
log.workflow('Running prepro on ALL sources')
for i, s in enumerate(utils.DEM_SOURCES):
rs = i == 0
log.workflow('Running prepro on sources: {}'.format(s))
gdirs = workflow.init_glacier_directories(rgidf,
reset=rs,
force=rs)
workflow.execute_entity_task(tasks.define_glacier_region,
gdirs,
source=s)
workflow.execute_entity_task(_rename_dem_folder,
gdirs,
source=s)
# make a GeoTiff mask of the glacier, choose any source
workflow.execute_entity_task(gis.rasterio_glacier_mask,
gdirs,
source='ALL')
# Compress all in output directory
level_base_dir = os.path.join(output_base_dir, 'L1')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
_time_log()
return
# Force a given source
source = dem_source.upper() if dem_source else None
# L1 - go
workflow.execute_entity_task(tasks.define_glacier_region,
gdirs,
source=source)
# Glacier stats
sum_dir = os.path.join(output_base_dir, 'L1', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir,
'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# L1 OK - compress all in output directory
log.workflow('L1 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L1')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 1:
_time_log()
return
# L2 - Tasks
if start_level <= 1:
# Check which glaciers will be processed as what
if elev_bands:
gdirs_band = gdirs
gdirs_cent = []
elif centerlines_only:
gdirs_band = []
gdirs_cent = gdirs
else:
# Default is to centerlines_only, but it used to be a mix
# (e.g. bands for ice caps, etc)
# I still keep this logic here in case we want to mix again
gdirs_band = []
gdirs_cent = gdirs
log.workflow('Start flowline processing with: '
'N centerline type: {}, '
'N elev bands type: {}.'
''.format(len(gdirs_cent), len(gdirs_band)))
# HH2015 method
workflow.execute_entity_task(tasks.simple_glacier_masks, gdirs_band)
# Centerlines OGGM
workflow.execute_entity_task(tasks.glacier_masks, gdirs_cent)
if add_consensus:
from oggm.shop.bedtopo import add_consensus_thickness
workflow.execute_entity_task(add_consensus_thickness, gdirs_band)
workflow.execute_entity_task(add_consensus_thickness, gdirs_cent)
# Elev bands with var data
vn = 'consensus_ice_thickness'
workflow.execute_entity_task(tasks.elevation_band_flowline,
gdirs_band,
bin_variables=vn)
workflow.execute_entity_task(
tasks.fixed_dx_elevation_band_flowline,
gdirs_band,
bin_variables=vn)
else:
# HH2015 method without it
task_list = [
tasks.elevation_band_flowline,
tasks.fixed_dx_elevation_band_flowline,
]
for task in task_list:
workflow.execute_entity_task(task, gdirs_band)
# Centerlines OGGM
task_list = [
tasks.compute_centerlines,
tasks.initialize_flowlines,
tasks.catchment_area,
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
]
for task in task_list:
workflow.execute_entity_task(task, gdirs_cent)
# Same for all glaciers
if border >= 20:
task_list = [
tasks.compute_downstream_line,
tasks.compute_downstream_bedshape,
]
for task in task_list:
workflow.execute_entity_task(task, gdirs)
else:
log.workflow('L2: for map border values < 20, wont compute '
'downstream lines.')
# Glacier stats
sum_dir = os.path.join(output_base_dir, 'L2', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir,
'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# And for level 2: shapes
if len(gdirs_cent) > 0:
opath = os.path.join(sum_dir, 'centerlines_{}.shp'.format(rgi_reg))
utils.write_centerlines_to_shape(gdirs_cent,
to_tar=True,
path=opath)
# L2 OK - compress all in output directory
log.workflow('L2 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L2')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 2:
_time_log()
return
# L3 - Tasks
if start_level <= 2:
sum_dir = os.path.join(output_base_dir, 'L3', 'summary')
utils.mkdir(sum_dir)
# Climate
workflow.execute_entity_task(tasks.process_climate_data, gdirs)
if cfg.PARAMS['climate_qc_months'] > 0:
workflow.execute_entity_task(tasks.historical_climate_qc, gdirs)
if match_geodetic_mb_per_glacier:
utils.get_geodetic_mb_dataframe(
) # Small optim to avoid concurrency
workflow.execute_entity_task(
tasks.mu_star_calibration_from_geodetic_mb, gdirs)
workflow.execute_entity_task(tasks.apparent_mb_from_any_mb, gdirs)
else:
workflow.execute_entity_task(tasks.local_t_star, gdirs)
workflow.execute_entity_task(tasks.mu_star_calibration, gdirs)
# Inversion: we match the consensus
filter = border >= 20
workflow.calibrate_inversion_from_consensus(
gdirs,
apply_fs_on_mismatch=True,
error_on_mismatch=False,
filter_inversion_output=filter)
# Do we want to match geodetic estimates?
# This affects only the bias so we can actually do this *after*
# the inversion, but we really want to take calving into account here
if match_regional_geodetic_mb:
opath = os.path.join(
sum_dir, 'fixed_geometry_mass_balance_'
'before_match_{}.csv'.format(rgi_reg))
utils.compile_fixed_geometry_mass_balance(gdirs, path=opath)
workflow.match_regional_geodetic_mb(
gdirs, rgi_reg=rgi_reg, dataset=match_regional_geodetic_mb)
# We get ready for modelling
if border >= 20:
workflow.execute_entity_task(tasks.init_present_time_glacier,
gdirs)
else:
log.workflow(
'L3: for map border values < 20, wont initialize glaciers '
'for the run.')
# Glacier stats
opath = os.path.join(sum_dir,
'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
opath = os.path.join(sum_dir,
'climate_statistics_{}.csv'.format(rgi_reg))
utils.compile_climate_statistics(gdirs, path=opath)
opath = os.path.join(
sum_dir, 'fixed_geometry_mass_balance_{}.csv'.format(rgi_reg))
utils.compile_fixed_geometry_mass_balance(gdirs, path=opath)
# L3 OK - compress all in output directory
log.workflow('L3 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L3')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 3:
_time_log()
return
if border < 20:
log.workflow(
'L3: for map border values < 20, wont compute L4 and L5.')
_time_log()
return
# is needed to copy some files for L4 and L5
sum_dir_L3 = sum_dir
# L4 - No tasks: add some stats for consistency and make the dirs small
if start_level <= 3:
sum_dir = os.path.join(output_base_dir, 'L4', 'summary')
utils.mkdir(sum_dir)
# Copy L3 files for consistency
for bn in [
'glacier_statistics', 'climate_statistics',
'fixed_geometry_mass_balance'
]:
if start_level < 3:
ipath = os.path.join(sum_dir_L3,
bn + '_{}.csv'.format(rgi_reg))
else:
ipath = file_downloader(
os.path.join(
get_prepro_base_url(base_url=start_base_url,
rgi_version=rgi_version,
border=border,
prepro_level=start_level),
'summary', bn + '_{}.csv'.format(rgi_reg)))
opath = os.path.join(sum_dir, bn + '_{}.csv'.format(rgi_reg))
shutil.copyfile(ipath, opath)
# Copy mini data to new dir
mini_base_dir = os.path.join(working_dir, 'mini_perglacier',
'RGI{}'.format(rgi_version),
'b_{:03d}'.format(border))
mini_gdirs = workflow.execute_entity_task(tasks.copy_to_basedir,
gdirs,
base_dir=mini_base_dir)
# L4 OK - compress all in output directory
log.workflow('L4 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L4')
workflow.execute_entity_task(utils.gdir_to_tar,
mini_gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 4:
_time_log()
return
# use mini_gdirs for L5
gdirs = mini_gdirs
# L5 - spinup run in mini gdirs
# Get end date. The first gdir might have blown up, try some others
i = 0
while True:
if i >= len(gdirs):
raise RuntimeError('Found no valid glaciers!')
try:
y0 = gdirs[i].get_climate_info()['baseline_hydro_yr_0']
# One adds 1 because the run ends at the end of the year
ye = gdirs[i].get_climate_info()['baseline_hydro_yr_1'] + 1
break
except BaseException:
i += 1
# Which model?
if evolution_model == 'massredis':
from oggm.core.flowline import MassRedistributionCurveModel
evolution_model = MassRedistributionCurveModel
else:
from oggm.core.flowline import FluxBasedModel
evolution_model = FluxBasedModel
# OK - run
if dynamic_spinup:
if y0 > dynamic_spinup_start_year:
dynamic_spinup_start_year = y0
workflow.execute_entity_task(
tasks.run_dynamic_spinup,
gdirs,
evolution_model=evolution_model,
minimise_for=dynamic_spinup,
spinup_start_yr=dynamic_spinup_start_year,
output_filesuffix='_dynamic_spinup',
)
workflow.execute_entity_task(tasks.run_from_climate_data,
gdirs,
min_ys=y0,
ye=ye,
evolution_model=evolution_model,
init_model_filesuffix='_dynamic_spinup',
output_filesuffix='_hist_spin')
workflow.execute_entity_task(tasks.merge_consecutive_run_outputs,
gdirs,
input_filesuffix_1='_dynamic_spinup',
input_filesuffix_2='_hist_spin',
output_filesuffix='_historical_spinup',
delete_input=True)
workflow.execute_entity_task(tasks.run_from_climate_data,
gdirs,
min_ys=y0,
ye=ye,
evolution_model=evolution_model,
output_filesuffix='_historical')
# Now compile the output
sum_dir = os.path.join(output_base_dir, 'L5', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, f'historical_run_output_{rgi_reg}.nc')
utils.compile_run_output(gdirs, path=opath, input_filesuffix='_historical')
if dynamic_spinup:
opath = os.path.join(sum_dir,
f'historical_spinup_run_output_{rgi_reg}.nc')
utils.compile_run_output(gdirs,
path=opath,
input_filesuffix='_historical_spinup')
# Glacier statistics we recompute here for error analysis
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# Other stats for consistency
for bn in ['climate_statistics', 'fixed_geometry_mass_balance']:
if start_level < 3:
ipath = os.path.join(sum_dir_L3, bn + '_{}.csv'.format(rgi_reg))
else:
ipath = file_downloader(
os.path.join(
get_prepro_base_url(base_url=start_base_url,
rgi_version=rgi_version,
border=border,
prepro_level=start_level), 'summary',
bn + '_{}.csv'.format(rgi_reg)))
opath = os.path.join(sum_dir, bn + '_{}.csv'.format(rgi_reg))
shutil.copyfile(ipath, opath)
# Add the extended files
pf = os.path.join(sum_dir, 'historical_run_output_{}.nc'.format(rgi_reg))
mf = os.path.join(sum_dir,
'fixed_geometry_mass_balance_{}.csv'.format(rgi_reg))
# This is crucial - extending calving only possible with L3 data!!!
if start_level < 3:
sf = os.path.join(sum_dir_L3,
'glacier_statistics_{}.csv'.format(rgi_reg))
else:
sf = file_downloader(
os.path.join(
get_prepro_base_url(base_url=start_base_url,
rgi_version=rgi_version,
border=border,
prepro_level=start_level), 'summary',
'glacier_statistics_{}.csv'.format(rgi_reg)))
opath = os.path.join(
sum_dir, 'historical_run_output_extended_{}.nc'.format(rgi_reg))
utils.extend_past_climate_run(past_run_file=pf,
fixed_geometry_mb_file=mf,
glacier_statistics_file=sf,
path=opath)
# L5 OK - compress all in output directory
log.workflow('L5 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L5')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
_time_log()
def _reproject_and_scale(gdir, do_error=False):
"""Reproject and scale itslive data, avoid code duplication for error"""
reg = find_region(gdir)
if reg is None:
raise InvalidWorkflowError('There does not seem to be its_live data '
'available for this glacier')
vnx = 'vx'
vny = 'vy'
if do_error:
vnx += '_err'
vny += '_err'
with utils.get_lock():
fx = utils.file_downloader(region_files[reg][vnx])
fy = utils.file_downloader(region_files[reg][vny])
# Open the files
dsx = salem.GeoTiff(fx)
dsy = salem.GeoTiff(fy)
# subset them to our map
grid_gla = gdir.grid.center_grid
proj_vel = dsx.grid.proj
x0, x1, y0, y1 = grid_gla.extent_in_crs(proj_vel)
dsx.set_subset(corners=((x0, y0), (x1, y1)), crs=proj_vel, margin=4)
dsy.set_subset(corners=((x0, y0), (x1, y1)), crs=proj_vel, margin=4)
grid_vel = dsx.grid.center_grid
# TODO: this should be taken care of by salem
# https://github.com/fmaussion/salem/issues/171
with rasterio.Env():
with rasterio.open(fx) as src:
nodata = getattr(src, 'nodata', -32767.0)
# Error files are wrong
if nodata == 0:
nodata = -32767.0
# Get the coords at t0
xx0, yy0 = grid_vel.center_grid.xy_coordinates
# Compute coords at t1
xx1 = dsx.get_vardata()
yy1 = dsy.get_vardata()
non_valid = (xx1 == nodata) | (yy1 == nodata)
xx1[non_valid] = np.NaN
yy1[non_valid] = np.NaN
orig_vel = np.sqrt(xx1**2 + yy1**2)
xx1 += xx0
yy1 += yy0
# Transform both to glacier proj
xx0, yy0 = salem.transform_proj(proj_vel, grid_gla.proj, xx0, yy0)
xx1, yy1 = salem.transform_proj(proj_vel, grid_gla.proj, xx1, yy1)
# Correct no data after proj as well (inf)
xx1[non_valid] = np.NaN
yy1[non_valid] = np.NaN
# Compute velocities from there
vx = xx1 - xx0
vy = yy1 - yy0
# Scale back velocities - https://github.com/OGGM/oggm/issues/1014
new_vel = np.sqrt(vx**2 + vy**2)
p_ok = new_vel > 1e-5 # avoid div by zero
vx[p_ok] = vx[p_ok] * orig_vel[p_ok] / new_vel[p_ok]
vy[p_ok] = vy[p_ok] * orig_vel[p_ok] / new_vel[p_ok]
# And transform to local map
vx = grid_gla.map_gridded_data(vx, grid=grid_vel, interp='linear')
vy = grid_gla.map_gridded_data(vy, grid=grid_vel, interp='linear')
# Write
with utils.ncDataset(gdir.get_filepath('gridded_data'), 'a') as nc:
vn = 'obs_icevel_x'
if do_error:
vn = vn.replace('obs', 'err')
if vn in nc.variables:
v = nc.variables[vn]
else:
v = nc.createVariable(vn, 'f4', ('y', 'x', ), zlib=True)
v.units = 'm yr-1'
ln = 'ITS LIVE velocity data in x map direction'
if do_error:
ln = 'Uncertainty of ' + ln
v.long_name = ln
v[:] = vx.filled(np.nan)
vn = 'obs_icevel_y'
if do_error:
vn = vn.replace('obs', 'err')
if vn in nc.variables:
v = nc.variables[vn]
else:
v = nc.createVariable(vn, 'f4', ('y', 'x', ), zlib=True)
v.units = 'm yr-1'
ln = 'ITS LIVE velocity data in y map direction'
if do_error:
ln = 'Uncertainty of ' + ln
v.long_name = ln
v[:] = vy.filled(np.nan)
def get_cru_cl_file():
"""Returns the path to the unpacked CRU CL file."""
return utils.file_extractor(utils.file_downloader(CRU_CL))
import oggm
from oggm import cfg, tasks, graphics
from oggm.sandbox.gmd_paper import PLOT_DIR
from oggm import utils
cfg.initialize()
cfg.PARAMS['border'] = 20
cfg.PARAMS['auto_skip_task'] = True
base_dir = os.path.join(os.path.expanduser('~/tmp'), 'OGGM_GMD', 'Workflow')
cfg.PATHS['working_dir'] = base_dir
utils.mkdir(base_dir)
rgif = 'https://www.dropbox.com/s/bvku83j9bci9r3p/rgiv5_tasman.zip?dl=1'
rgif = utils.file_downloader(rgif)
with zipfile.ZipFile(rgif) as zf:
zf.extractall(base_dir)
rgif = os.path.join(base_dir, 'rgiv5_tasman.shp')
rgidf = gpd.read_file(rgif)
entity = rgidf.iloc[0]
gdir = oggm.GlacierDirectory(entity, base_dir=base_dir)
tasks.define_glacier_region(gdir, entity=entity)
tasks.glacier_masks(gdir)
tasks.compute_centerlines(gdir)
tasks.initialize_flowlines(gdir)
tasks.compute_downstream_line(gdir)
tasks.compute_downstream_bedshape(gdir)
tasks.catchment_area(gdir)
def velocity_to_gdir(gdir):
"""Reproject the its_live files to the given glacier directory.
Variables are added to the gridded_data nc file.
Reprojecting velocities from one map proj to another is done
reprojecting the vector distances. In this process, absolute velocities
might change as well because map projections do not always preserve
distances -> we scale them back to the original velocities as per the
ITS_LIVE documentation that states that velocities are given in
ground units, i.e. absolute velocities.
We use bilinear interpolation to reproject the velocities to the local
glacier map.
Parameters
----------
gdir : :py:class:`oggm.GlacierDirectory`
where to write the data
"""
reg = find_region(gdir)
if reg is None:
raise InvalidWorkflowError('There does not seem to be its_live data '
'available for this glacier')
if not gdir.has_file('gridded_data'):
raise InvalidWorkflowError('Please run `glacier_masks` before running '
'this task')
with utils.get_lock():
fx = utils.file_downloader(region_files[reg]['vx'])
fy = utils.file_downloader(region_files[reg]['vy'])
# Open the files
dsx = salem.GeoTiff(fx)
dsy = salem.GeoTiff(fy)
# subset them to our map
grid_gla = gdir.grid.center_grid
proj_vel = dsx.grid.proj
x0, x1, y0, y1 = grid_gla.extent_in_crs(proj_vel)
dsx.set_subset(corners=((x0, y0), (x1, y1)), crs=proj_vel, margin=4)
dsy.set_subset(corners=((x0, y0), (x1, y1)), crs=proj_vel, margin=4)
grid_vel = dsx.grid.center_grid
# TODO: this should be taken care of by salem
# https://github.com/fmaussion/salem/issues/171
with rasterio.Env():
with rasterio.open(fx) as src:
nodata = getattr(src, 'nodata', -32767.0)
# Get the coords at t0
xx0, yy0 = grid_vel.center_grid.xy_coordinates
# Compute coords at t1
xx1 = dsx.get_vardata()
yy1 = dsy.get_vardata()
non_valid = (xx1 == nodata) | (yy1 == nodata)
xx1[non_valid] = np.NaN
yy1[non_valid] = np.NaN
orig_vel = np.sqrt(xx1**2 + yy1**2)
xx1 += xx0
yy1 += yy0
# Transform both to glacier proj
xx0, yy0 = salem.transform_proj(proj_vel, grid_gla.proj, xx0, yy0)
xx1, yy1 = salem.transform_proj(proj_vel, grid_gla.proj, xx1, yy1)
# Correct no data after proj as well (inf)
xx1[non_valid] = np.NaN
yy1[non_valid] = np.NaN
# Compute velocities from there
vx = xx1 - xx0
vy = yy1 - yy0
# Scale back velocities - https://github.com/OGGM/oggm/issues/1014
new_vel = np.sqrt(vx**2 + vy**2)
p_ok = new_vel > 0.1 # avoid div by zero
vx[p_ok] = vx[p_ok] * orig_vel[p_ok] / new_vel[p_ok]
vy[p_ok] = vy[p_ok] * orig_vel[p_ok] / new_vel[p_ok]
# And transform to local map
vx = grid_gla.map_gridded_data(vx, grid=grid_vel, interp='linear')
vy = grid_gla.map_gridded_data(vy, grid=grid_vel, interp='linear')
# Write
with utils.ncDataset(gdir.get_filepath('gridded_data'), 'a') as nc:
vn = 'obs_icevel_x'
if vn in nc.variables:
v = nc.variables[vn]
else:
v = nc.createVariable(vn, 'f4', (
'y',
'x',
), zlib=True)
v.units = 'm yr-1'
v.long_name = 'ITS LIVE velocity data in x map direction'
v[:] = vx
vn = 'obs_icevel_y'
if vn in nc.variables:
v = nc.variables[vn]
else:
v = nc.createVariable(vn, 'f4', (
'y',
'x',
), zlib=True)
v.units = 'm yr-1'
v.long_name = 'ITS LIVE velocity data in xy map direction'
v[:] = vy
import scipy
import oggm
from oggm import cfg, utils, workflow, tasks
import pytest
from numpy.testing import assert_allclose
# import the MBsandbox modules
from MBsandbox.mbmod_daily_oneflowline import process_w5e5_data
from MBsandbox.help_func import melt_f_calib_geod_prep_inversion
from MBsandbox.flowline_TIModel import (run_from_climate_data_TIModel,
run_random_climate_TIModel)
# get the geodetic calibration data
url = 'https://cluster.klima.uni-bremen.de/~oggm/geodetic_ref_mb/hugonnet_2021_ds_rgi60_pergla_rates_10_20_worldwide.csv'
path = utils.file_downloader(url)
pd_geodetic = pd.read_csv(path, index_col='rgiid')
pd_geodetic = pd_geodetic.loc[pd_geodetic.period == '2000-01-01_2020-01-01']
DOM_BORDER = 80
ALL_DIAGS = ['volume', 'volume_bsl', 'volume_bwl', 'area', 'length',
'calving', 'calving_rate', 'off_area', 'on_area', 'melt_off_glacier',
'melt_on_glacier', 'liq_prcp_off_glacier', 'liq_prcp_on_glacier',
'snowfall_off_glacier', 'snowfall_on_glacier', 'model_mb',
'residual_mb', 'snow_bucket']
#@pytest.fixture(scope='class')
#def inversion_params(gdir):
# diag = gdir.get_diagnostics()
# return {k: diag[k] for k in ('inversion_glen_a', 'inversion_fs')}
def run_and_store_from_disk(rgi, histalp_storage, storage):
cmip = [
'CCSM4', 'CNRM-CM5', 'CSIRO-Mk3-6-0', 'CanESM2', 'GFDL-CM3',
'GFDL-ESM2G', 'GISS-E2-R', 'IPSL-CM5A-LR', 'MPI-ESM-LR', 'NorESM1-M'
]
bp = 'https://cluster.klima.uni-bremen.de/~oggm/cmip5-ng/pr/pr_mon_{}_{}_r1i1p1_g025.nc'
bt = 'https://cluster.klima.uni-bremen.de/~oggm/cmip5-ng/tas/tas_mon_{}_{}_r1i1p1_g025.nc'
for i in np.arange(999):
# Local working directory (where OGGM will write its output)
storage_dir = os.path.join(histalp_storage, rgi, '{:02d}'.format(i),
rgi[:8], rgi[:11], rgi)
new_dir = os.path.join(cfg.PATHS['working_dir'], 'per_glacier',
rgi[:8], rgi[:11], rgi)
# make sure directory is empty:
try:
shutil.rmtree(new_dir)
except FileNotFoundError:
pass
# if path does not exist, we handled all ensemble members:
try:
shutil.copytree(storage_dir, new_dir)
except FileNotFoundError:
log.info('processed {:02d} ensemble members'.format(i))
break
gdir = GlacierDirectory(rgi)
pdict = gdir.get_climate_info()['ensemble_calibration']
cfg.PARAMS['prcp_scaling_factor'] = pdict['prcp_scaling_factor']
default_glena = 2.4e-24
cfg.PARAMS['glen_a'] = pdict['glena_factor'] * default_glena
cfg.PARAMS['inversion_glen_a'] = pdict['glena_factor'] * default_glena
mbbias = pdict['mbbias']
tmp_mod = FileModel(
gdir.get_filepath('model_run',
filesuffix='_histalp_{:02d}'.format(i)))
tmp_mod.run_until(tmp_mod.last_yr)
for cm in cmip:
for rcp in ['rcp26', 'rcp45', 'rcp60', 'rcp85']:
ft = utils.file_downloader(bt.format(cm, rcp))
fp = utils.file_downloader(bp.format(cm, rcp))
if ft is None:
log.warning('no {} for model {}'.format(rcp, cm))
continue
filesuffix = '_{}_{}'.format(cm, rcp)
# bias correct them
if '_merged' in rgi:
process_cmip_for_merged_glacier(gdir, filesuffix, ft, fp)
else:
gcm_climate.process_cmip5_data(gdir,
filesuffix=filesuffix,
fpath_temp=ft,
fpath_precip=fp)
rid = '_{}_{}'.format(cm, rcp)
rid_out = '{}_{:02d}'.format(rid, i)
run_from_climate_data(gdir,
ys=2014,
ye=2100,
climate_filename='gcm_data',
climate_input_filesuffix=rid,
init_model_fls=tmp_mod.fls,
output_filesuffix=rid_out,
bias=mbbias)
fn1 = 'model_diagnostics{}.nc'.format(rid_out)
shutil.copyfile(
gdir.get_filepath('model_diagnostics', filesuffix=rid_out),
os.path.join(storage, fn1))
fn4 = 'model_run{}.nc'.format(rid_out)
shutil.copyfile(
gdir.get_filepath('model_run', filesuffix=rid_out),
os.path.join(storage, fn4))
