def up_to_climate(reset=False):
"""Run the tasks you want."""
# test directory
if not os.path.exists(TEST_DIR):
os.makedirs(TEST_DIR)
if reset:
clean_dir(TEST_DIR)
if not os.path.exists(CLI_LOGF):
with open(CLI_LOGF, 'wb') as f:
pickle.dump('none', f)
# Init
cfg.initialize()
# Use multiprocessing
cfg.PARAMS['use_multiprocessing'] = use_multiprocessing()
# Working dir
cfg.PATHS['working_dir'] = TEST_DIR
cfg.PATHS['dem_file'] = get_demo_file('srtm_oetztal.tif')
cfg.set_intersects_db(get_demo_file('rgi_intersect_oetztal.shp'))
# Read in the RGI file
rgi_file = get_demo_file('rgi_oetztal.shp')
rgidf = gpd.read_file(rgi_file)
# Be sure data is downloaded
utils.get_cru_cl_file()
# Params
cfg.PARAMS['border'] = 70
cfg.PARAMS['tstar_search_window'] = [1902, 0]
cfg.PARAMS['run_mb_calibration'] = True
# Go
gdirs = workflow.init_glacier_regions(rgidf)
try:
tasks.catchment_width_correction(gdirs[0])
except Exception:
reset = True
if reset:
# First preprocessing tasks
workflow.gis_prepro_tasks(gdirs)
return gdirs
def up_to_climate(reset=False):
"""Run the tasks you want."""
# test directory
if not os.path.exists(TEST_DIR):
os.makedirs(TEST_DIR)
if reset:
clean_dir(TEST_DIR)
if not os.path.exists(CLI_LOGF):
with open(CLI_LOGF, 'wb') as f:
pickle.dump('none', f)
# Init
cfg.initialize()
# Use multiprocessing
cfg.PARAMS['use_multiprocessing'] = use_multiprocessing()
# Working dir
cfg.PATHS['working_dir'] = TEST_DIR
cfg.PATHS['dem_file'] = get_demo_file('srtm_oetztal.tif')
cfg.set_intersects_db(get_demo_file('rgi_intersect_oetztal.shp'))
# Read in the RGI file
rgi_file = get_demo_file('rgi_oetztal.shp')
rgidf = gpd.read_file(rgi_file)
# Be sure data is downloaded
utils.get_cru_cl_file()
# Params
cfg.PARAMS['border'] = 70
cfg.PARAMS['tstar_search_window'] = [1902, 0]
cfg.PARAMS['run_mb_calibration'] = True
# Go
gdirs = workflow.init_glacier_regions(rgidf)
try:
tasks.catchment_width_correction(gdirs[0])
except Exception:
reset = True
if reset:
# First preprocessing tasks
workflow.gis_prepro_tasks(gdirs)
return gdirs
def up_to_climate(reset=False):
"""Run the tasks you want."""
# test directory
if not os.path.exists(TEST_DIR):
os.makedirs(TEST_DIR)
if reset:
clean_dir(TEST_DIR)
if not os.path.exists(CLI_LOGF):
with open(CLI_LOGF, 'wb') as f:
pickle.dump('none', f)
# Init
cfg.initialize()
# Use multiprocessing
# We don't use mp on TRAVIS because unsure if compatible with test coverage
cfg.PARAMS['use_multiprocessing'] = not ON_TRAVIS
# Working dir
cfg.PATHS['working_dir'] = TEST_DIR
cfg.PATHS['dem_file'] = get_demo_file('srtm_oetztal.tif')
# Read in the RGI file
rgi_file = get_demo_file('rgi_oetztal.shp')
rgidf = gpd.GeoDataFrame.from_file(rgi_file)
# Be sure data is downloaded because lock doesn't work
cl = utils.get_cru_cl_file()
# Params
cfg.PARAMS['border'] = 70
cfg.PARAMS['use_optimized_inversion_params'] = True
cfg.PARAMS['tstar_search_window'] = [1902, 0]
cfg.PARAMS['invert_with_rectangular'] = False
# Go
gdirs = workflow.init_glacier_regions(rgidf)
assert gdirs[14].name == 'Hintereisferner'
try:
tasks.catchment_width_correction(gdirs[0])
except Exception:
reset = True
if reset:
# First preprocessing tasks
workflow.gis_prepro_tasks(gdirs)
return gdirs
def process_cru_data(gdir):
"""Processes and writes the climate data for this glacier.
Interpolates the CRU TS data to the high-resolution CL2 climatologies
(provided with OGGM) and writes everything to a NetCDF file.
"""
# read the climatology
clfile = utils.get_cru_cl_file()
ncclim = salem.GeoNetcdf(clfile)
# and the TS data
nc_ts_tmp = salem.GeoNetcdf(utils.get_cru_file('tmp'), monthbegin=True)
nc_ts_pre = salem.GeoNetcdf(utils.get_cru_file('pre'), monthbegin=True)
# set temporal subset for the ts data (hydro years)
yrs = nc_ts_pre.time.year
y0, y1 = yrs[0], yrs[-1]
nc_ts_tmp.set_period(t0='{}-10-01'.format(y0), t1='{}-09-01'.format(y1))
nc_ts_pre.set_period(t0='{}-10-01'.format(y0), t1='{}-09-01'.format(y1))
time = nc_ts_pre.time
ny, r = divmod(len(time), 12)
assert r == 0
# gradient default params
use_grad = cfg.PARAMS['temp_use_local_gradient']
def_grad = cfg.PARAMS['temp_default_gradient']
g_minmax = cfg.PARAMS['temp_local_gradient_bounds']
lon = gdir.cenlon
lat = gdir.cenlat
# This is guaranteed to work because I prepared the file (I hope)
ncclim.set_subset(corners=((lon, lat), (lon, lat)), margin=1)
# get climatology data
loc_hgt = ncclim.get_vardata('elev')
loc_tmp = ncclim.get_vardata('temp')
loc_pre = ncclim.get_vardata('prcp')
loc_lon = ncclim.get_vardata('lon')
loc_lat = ncclim.get_vardata('lat')
# see if the center is ok
if not np.isfinite(loc_hgt[1, 1]):
# take another candidate where finite
isok = np.isfinite(loc_hgt)
# wait: some areas are entirely NaNs, make the subset larger
_margin = 1
while not np.any(isok):
_margin += 1
ncclim.set_subset(corners=((lon, lat), (lon, lat)), margin=_margin)
loc_hgt = ncclim.get_vardata('elev')
isok = np.isfinite(loc_hgt)
if _margin > 1:
log.debug('%s: I had to look up for far climate pixels: %s',
gdir.rgi_id, _margin)
# Take the first candidate (doesn't matter which)
lon, lat = ncclim.grid.ll_coordinates
lon = lon[isok][0]
lat = lat[isok][0]
# Resubset
ncclim.set_subset()
ncclim.set_subset(corners=((lon, lat), (lon, lat)), margin=1)
loc_hgt = ncclim.get_vardata('elev')
loc_tmp = ncclim.get_vardata('temp')
loc_pre = ncclim.get_vardata('prcp')
loc_lon = ncclim.get_vardata('lon')
loc_lat = ncclim.get_vardata('lat')
assert np.isfinite(loc_hgt[1, 1])
isok = np.isfinite(loc_hgt)
hgt_f = loc_hgt[isok].flatten()
assert len(hgt_f) > 0.
ts_grad = np.zeros(12) + def_grad
if use_grad and len(hgt_f) >= 5:
for i in range(12):
loc_tmp_mth = loc_tmp[i, ...][isok].flatten()
slope, _, _, p_val, _ = stats.linregress(hgt_f, loc_tmp_mth)
ts_grad[i] = slope if (p_val < 0.01) else def_grad
# ... but dont exaggerate too much
ts_grad = np.clip(ts_grad, g_minmax[0], g_minmax[1])
# convert to timeserie and hydroyears
ts_grad = ts_grad.tolist()
ts_grad = ts_grad[9:] + ts_grad[0:9]
ts_grad = np.asarray(ts_grad * ny)
# maybe this will throw out of bounds warnings
nc_ts_tmp.set_subset(corners=((lon, lat), (lon, lat)), margin=1)
nc_ts_pre.set_subset(corners=((lon, lat), (lon, lat)), margin=1)
# compute monthly anomalies
# of temp
ts_tmp = nc_ts_tmp.get_vardata('tmp', as_xarray=True)
ts_tmp_avg = ts_tmp.sel(time=slice('1961-01-01', '1990-12-01'))
ts_tmp_avg = ts_tmp_avg.groupby('time.month').mean(dim='time')
ts_tmp = ts_tmp.groupby('time.month') - ts_tmp_avg
# of precip
ts_pre = nc_ts_pre.get_vardata('pre', as_xarray=True)
ts_pre_avg = ts_pre.sel(time=slice('1961-01-01', '1990-12-01'))
ts_pre_avg = ts_pre_avg.groupby('time.month').mean(dim='time')
ts_pre = ts_pre.groupby('time.month') - ts_pre_avg
# interpolate to HR grid
if np.any(~np.isfinite(ts_tmp[:, 1, 1])):
# Extreme case, middle pix is not valid
# take any valid pix from the 3*3 (and hope there's one)
found_it = False
for idi in range(2):
for idj in range(2):
if np.all(np.isfinite(ts_tmp[:, idj, idi])):
ts_tmp[:, 1, 1] = ts_tmp[:, idj, idi]
ts_pre[:, 1, 1] = ts_pre[:, idj, idi]
found_it = True
if not found_it:
msg = '{}: OMG there is no climate data'.format(gdir.rgi_id)
raise RuntimeError(msg)
elif np.any(~np.isfinite(ts_tmp)):
# maybe the side is nan, but we can do nearest
ts_tmp = ncclim.grid.map_gridded_data(ts_tmp.values, nc_ts_tmp.grid,
interp='nearest')
ts_pre = ncclim.grid.map_gridded_data(ts_pre.values, nc_ts_pre.grid,
interp='nearest')
else:
# We can do bilinear
ts_tmp = ncclim.grid.map_gridded_data(ts_tmp.values, nc_ts_tmp.grid,
interp='linear')
ts_pre = ncclim.grid.map_gridded_data(ts_pre.values, nc_ts_pre.grid,
interp='linear')
# take the center pixel and add it to the CRU CL clim
# for temp
loc_tmp = xr.DataArray(loc_tmp[:, 1, 1], dims=['month'],
coords={'month':ts_tmp_avg.month})
ts_tmp = xr.DataArray(ts_tmp[:, 1, 1], dims=['time'],
coords={'time':time})
ts_tmp = ts_tmp.groupby('time.month') + loc_tmp
# for prcp
loc_pre = xr.DataArray(loc_pre[:, 1, 1], dims=['month'],
coords={'month':ts_pre_avg.month})
ts_pre = xr.DataArray(ts_pre[:, 1, 1], dims=['time'],
coords={'time':time})
ts_pre = ts_pre.groupby('time.month') + loc_pre
# done
loc_hgt = loc_hgt[1, 1]
loc_lon = loc_lon[1]
loc_lat = loc_lat[1]
assert np.isfinite(loc_hgt)
assert np.all(np.isfinite(ts_pre.values))
assert np.all(np.isfinite(ts_tmp.values))
assert np.all(np.isfinite(ts_grad))
gdir.write_monthly_climate_file(time, ts_pre.values, ts_tmp.values,
ts_grad, loc_hgt, loc_lon, loc_lat)
ncclim._nc.close()
nc_ts_tmp._nc.close()
nc_ts_pre._nc.close()
# metadata
out = {'climate_source': 'CRU data', 'hydro_yr_0': y0+1, 'hydro_yr_1': y1}
gdir.write_pickle(out, 'climate_info')
# Set to True for cluster runs
if Cluster:
cfg.PARAMS['use_multiprocessing'] = True
cfg.PARAMS['continue_on_error'] = True
else:
cfg.PARAMS['use_multiprocessing'] = False
cfg.PARAMS['continue_on_error'] = False
# We use intersects
# (this is slow, it could be replaced with a subset of the global file)
rgi_dir = utils.get_rgi_intersects_dir()
cfg.set_intersects_db(os.path.join(rgi_dir, '00_rgi50_AllRegs',
'intersects_rgi50_AllRegs.shp'))
# Pre-download other files which will be needed later
utils.get_cru_cl_file()
utils.get_cru_file(var='tmp')
utils.get_cru_file(var='pre')
# Some globals for more control on what to run
RUN_GIS_mask = False
RUN_GIS_PREPRO = False # run GIS pre-processing tasks (before climate)
RUN_CLIMATE_PREPRO = False # run climate pre-processing tasks
RUN_INVERSION = False # run bed inversion
# Read RGI file
rgidf = salem.read_shapefile(RGI_FILE, cached=True)
# get WGMS glaciers
flink, mbdatadir = utils.get_wgms_files()
ids_with_mb = flink['RGI50_ID'].values
def _distribute_cru_style_nonparallel(gdirs):
"""More general solution for OGGM globally.
It uses the CRU CL2 ten-minutes climatology as baseline
(provided with OGGM)
"""
# read the climatology
clfile = utils.get_cru_cl_file()
ncclim = salem.GeoNetcdf(clfile)
# and the TS data
nc_ts_tmp = salem.GeoNetcdf(utils.get_cru_file('tmp'), monthbegin=True)
nc_ts_pre = salem.GeoNetcdf(utils.get_cru_file('pre'), monthbegin=True)
# set temporal subset for the ts data (hydro years)
nc_ts_tmp.set_period(t0='1901-10-01', t1='2014-09-01')
nc_ts_pre.set_period(t0='1901-10-01', t1='2014-09-01')
time = nc_ts_pre.time
ny, r = divmod(len(time), 12)
assert r == 0
# gradient default params
use_grad = cfg.PARAMS['temp_use_local_gradient']
def_grad = cfg.PARAMS['temp_default_gradient']
g_minmax = cfg.PARAMS['temp_local_gradient_bounds']
prcp_scaling_factor = cfg.PARAMS['prcp_scaling_factor']
for gdir in gdirs:
log.info('%s: %s', gdir.rgi_id, 'distribute_cru_style')
lon = gdir.cenlon
lat = gdir.cenlat
# This is guaranteed to work because I prepared the file (I hope)
ncclim.set_subset(corners=((lon, lat), (lon, lat)), margin=1)
# get monthly gradient ...
loc_hgt = ncclim.get_vardata('elev')
loc_tmp = ncclim.get_vardata('temp')
loc_pre = ncclim.get_vardata('prcp')
isok = np.isfinite(loc_hgt)
hgt_f = loc_hgt[isok].flatten()
ts_grad = np.zeros(12) + def_grad
if use_grad and len(hgt_f) >= 5:
for i in range(12):
loc_tmp_mth = loc_tmp[i, ...][isok].flatten()
slope, _, _, p_val, _ = stats.linregress(hgt_f, loc_tmp_mth)
ts_grad[i] = slope if (p_val < 0.01) else def_grad
# ... but dont exaggerate too much
ts_grad = np.clip(ts_grad, g_minmax[0], g_minmax[1])
# convert to timeserie and hydroyears
ts_grad = ts_grad.tolist()
ts_grad = ts_grad[9:] + ts_grad[0:9]
ts_grad = np.asarray(ts_grad * ny)
# maybe this will throw out of bounds warnings
nc_ts_tmp.set_subset(corners=((lon, lat), (lon, lat)), margin=1)
nc_ts_pre.set_subset(corners=((lon, lat), (lon, lat)), margin=1)
# compute monthly anomalies
# of temp
ts_tmp = nc_ts_tmp.get_vardata('tmp', as_xarray=True)
ts_tmp_avg = ts_tmp.sel(time=slice('1961-01-01', '1990-12-01'))
ts_tmp_avg = ts_tmp_avg.groupby('time.month').mean(dim='time')
ts_tmp = ts_tmp.groupby('time.month') - ts_tmp_avg
# of precip
ts_pre = nc_ts_pre.get_vardata('pre', as_xarray=True)
ts_pre_avg = ts_pre.sel(time=slice('1961-01-01', '1990-12-01'))
ts_pre_avg = ts_pre_avg.groupby('time.month').mean(dim='time')
ts_pre = ts_pre.groupby('time.month') - ts_pre_avg
# interpolate to HR grid
if np.any(~np.isfinite(ts_tmp[:, 1, 1])):
# Extreme case, middle pix is not valid
# take any valid pix from the 3*3 (and hope theres one)
found_it = False
for idi in range(2):
for idj in range(2):
if np.all(np.isfinite(ts_tmp[:, idj, idi])):
ts_tmp[:, 1, 1] = ts_tmp[:, idj, idi]
ts_pre[:, 1, 1] = ts_pre[:, idj, idi]
found_it = True
if not found_it:
msg = '{}: OMG there is no climate data'.format(gdir.rgi_id)
raise RuntimeError(msg)
elif np.any(~np.isfinite(ts_tmp)):
# maybe the side is nan, but we can do nearest
ts_tmp = ncclim.grid.map_gridded_data(ts_tmp.values,
nc_ts_tmp.grid,
interp='nearest')
ts_pre = ncclim.grid.map_gridded_data(ts_pre.values,
nc_ts_pre.grid,
interp='nearest')
else:
# We can do bilinear
ts_tmp = ncclim.grid.map_gridded_data(ts_tmp.values,
nc_ts_tmp.grid,
interp='linear')
ts_pre = ncclim.grid.map_gridded_data(ts_pre.values,
nc_ts_pre.grid,
interp='linear')
# take the center pixel and add it to the CRU CL clim
# for temp
loc_tmp = xr.DataArray(loc_tmp[:, 1, 1],
dims=['month'],
coords={'month': ts_pre_avg.month})
ts_tmp = xr.DataArray(ts_tmp[:, 1, 1],
dims=['time'],
coords={'time': time})
ts_tmp = ts_tmp.groupby('time.month') + loc_tmp
# for prcp
loc_pre = xr.DataArray(loc_pre[:, 1, 1],
dims=['month'],
coords={'month': ts_pre_avg.month})
ts_pre = xr.DataArray(ts_pre[:, 1, 1],
dims=['time'],
coords={'time': time})
ts_pre = ts_pre.groupby('time.month') + loc_pre * prcp_scaling_factor
# done
loc_hgt = loc_hgt[1, 1]
assert np.isfinite(loc_hgt)
assert np.all(np.isfinite(ts_pre.values))
assert np.all(np.isfinite(ts_tmp.values))
assert np.all(np.isfinite(ts_grad))
gdir.write_monthly_climate_file(time, ts_pre.values, ts_tmp.values,
ts_grad, loc_hgt)
def up_to_inversion(reset=False):
"""Run the tasks you want."""
# test directory
if not os.path.exists(TEST_DIR):
os.makedirs(TEST_DIR)
if reset:
clean_dir(TEST_DIR)
# Init
cfg.initialize()
# Use multiprocessing
cfg.PARAMS['use_multiprocessing'] = not ON_TRAVIS
# Working dir
cfg.PATHS['working_dir'] = TEST_DIR
cfg.PATHS['dem_file'] = get_demo_file('srtm_oetztal.tif')
# Set up the paths and other stuffs
cfg.set_divides_db(get_demo_file('divides_workflow.shp'))
cfg.PATHS['wgms_rgi_links'] = get_demo_file('RGI_WGMS_oetztal.csv')
cfg.PATHS['glathida_rgi_links'] = get_demo_file('RGI_GLATHIDA_oetztal.csv')
# Read in the RGI file
rgi_file = get_demo_file('rgi_oetztal.shp')
rgidf = gpd.GeoDataFrame.from_file(rgi_file)
# Be sure data is downloaded because lock doesn't work
cl = utils.get_cru_cl_file()
# Params
cfg.PARAMS['border'] = 70
cfg.PARAMS['use_optimized_inversion_params'] = True
# Go
gdirs = workflow.init_glacier_regions(rgidf)
try:
flowline.init_present_time_glacier(gdirs[0])
except Exception:
reset = True
if reset:
# First preprocessing tasks
workflow.gis_prepro_tasks(gdirs)
# Climate related tasks
# See if CRU is running
cfg.PARAMS['temp_use_local_gradient'] = False
cfg.PATHS['climate_file'] = '~'
cru_dir = get_demo_file('cru_ts3.23.1901.2014.tmp.dat.nc')
cfg.PATHS['cru_dir'] = os.path.dirname(cru_dir)
with warnings.catch_warnings():
# There is a warning from salem
warnings.simplefilter("ignore")
workflow.execute_entity_task(tasks.distribute_cru_style, gdirs)
tasks.compute_ref_t_stars(gdirs)
tasks.distribute_t_stars(gdirs)
# Use histalp for the actual test
cfg.PARAMS['temp_use_local_gradient'] = True
cfg.PATHS['climate_file'] = get_demo_file('HISTALP_oetztal.nc')
cfg.PATHS['cru_dir'] = '~'
workflow.climate_tasks(gdirs)
# Inversion
workflow.inversion_tasks(gdirs)
return gdirs
def initialize(file=None, logging_level='INFO'):
"""Read the configuration file containing the run's parameters.
This should be the first call, before using any of the other OGGM modules
for most (all?) OGGM simulations.
Parameters
----------
file : str
path to the configuration file (default: OGGM params.cfg)
logging_level : str
set a logging level. See :func:`set_logging_config` for options.
"""
global IS_INITIALIZED
global PARAMS
global PATHS
global DATA
set_logging_config(logging_level=logging_level)
if file is None:
file = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'params.cfg')
try:
cp = ConfigObj(file, file_error=True)
except (ConfigObjError, IOError) as e:
log.critical('Config file could not be parsed (%s): %s', file, e)
sys.exit()
log.workflow('Using configuration file: %s', file)
# Paths
oggm_static_paths()
PATHS['working_dir'] = cp['working_dir']
PATHS['dem_file'] = cp['dem_file']
PATHS['climate_file'] = cp['climate_file']
# Multiprocessing pool
PARAMS['use_multiprocessing'] = cp.as_bool('use_multiprocessing')
PARAMS['mp_processes'] = cp.as_int('mp_processes')
# Some non-trivial params
PARAMS['continue_on_error'] = cp.as_bool('continue_on_error')
PARAMS['grid_dx_method'] = cp['grid_dx_method']
PARAMS['topo_interp'] = cp['topo_interp']
PARAMS['use_intersects'] = cp.as_bool('use_intersects')
PARAMS['use_compression'] = cp.as_bool('use_compression')
PARAMS['mpi_recv_buf_size'] = cp.as_int('mpi_recv_buf_size')
PARAMS['use_multiple_flowlines'] = cp.as_bool('use_multiple_flowlines')
PARAMS['filter_min_slope'] = cp.as_bool('filter_min_slope')
PARAMS['auto_skip_task'] = cp.as_bool('auto_skip_task')
PARAMS['correct_for_neg_flux'] = cp.as_bool('correct_for_neg_flux')
PARAMS['filter_for_neg_flux'] = cp.as_bool('filter_for_neg_flux')
PARAMS['run_mb_calibration'] = cp.as_bool('run_mb_calibration')
PARAMS['rgi_version'] = cp['rgi_version']
PARAMS['use_rgi_area'] = cp.as_bool('use_rgi_area')
PARAMS['compress_climate_netcdf'] = cp.as_bool('compress_climate_netcdf')
PARAMS['use_tar_shapefiles'] = cp.as_bool('use_tar_shapefiles')
PARAMS['clip_mu_star'] = cp.as_bool('clip_mu_star')
PARAMS['clip_tidewater_border'] = cp.as_bool('clip_tidewater_border')
PARAMS['dl_verify'] = cp.as_bool('dl_verify')
# Climate
PARAMS['baseline_climate'] = cp['baseline_climate'].strip().upper()
PARAMS['baseline_y0'] = cp.as_int('baseline_y0')
PARAMS['baseline_y1'] = cp.as_int('baseline_y1')
PARAMS['hydro_month_nh'] = cp.as_int('hydro_month_nh')
PARAMS['hydro_month_sh'] = cp.as_int('hydro_month_sh')
PARAMS['temp_use_local_gradient'] = cp.as_bool('temp_use_local_gradient')
PARAMS['tstar_search_glacierwide'] = cp.as_bool('tstar_search_glacierwide')
k = 'temp_local_gradient_bounds'
PARAMS[k] = [float(vk) for vk in cp.as_list(k)]
k = 'tstar_search_window'
PARAMS[k] = [int(vk) for vk in cp.as_list(k)]
PARAMS['use_bias_for_run'] = cp.as_bool('use_bias_for_run')
# Inversion
k = 'use_shape_factor_for_inversion'
PARAMS[k] = cp[k]
# Flowline model
k = 'use_shape_factor_for_fluxbasedmodel'
PARAMS[k] = cp[k]
# Make sure we have a proper cache dir
from oggm.utils import download_oggm_files, get_demo_file
download_oggm_files()
# Delete non-floats
ltr = [
'working_dir', 'dem_file', 'climate_file', 'use_tar_shapefiles',
'grid_dx_method', 'run_mb_calibration', 'compress_climate_netcdf',
'mp_processes', 'use_multiprocessing', 'baseline_y0', 'baseline_y1',
'temp_use_local_gradient', 'temp_local_gradient_bounds', 'topo_interp',
'use_compression', 'bed_shape', 'continue_on_error',
'use_multiple_flowlines', 'tstar_search_glacierwide',
'mpi_recv_buf_size', 'hydro_month_nh', 'clip_mu_star',
'tstar_search_window', 'use_bias_for_run', 'hydro_month_sh',
'use_intersects', 'filter_min_slope', 'clip_tidewater_border',
'auto_skip_task', 'correct_for_neg_flux', 'filter_for_neg_flux',
'rgi_version', 'dl_verify', 'use_shape_factor_for_inversion',
'use_rgi_area', 'use_shape_factor_for_fluxbasedmodel',
'baseline_climate'
]
for k in ltr:
cp.pop(k, None)
# Other params are floats
for k in cp:
PARAMS[k] = cp.as_float(k)
# Read-in the reference t* data for all available models types (oggm, vas)
model_prefixes = ['oggm_', 'vas_']
for prefix in model_prefixes:
fns = [
'ref_tstars_rgi5_cru4', 'ref_tstars_rgi6_cru4',
'ref_tstars_rgi5_histalp', 'ref_tstars_rgi6_histalp'
]
for fn in fns:
fpath = get_demo_file(prefix + fn + '.csv')
PARAMS[prefix + fn] = pd.read_csv(fpath)
fpath = get_demo_file(prefix + fn + '_calib_params.json')
with open(fpath, 'r') as fp:
mbpar = json.load(fp)
PARAMS[prefix + fn + '_calib_params'] = mbpar
# Empty defaults
set_intersects_db()
IS_INITIALIZED = True
# Pre extract cru cl to avoid problems by multiproc
from oggm.utils import get_cru_cl_file
get_cru_cl_file()
# Read in the demo glaciers
file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data',
'demo_glaciers.csv')
DATA['demo_glaciers'] = pd.read_csv(file, index_col=0)
# Add other things
if 'dem_grids' not in DATA:
grids = {}
for grid_json in [
'gimpdem_90m_v01.1.json', 'arcticdem_mosaic_100m_v3.0.json',
'AntarcticDEM_wgs84.json', 'REMA_100m_dem.json'
]:
if grid_json not in grids:
fp = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'data', grid_json)
try:
grids[grid_json] = salem.Grid.from_json(fp)
except NameError:
pass
DATA['dem_grids'] = grids
def initialize(file=None):
"""Read the configuration file containing the run's parameters."""
global IS_INITIALIZED
global PARAMS
global PATHS
if file is None:
file = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'params.cfg')
log.info('Parameter file: %s', file)
try:
cp = ConfigObj(file, file_error=True)
except (ConfigObjError, IOError) as e:
log.critical('Param file could not be parsed (%s): %s', file, e)
sys.exit()
# Paths
oggm_static_paths()
PATHS['working_dir'] = cp['working_dir']
PATHS['dem_file'] = cp['dem_file']
PATHS['climate_file'] = cp['climate_file']
# Multiprocessing pool
PARAMS['use_multiprocessing'] = cp.as_bool('use_multiprocessing')
PARAMS['mp_processes'] = cp.as_int('mp_processes')
# Some non-trivial params
PARAMS['continue_on_error'] = cp.as_bool('continue_on_error')
PARAMS['grid_dx_method'] = cp['grid_dx_method']
PARAMS['topo_interp'] = cp['topo_interp']
PARAMS['use_intersects'] = cp.as_bool('use_intersects')
PARAMS['use_compression'] = cp.as_bool('use_compression')
PARAMS['mpi_recv_buf_size'] = cp.as_int('mpi_recv_buf_size')
PARAMS['use_multiple_flowlines'] = cp.as_bool('use_multiple_flowlines')
PARAMS['optimize_thick'] = cp.as_bool('optimize_thick')
PARAMS['filter_min_slope'] = cp.as_bool('filter_min_slope')
PARAMS['auto_skip_task'] = cp.as_bool('auto_skip_task')
PARAMS['correct_for_neg_flux'] = cp.as_bool('correct_for_neg_flux')
PARAMS['filter_for_neg_flux'] = cp.as_bool('filter_for_neg_flux')
PARAMS['run_mb_calibration'] = cp.as_bool('run_mb_calibration')
PARAMS['rgi_version'] = cp['rgi_version']
PARAMS['hydro_month_nh'] = cp.as_int('hydro_month_nh')
PARAMS['hydro_month_sh'] = cp.as_int('hydro_month_sh')
PARAMS['use_rgi_area'] = cp.as_bool('use_rgi_area')
# Climate
PARAMS['temp_use_local_gradient'] = cp.as_bool('temp_use_local_gradient')
k = 'temp_local_gradient_bounds'
PARAMS[k] = [float(vk) for vk in cp.as_list(k)]
k = 'tstar_search_window'
PARAMS[k] = [int(vk) for vk in cp.as_list(k)]
PARAMS['use_bias_for_run'] = cp.as_bool('use_bias_for_run')
_factor = cp['prcp_scaling_factor']
if _factor not in ['stddev', 'stddev_perglacier']:
_factor = cp.as_float('prcp_scaling_factor')
PARAMS['prcp_scaling_factor'] = _factor
PARAMS['allow_negative_mustar'] = cp.as_bool('allow_negative_mustar')
# Inversion
PARAMS['invert_with_sliding'] = cp.as_bool('invert_with_sliding')
_k = 'optimize_inversion_params'
PARAMS[_k] = cp.as_bool(_k)
PARAMS['use_shape_factor_for_inversion'] = \
cp['use_shape_factor_for_inversion']
# Flowline model
_k = 'use_optimized_inversion_params'
PARAMS[_k] = cp.as_bool(_k)
PARAMS['use_shape_factor_for_fluxbasedmodel'] = \
cp['use_shape_factor_for_fluxbasedmodel']
# Make sure we have a proper cache dir
from oggm.utils import download_oggm_files, SAMPLE_DATA_COMMIT
download_oggm_files()
# Delete non-floats
ltr = [
'working_dir', 'dem_file', 'climate_file', 'grid_dx_method',
'run_mb_calibration', 'mp_processes', 'use_multiprocessing',
'temp_use_local_gradient', 'temp_local_gradient_bounds', 'topo_interp',
'use_compression', 'bed_shape', 'continue_on_error',
'use_optimized_inversion_params', 'invert_with_sliding',
'optimize_inversion_params', 'use_multiple_flowlines',
'optimize_thick', 'mpi_recv_buf_size', 'hydro_month_nh',
'tstar_search_window', 'use_bias_for_run', 'hydro_month_sh',
'prcp_scaling_factor', 'use_intersects', 'filter_min_slope',
'auto_skip_task', 'correct_for_neg_flux', 'filter_for_neg_flux',
'rgi_version', 'allow_negative_mustar',
'use_shape_factor_for_inversion', 'use_rgi_area',
'use_shape_factor_for_fluxbasedmodel'
]
for k in ltr:
cp.pop(k, None)
# Other params are floats
for k in cp:
PARAMS[k] = cp.as_float(k)
# Empty defaults
set_intersects_db()
IS_INITIALIZED = True
# Pre extract cru cl to avoid problems by multiproc
from oggm.utils import get_cru_cl_file
get_cru_cl_file()
def setup_cache(self):
setattr(full_workflow.setup_cache, "timeout", 360)
utils.mkdir(self.testdir, reset=True)
self.cfg_init()
# Pre-download other files which will be needed later
utils.get_cru_cl_file()
utils.get_cru_file(var='tmp')
utils.get_cru_file(var='pre')
# Get the RGI glaciers for the run.
rgi_list = ['RGI60-01.10299', 'RGI60-11.00897', 'RGI60-18.02342']
rgidf = utils.get_rgi_glacier_entities(rgi_list)
# We use intersects
db = utils.get_rgi_intersects_region_file(version='61',
rgi_ids=rgi_list)
cfg.set_intersects_db(db)
# Sort for more efficient parallel computing
rgidf = rgidf.sort_values('Area', ascending=False)
# Go - initialize working directories
gdirs = workflow.init_glacier_regions(rgidf)
# Preprocessing tasks
task_list = [
tasks.glacier_masks,
tasks.compute_centerlines,
tasks.initialize_flowlines,
tasks.compute_downstream_line,
tasks.compute_downstream_bedshape,
tasks.catchment_area,
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
]
for task in task_list:
execute_entity_task(task, gdirs)
# Climate tasks -- only data IO and tstar interpolation!
execute_entity_task(tasks.process_cru_data, gdirs)
execute_entity_task(tasks.local_mustar, gdirs)
execute_entity_task(tasks.apparent_mb, gdirs)
# Inversion tasks
execute_entity_task(tasks.prepare_for_inversion, gdirs)
# We use the default parameters for this run
execute_entity_task(tasks.mass_conservation_inversion, gdirs)
execute_entity_task(tasks.filter_inversion_output, gdirs)
# Final preparation for the run
execute_entity_task(tasks.init_present_time_glacier, gdirs)
# Random climate representative for the tstar climate, without bias
# In an ideal world this would imply that the glaciers remain stable,
# but it doesn't have to be so
execute_entity_task(tasks.run_constant_climate,
gdirs,
bias=0,
nyears=100,
output_filesuffix='_tstar')
execute_entity_task(tasks.run_constant_climate,
gdirs,
y0=1990,
nyears=100,
output_filesuffix='_pd')
# Compile output
utils.glacier_characteristics(gdirs)
utils.compile_run_output(gdirs, filesuffix='_tstar')
utils.compile_run_output(gdirs, filesuffix='_pd')
utils.compile_climate_input(gdirs)
return gdirs
def process_cru_data(gdir):
"""Processes and writes the climate data for this glacier.
Interpolates the CRU TS data to the high-resolution CL2 climatologies
(provided with OGGM) and writes everything to a NetCDF file.
"""
if cfg.PARAMS['baseline_climate'] != 'CRU':
raise ValueError("cfg.PARAMS['baseline_climate'] should be set to CRU")
# read the climatology
clfile = utils.get_cru_cl_file()
ncclim = salem.GeoNetcdf(clfile)
# and the TS data
nc_ts_tmp = salem.GeoNetcdf(utils.get_cru_file('tmp'), monthbegin=True)
nc_ts_pre = salem.GeoNetcdf(utils.get_cru_file('pre'), monthbegin=True)
# set temporal subset for the ts data (hydro years)
sm = cfg.PARAMS['hydro_month_' + gdir.hemisphere]
em = sm - 1 if (sm > 1) else 12
yrs = nc_ts_pre.time.year
y0, y1 = yrs[0], yrs[-1]
if cfg.PARAMS['baseline_y0'] != 0:
y0 = cfg.PARAMS['baseline_y0']
if cfg.PARAMS['baseline_y1'] != 0:
y1 = cfg.PARAMS['baseline_y1']
nc_ts_tmp.set_period(t0='{}-{:02d}-01'.format(y0, sm),
t1='{}-{:02d}-01'.format(y1, em))
nc_ts_pre.set_period(t0='{}-{:02d}-01'.format(y0, sm),
t1='{}-{:02d}-01'.format(y1, em))
time = nc_ts_pre.time
ny, r = divmod(len(time), 12)
assert r == 0
lon = gdir.cenlon
lat = gdir.cenlat
# This is guaranteed to work because I prepared the file (I hope)
ncclim.set_subset(corners=((lon, lat), (lon, lat)), margin=1)
# get climatology data
loc_hgt = ncclim.get_vardata('elev')
loc_tmp = ncclim.get_vardata('temp')
loc_pre = ncclim.get_vardata('prcp')
loc_lon = ncclim.get_vardata('lon')
loc_lat = ncclim.get_vardata('lat')
# see if the center is ok
if not np.isfinite(loc_hgt[1, 1]):
# take another candidate where finite
isok = np.isfinite(loc_hgt)
# wait: some areas are entirely NaNs, make the subset larger
_margin = 1
while not np.any(isok):
_margin += 1
ncclim.set_subset(corners=((lon, lat), (lon, lat)), margin=_margin)
loc_hgt = ncclim.get_vardata('elev')
isok = np.isfinite(loc_hgt)
if _margin > 1:
log.debug('(%s) I had to look up for far climate pixels: %s',
gdir.rgi_id, _margin)
# Take the first candidate (doesn't matter which)
lon, lat = ncclim.grid.ll_coordinates
lon = lon[isok][0]
lat = lat[isok][0]
# Resubset
ncclim.set_subset()
ncclim.set_subset(corners=((lon, lat), (lon, lat)), margin=1)
loc_hgt = ncclim.get_vardata('elev')
loc_tmp = ncclim.get_vardata('temp')
loc_pre = ncclim.get_vardata('prcp')
loc_lon = ncclim.get_vardata('lon')
loc_lat = ncclim.get_vardata('lat')
assert np.isfinite(loc_hgt[1, 1])
isok = np.isfinite(loc_hgt)
hgt_f = loc_hgt[isok].flatten()
assert len(hgt_f) > 0.
# Should we compute the gradient?
use_grad = cfg.PARAMS['temp_use_local_gradient']
ts_grad = None
if use_grad and len(hgt_f) >= 5:
ts_grad = np.zeros(12) * np.NaN
for i in range(12):
loc_tmp_mth = loc_tmp[i, ...][isok].flatten()
slope, _, _, p_val, _ = stats.linregress(hgt_f, loc_tmp_mth)
ts_grad[i] = slope if (p_val < 0.01) else np.NaN
# convert to a timeseries and hydrological years
ts_grad = ts_grad.tolist()
ts_grad = ts_grad[em:] + ts_grad[0:em]
ts_grad = np.asarray(ts_grad * ny)
# maybe this will throw out of bounds warnings
nc_ts_tmp.set_subset(corners=((lon, lat), (lon, lat)), margin=1)
nc_ts_pre.set_subset(corners=((lon, lat), (lon, lat)), margin=1)
# compute monthly anomalies
# of temp
ts_tmp = nc_ts_tmp.get_vardata('tmp', as_xarray=True)
ts_tmp_avg = ts_tmp.sel(time=slice('1961-01-01', '1990-12-01'))
ts_tmp_avg = ts_tmp_avg.groupby('time.month').mean(dim='time')
ts_tmp = ts_tmp.groupby('time.month') - ts_tmp_avg
# of precip
ts_pre = nc_ts_pre.get_vardata('pre', as_xarray=True)
ts_pre_avg = ts_pre.sel(time=slice('1961-01-01', '1990-12-01'))
ts_pre_avg = ts_pre_avg.groupby('time.month').mean(dim='time')
ts_pre_ano = ts_pre.groupby('time.month') - ts_pre_avg
# scaled anomalies is the default. Standard anomalies above
# are used later for where ts_pre_avg == 0
ts_pre = ts_pre.groupby('time.month') / ts_pre_avg
# interpolate to HR grid
if np.any(~np.isfinite(ts_tmp[:, 1, 1])):
# Extreme case, middle pix is not valid
# take any valid pix from the 3*3 (and hope there's one)
found_it = False
for idi in range(2):
for idj in range(2):
if np.all(np.isfinite(ts_tmp[:, idj, idi])):
ts_tmp[:, 1, 1] = ts_tmp[:, idj, idi]
ts_pre[:, 1, 1] = ts_pre[:, idj, idi]
ts_pre_ano[:, 1, 1] = ts_pre_ano[:, idj, idi]
found_it = True
if not found_it:
msg = '({}) there is no climate data'.format(gdir.rgi_id)
raise RuntimeError(msg)
elif np.any(~np.isfinite(ts_tmp)):
# maybe the side is nan, but we can do nearest
ts_tmp = ncclim.grid.map_gridded_data(ts_tmp.values, nc_ts_tmp.grid,
interp='nearest')
ts_pre = ncclim.grid.map_gridded_data(ts_pre.values, nc_ts_pre.grid,
interp='nearest')
ts_pre_ano = ncclim.grid.map_gridded_data(ts_pre_ano.values,
nc_ts_pre.grid,
interp='nearest')
else:
# We can do bilinear
ts_tmp = ncclim.grid.map_gridded_data(ts_tmp.values, nc_ts_tmp.grid,
interp='linear')
ts_pre = ncclim.grid.map_gridded_data(ts_pre.values, nc_ts_pre.grid,
interp='linear')
ts_pre_ano = ncclim.grid.map_gridded_data(ts_pre_ano.values,
nc_ts_pre.grid,
interp='linear')
# take the center pixel and add it to the CRU CL clim
# for temp
loc_tmp = xr.DataArray(loc_tmp[:, 1, 1], dims=['month'],
coords={'month': ts_tmp_avg.month})
ts_tmp = xr.DataArray(ts_tmp[:, 1, 1], dims=['time'],
coords={'time': time})
ts_tmp = ts_tmp.groupby('time.month') + loc_tmp
# for prcp
loc_pre = xr.DataArray(loc_pre[:, 1, 1], dims=['month'],
coords={'month': ts_pre_avg.month})
ts_pre = xr.DataArray(ts_pre[:, 1, 1], dims=['time'],
coords={'time': time})
ts_pre_ano = xr.DataArray(ts_pre_ano[:, 1, 1], dims=['time'],
coords={'time': time})
# scaled anomalies
ts_pre = ts_pre.groupby('time.month') * loc_pre
# standard anomalies
ts_pre_ano = ts_pre_ano.groupby('time.month') + loc_pre
# Correct infinite values with standard anomalies
ts_pre.values = np.where(np.isfinite(ts_pre.values),
ts_pre.values,
ts_pre_ano.values)
# The last step might create negative values (unlikely). Clip them
ts_pre.values = ts_pre.values.clip(0)
# done
loc_hgt = loc_hgt[1, 1]
loc_lon = loc_lon[1]
loc_lat = loc_lat[1]
assert np.isfinite(loc_hgt)
assert np.all(np.isfinite(ts_pre.values))
assert np.all(np.isfinite(ts_tmp.values))
gdir.write_monthly_climate_file(time, ts_pre.values, ts_tmp.values,
loc_hgt, loc_lon, loc_lat,
gradient=ts_grad)
source = nc_ts_tmp._nc.title[:10]
ncclim._nc.close()
nc_ts_tmp._nc.close()
nc_ts_pre._nc.close()
# metadata
out = {'baseline_climate_source': source,
'baseline_hydro_yr_0': y0+1,
'baseline_hydro_yr_1': y1}
gdir.write_pickle(out, 'climate_info')
def _distribute_cru_style_nonparallel(gdirs):
"""More general solution for OGGM globally.
It uses the CRU CL2 ten-minutes climatology as baseline
(provided with OGGM)
"""
# read the climatology
clfile = utils.get_cru_cl_file()
ncclim = salem.GeoNetcdf(clfile)
# and the TS data
nc_ts_tmp = salem.GeoNetcdf(utils.get_cru_file('tmp'), monthbegin=True)
nc_ts_pre = salem.GeoNetcdf(utils.get_cru_file('pre'), monthbegin=True)
# set temporal subset for the ts data (hydro years)
nc_ts_tmp.set_period(t0='1901-10-01', t1='2014-09-01')
nc_ts_pre.set_period(t0='1901-10-01', t1='2014-09-01')
time = nc_ts_pre.time
ny, r = divmod(len(time), 12)
assert r == 0
# gradient default params
use_grad = cfg.PARAMS['temp_use_local_gradient']
def_grad = cfg.PARAMS['temp_default_gradient']
g_minmax = cfg.PARAMS['temp_local_gradient_bounds']
prcp_scaling_factor = cfg.PARAMS['prcp_scaling_factor']
for gdir in gdirs:
log.info('%s: %s', gdir.rgi_id, 'distribute_cru_style')
lon = gdir.cenlon
lat = gdir.cenlat
# This is guaranteed to work because I prepared the file (I hope)
ncclim.set_subset(corners=((lon, lat), (lon, lat)), margin=1)
# get monthly gradient ...
loc_hgt = ncclim.get_vardata('elev')
loc_tmp = ncclim.get_vardata('temp')
loc_pre = ncclim.get_vardata('prcp')
isok = np.isfinite(loc_hgt)
hgt_f = loc_hgt[isok].flatten()
ts_grad = np.zeros(12) + def_grad
if use_grad and len(hgt_f) >= 5:
for i in range(12):
loc_tmp_mth = loc_tmp[i, ...][isok].flatten()
slope, _, _, p_val, _ = stats.linregress(hgt_f, loc_tmp_mth)
ts_grad[i] = slope if (p_val < 0.01) else def_grad
# ... but dont exaggerate too much
ts_grad = np.clip(ts_grad, g_minmax[0], g_minmax[1])
# convert to timeserie and hydroyears
ts_grad = ts_grad.tolist()
ts_grad = ts_grad[9:] + ts_grad[0:9]
ts_grad = np.asarray(ts_grad * ny)
# maybe this will throw out of bounds warnings
nc_ts_tmp.set_subset(corners=((lon, lat), (lon, lat)), margin=1)
nc_ts_pre.set_subset(corners=((lon, lat), (lon, lat)), margin=1)
# compute monthly anomalies
# of temp
ts_tmp = nc_ts_tmp.get_vardata('tmp', as_xarray=True)
ts_tmp_avg = ts_tmp.sel(time=slice('1961-01-01', '1990-12-01'))
ts_tmp_avg = ts_tmp_avg.groupby('time.month').mean(dim='time')
ts_tmp = ts_tmp.groupby('time.month') - ts_tmp_avg
# of precip
ts_pre = nc_ts_pre.get_vardata('pre', as_xarray=True)
ts_pre_avg = ts_pre.sel(time=slice('1961-01-01', '1990-12-01'))
ts_pre_avg = ts_pre_avg.groupby('time.month').mean(dim='time')
ts_pre = ts_pre.groupby('time.month') - ts_pre_avg
# interpolate to HR grid
if np.any(~np.isfinite(ts_tmp[:, 1, 1])):
# Extreme case, middle pix is not valid
# take any valid pix from the 3*3 (and hope theres one)
found_it = False
for idi in range(2):
for idj in range(2):
if np.all(np.isfinite(ts_tmp[:, idj, idi])):
ts_tmp[:, 1, 1] = ts_tmp[:, idj, idi]
ts_pre[:, 1, 1] = ts_pre[:, idj, idi]
found_it = True
if not found_it:
msg = '{}: OMG there is no climate data'.format(gdir.rgi_id)
raise RuntimeError(msg)
elif np.any(~np.isfinite(ts_tmp)):
# maybe the side is nan, but we can do nearest
ts_tmp = ncclim.grid.map_gridded_data(ts_tmp.values, nc_ts_tmp.grid,
interp='nearest')
ts_pre = ncclim.grid.map_gridded_data(ts_pre.values, nc_ts_pre.grid,
interp='nearest')
else:
# We can do bilinear
ts_tmp = ncclim.grid.map_gridded_data(ts_tmp.values, nc_ts_tmp.grid,
interp='linear')
ts_pre = ncclim.grid.map_gridded_data(ts_pre.values, nc_ts_pre.grid,
interp='linear')
# take the center pixel and add it to the CRU CL clim
# for temp
loc_tmp = xr.DataArray(loc_tmp[:, 1, 1], dims=['month'],
coords={'month':ts_pre_avg.month})
ts_tmp = xr.DataArray(ts_tmp[:, 1, 1], dims=['time'],
coords={'time':time})
ts_tmp = ts_tmp.groupby('time.month') + loc_tmp
# for prcp
loc_pre = xr.DataArray(loc_pre[:, 1, 1], dims=['month'],
coords={'month':ts_pre_avg.month})
ts_pre = xr.DataArray(ts_pre[:, 1, 1], dims=['time'],
coords={'time':time})
ts_pre = ts_pre.groupby('time.month') + loc_pre * prcp_scaling_factor
# done
loc_hgt = loc_hgt[1, 1]
assert np.isfinite(loc_hgt)
assert np.all(np.isfinite(ts_pre.values))
assert np.all(np.isfinite(ts_tmp.values))
assert np.all(np.isfinite(ts_grad))
gdir.write_monthly_climate_file(time, ts_pre.values, ts_tmp.values,
ts_grad, loc_hgt)
def initialize(file=None):
"""Read the configuration file containing the run's parameters."""
global IS_INITIALIZED
global PARAMS
global PATHS
if file is None:
file = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'params.cfg')
log.info('Parameter file: %s', file)
try:
cp = ConfigObj(file, file_error=True)
except (ConfigObjError, IOError) as e:
log.critical('Param file could not be parsed (%s): %s', file, e)
sys.exit()
# Paths
oggm_static_paths()
PATHS['working_dir'] = cp['working_dir']
PATHS['dem_file'] = cp['dem_file']
PATHS['climate_file'] = cp['climate_file']
# Multiprocessing pool
PARAMS['use_multiprocessing'] = cp.as_bool('use_multiprocessing')
PARAMS['mp_processes'] = cp.as_int('mp_processes')
# Some non-trivial params
PARAMS['continue_on_error'] = cp.as_bool('continue_on_error')
PARAMS['grid_dx_method'] = cp['grid_dx_method']
PARAMS['topo_interp'] = cp['topo_interp']
PARAMS['use_intersects'] = cp.as_bool('use_intersects')
PARAMS['use_compression'] = cp.as_bool('use_compression')
PARAMS['mpi_recv_buf_size'] = cp.as_int('mpi_recv_buf_size')
PARAMS['use_multiple_flowlines'] = cp.as_bool('use_multiple_flowlines')
PARAMS['filter_min_slope'] = cp.as_bool('filter_min_slope')
PARAMS['auto_skip_task'] = cp.as_bool('auto_skip_task')
PARAMS['correct_for_neg_flux'] = cp.as_bool('correct_for_neg_flux')
PARAMS['filter_for_neg_flux'] = cp.as_bool('filter_for_neg_flux')
PARAMS['run_mb_calibration'] = cp.as_bool('run_mb_calibration')
PARAMS['rgi_version'] = cp['rgi_version']
PARAMS['use_rgi_area'] = cp.as_bool('use_rgi_area')
PARAMS['compress_climate_netcdf'] = cp.as_bool('compress_climate_netcdf')
# Climate
PARAMS['baseline_climate'] = cp['baseline_climate'].strip().upper()
PARAMS['baseline_y0'] = cp.as_int('baseline_y0')
PARAMS['baseline_y1'] = cp.as_int('baseline_y1')
PARAMS['hydro_month_nh'] = cp.as_int('hydro_month_nh')
PARAMS['hydro_month_sh'] = cp.as_int('hydro_month_sh')
PARAMS['temp_use_local_gradient'] = cp.as_bool('temp_use_local_gradient')
k = 'temp_local_gradient_bounds'
PARAMS[k] = [float(vk) for vk in cp.as_list(k)]
k = 'tstar_search_window'
PARAMS[k] = [int(vk) for vk in cp.as_list(k)]
PARAMS['use_bias_for_run'] = cp.as_bool('use_bias_for_run')
PARAMS['allow_negative_mustar'] = cp.as_bool('allow_negative_mustar')
# Inversion
k = 'use_shape_factor_for_inversion'
PARAMS[k] = cp[k]
# Flowline model
k = 'use_shape_factor_for_fluxbasedmodel'
PARAMS[k] = cp[k]
# Make sure we have a proper cache dir
from oggm.utils import download_oggm_files, get_demo_file
download_oggm_files()
# Delete non-floats
ltr = ['working_dir', 'dem_file', 'climate_file',
'grid_dx_method', 'run_mb_calibration', 'compress_climate_netcdf',
'mp_processes', 'use_multiprocessing', 'baseline_y0', 'baseline_y1',
'temp_use_local_gradient', 'temp_local_gradient_bounds',
'topo_interp', 'use_compression', 'bed_shape', 'continue_on_error',
'use_multiple_flowlines',
'mpi_recv_buf_size', 'hydro_month_nh',
'tstar_search_window', 'use_bias_for_run', 'hydro_month_sh',
'use_intersects', 'filter_min_slope',
'auto_skip_task', 'correct_for_neg_flux', 'filter_for_neg_flux',
'rgi_version', 'allow_negative_mustar',
'use_shape_factor_for_inversion', 'use_rgi_area',
'use_shape_factor_for_fluxbasedmodel', 'baseline_climate']
for k in ltr:
cp.pop(k, None)
# Other params are floats
for k in cp:
PARAMS[k] = cp.as_float(k)
# Read-in the reference t* data - maybe it will be used, maybe not
fns = ['ref_tstars_rgi5_cru4', 'ref_tstars_rgi6_cru4',
'ref_tstars_rgi5_histalp', 'ref_tstars_rgi6_histalp']
for fn in fns:
PARAMS[fn] = pd.read_csv(get_demo_file('oggm_' + fn + '.csv'))
fpath = get_demo_file('oggm_' + fn + '_calib_params.json')
with open(fpath, 'r') as fp:
mbpar = json.load(fp)
PARAMS[fn+'_calib_params'] = mbpar
# Empty defaults
set_intersects_db()
IS_INITIALIZED = True
# Pre extract cru cl to avoid problems by multiproc
from oggm.utils import get_cru_cl_file
get_cru_cl_file()
def initialize(file=None, logging_level='INFO'):
"""Read the configuration file containing the run's parameters.
This should be the first call, before using any of the other OGGM modules
for most (all?) OGGM simulations.
Parameters
----------
file : str
path to the configuration file (default: OGGM params.cfg)
logging_level : str
set a logging level. See :func:`set_logging_config` for options.
"""
global PARAMS
global DATA
initialize_minimal(file=file, logging_level=logging_level)
# Make sure we have a proper cache dir
from oggm.utils import download_oggm_files, get_demo_file
download_oggm_files()
# Read-in the reference t* data for all available models types (oggm, vas)
model_prefixes = ['oggm_', 'vas_']
for prefix in model_prefixes:
fns = [
'ref_tstars_rgi5_cru4', 'ref_tstars_rgi6_cru4',
'ref_tstars_rgi5_histalp', 'ref_tstars_rgi6_histalp'
]
for fn in fns:
fpath = get_demo_file(prefix + fn + '.csv')
PARAMS[prefix + fn] = pd.read_csv(fpath)
fpath = get_demo_file(prefix + fn + '_calib_params.json')
with open(fpath, 'r') as fp:
mbpar = json.load(fp)
PARAMS[prefix + fn + '_calib_params'] = mbpar
# Pre extract cru cl to avoid problems by multiproc
from oggm.utils import get_cru_cl_file
get_cru_cl_file()
# Read in the demo glaciers
file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data',
'demo_glaciers.csv')
DATA['demo_glaciers'] = pd.read_csv(file, index_col=0)
# Add other things
if 'dem_grids' not in DATA:
grids = {}
for grid_json in [
'gimpdem_90m_v01.1.json', 'arcticdem_mosaic_100m_v3.0.json',
'Alaska_albers_V3.json', 'AntarcticDEM_wgs84.json',
'REMA_100m_dem.json'
]:
if grid_json not in grids:
fp = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'data', grid_json)
try:
grids[grid_json] = salem.Grid.from_json(fp)
except NameError:
pass
DATA['dem_grids'] = grids
# Download verification dictionary
DATA['dl_verify_data'] = None
def process_cru_data(gdir):
"""Processes and writes the climate data for this glacier.
Interpolates the CRU TS data to the high-resolution CL2 climatologies
(provided with OGGM) and writes everything to a NetCDF file.
"""
# read the climatology
clfile = utils.get_cru_cl_file()
ncclim = salem.GeoNetcdf(clfile)
# and the TS data
nc_ts_tmp = salem.GeoNetcdf(utils.get_cru_file('tmp'), monthbegin=True)
nc_ts_pre = salem.GeoNetcdf(utils.get_cru_file('pre'), monthbegin=True)
# set temporal subset for the ts data (hydro years)
yrs = nc_ts_pre.time.year
y0, y1 = yrs[0], yrs[-1]
nc_ts_tmp.set_period(t0='{}-10-01'.format(y0), t1='{}-09-01'.format(y1))
nc_ts_pre.set_period(t0='{}-10-01'.format(y0), t1='{}-09-01'.format(y1))
time = nc_ts_pre.time
ny, r = divmod(len(time), 12)
assert r == 0
# gradient default params
use_grad = cfg.PARAMS['temp_use_local_gradient']
def_grad = cfg.PARAMS['temp_default_gradient']
g_minmax = cfg.PARAMS['temp_local_gradient_bounds']
lon = gdir.cenlon
lat = gdir.cenlat
# This is guaranteed to work because I prepared the file (I hope)
ncclim.set_subset(corners=((lon, lat), (lon, lat)), margin=1)
# get climatology data
loc_hgt = ncclim.get_vardata('elev')
loc_tmp = ncclim.get_vardata('temp')
loc_pre = ncclim.get_vardata('prcp')
loc_lon = ncclim.get_vardata('lon')
loc_lat = ncclim.get_vardata('lat')
# see if the center is ok
if not np.isfinite(loc_hgt[1, 1]):
# take another candidate where finite
isok = np.isfinite(loc_hgt)
# wait: some areas are entirely NaNs, make the subset larger
_margin = 1
while not np.any(isok):
_margin += 1
ncclim.set_subset(corners=((lon, lat), (lon, lat)), margin=_margin)
loc_hgt = ncclim.get_vardata('elev')
isok = np.isfinite(loc_hgt)
if _margin > 1:
log.debug('%s: I had to look up for far climate pixels: %s',
gdir.rgi_id, _margin)
# Take the first candidate (doesn't matter which)
lon, lat = ncclim.grid.ll_coordinates
lon = lon[isok][0]
lat = lat[isok][0]
# Resubset
ncclim.set_subset()
ncclim.set_subset(corners=((lon, lat), (lon, lat)), margin=1)
loc_hgt = ncclim.get_vardata('elev')
loc_tmp = ncclim.get_vardata('temp')
loc_pre = ncclim.get_vardata('prcp')
loc_lon = ncclim.get_vardata('lon')
loc_lat = ncclim.get_vardata('lat')
assert np.isfinite(loc_hgt[1, 1])
isok = np.isfinite(loc_hgt)
hgt_f = loc_hgt[isok].flatten()
assert len(hgt_f) > 0.
ts_grad = np.zeros(12) + def_grad
if use_grad and len(hgt_f) >= 5:
for i in range(12):
loc_tmp_mth = loc_tmp[i, ...][isok].flatten()
slope, _, _, p_val, _ = stats.linregress(hgt_f, loc_tmp_mth)
ts_grad[i] = slope if (p_val < 0.01) else def_grad
# ... but dont exaggerate too much
ts_grad = np.clip(ts_grad, g_minmax[0], g_minmax[1])
# convert to timeserie and hydroyears
ts_grad = ts_grad.tolist()
ts_grad = ts_grad[9:] + ts_grad[0:9]
ts_grad = np.asarray(ts_grad * ny)
# maybe this will throw out of bounds warnings
nc_ts_tmp.set_subset(corners=((lon, lat), (lon, lat)), margin=1)
nc_ts_pre.set_subset(corners=((lon, lat), (lon, lat)), margin=1)
# compute monthly anomalies
# of temp
ts_tmp = nc_ts_tmp.get_vardata('tmp', as_xarray=True)
ts_tmp_avg = ts_tmp.sel(time=slice('1961-01-01', '1990-12-01'))
ts_tmp_avg = ts_tmp_avg.groupby('time.month').mean(dim='time')
ts_tmp = ts_tmp.groupby('time.month') - ts_tmp_avg
# of precip
ts_pre = nc_ts_pre.get_vardata('pre', as_xarray=True)
ts_pre_avg = ts_pre.sel(time=slice('1961-01-01', '1990-12-01'))
ts_pre_avg = ts_pre_avg.groupby('time.month').mean(dim='time')
ts_pre = ts_pre.groupby('time.month') - ts_pre_avg
# interpolate to HR grid
if np.any(~np.isfinite(ts_tmp[:, 1, 1])):
# Extreme case, middle pix is not valid
# take any valid pix from the 3*3 (and hope there's one)
found_it = False
for idi in range(2):
for idj in range(2):
if np.all(np.isfinite(ts_tmp[:, idj, idi])):
ts_tmp[:, 1, 1] = ts_tmp[:, idj, idi]
ts_pre[:, 1, 1] = ts_pre[:, idj, idi]
found_it = True
if not found_it:
msg = '{}: OMG there is no climate data'.format(gdir.rgi_id)
raise RuntimeError(msg)
elif np.any(~np.isfinite(ts_tmp)):
# maybe the side is nan, but we can do nearest
ts_tmp = ncclim.grid.map_gridded_data(ts_tmp.values, nc_ts_tmp.grid,
interp='nearest')
ts_pre = ncclim.grid.map_gridded_data(ts_pre.values, nc_ts_pre.grid,
interp='nearest')
else:
# We can do bilinear
ts_tmp = ncclim.grid.map_gridded_data(ts_tmp.values, nc_ts_tmp.grid,
interp='linear')
ts_pre = ncclim.grid.map_gridded_data(ts_pre.values, nc_ts_pre.grid,
interp='linear')
# take the center pixel and add it to the CRU CL clim
# for temp
loc_tmp = xr.DataArray(loc_tmp[:, 1, 1], dims=['month'],
coords={'month':ts_pre_avg.month})
ts_tmp = xr.DataArray(ts_tmp[:, 1, 1], dims=['time'],
coords={'time':time})
ts_tmp = ts_tmp.groupby('time.month') + loc_tmp
# for prcp
loc_pre = xr.DataArray(loc_pre[:, 1, 1], dims=['month'],
coords={'month':ts_pre_avg.month})
ts_pre = xr.DataArray(ts_pre[:, 1, 1], dims=['time'],
coords={'time':time})
ts_pre = ts_pre.groupby('time.month') + loc_pre
# done
loc_hgt = loc_hgt[1, 1]
loc_lon = loc_lon[1]
loc_lat = loc_lat[1]
assert np.isfinite(loc_hgt)
assert np.all(np.isfinite(ts_pre.values))
assert np.all(np.isfinite(ts_tmp.values))
assert np.all(np.isfinite(ts_grad))
gdir.write_monthly_climate_file(time, ts_pre.values, ts_tmp.values,
ts_grad, loc_hgt, loc_lon, loc_lat)
ncclim._nc.close()
nc_ts_tmp._nc.close()
nc_ts_pre._nc.close()
# metadata
out = {'climate_source': 'CRU data', 'hydro_yr_0': y0+1, 'hydro_yr_1': y1}
gdir.write_pickle(out, 'climate_info')
def initialize(file=None, logging_level='INFO'):
"""Read the configuration file containing the run's parameters.
This should be the first call, before using any of the other OGGM modules
for most (all?) OGGM simulations.
Parameters
----------
file : str
path to the configuration file (default: OGGM params.cfg)
logging_level : str
set a logging level. See :func:`set_logging_config` for options.
"""
global IS_INITIALIZED
global PARAMS
global PATHS
global DEMO_GLACIERS
set_logging_config(logging_level=logging_level)
if file is None:
file = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'params.cfg')
try:
cp = ConfigObj(file, file_error=True)
except (ConfigObjError, IOError) as e:
log.critical('Config file could not be parsed (%s): %s', file, e)
sys.exit()
log.workflow('Using configuration file: %s', file)
# Paths
oggm_static_paths()
PATHS['working_dir'] = cp['working_dir']
PATHS['dem_file'] = cp['dem_file']
PATHS['climate_file'] = cp['climate_file']
# Multiprocessing pool
PARAMS['use_multiprocessing'] = cp.as_bool('use_multiprocessing')
PARAMS['mp_processes'] = cp.as_int('mp_processes')
# Some non-trivial params
PARAMS['continue_on_error'] = cp.as_bool('continue_on_error')
PARAMS['grid_dx_method'] = cp['grid_dx_method']
PARAMS['topo_interp'] = cp['topo_interp']
PARAMS['use_intersects'] = cp.as_bool('use_intersects')
PARAMS['use_compression'] = cp.as_bool('use_compression')
PARAMS['mpi_recv_buf_size'] = cp.as_int('mpi_recv_buf_size')
PARAMS['use_multiple_flowlines'] = cp.as_bool('use_multiple_flowlines')
PARAMS['filter_min_slope'] = cp.as_bool('filter_min_slope')
PARAMS['auto_skip_task'] = cp.as_bool('auto_skip_task')
PARAMS['correct_for_neg_flux'] = cp.as_bool('correct_for_neg_flux')
PARAMS['filter_for_neg_flux'] = cp.as_bool('filter_for_neg_flux')
PARAMS['run_mb_calibration'] = cp.as_bool('run_mb_calibration')
PARAMS['rgi_version'] = cp['rgi_version']
PARAMS['use_rgi_area'] = cp.as_bool('use_rgi_area')
PARAMS['compress_climate_netcdf'] = cp.as_bool('compress_climate_netcdf')
PARAMS['use_tar_shapefiles'] = cp.as_bool('use_tar_shapefiles')
PARAMS['clip_mu_star'] = cp.as_bool('clip_mu_star')
PARAMS['clip_tidewater_border'] = cp.as_bool('clip_tidewater_border')
PARAMS['dl_verify'] = cp.as_bool('dl_verify')
# Climate
PARAMS['baseline_climate'] = cp['baseline_climate'].strip().upper()
PARAMS['baseline_y0'] = cp.as_int('baseline_y0')
PARAMS['baseline_y1'] = cp.as_int('baseline_y1')
PARAMS['hydro_month_nh'] = cp.as_int('hydro_month_nh')
PARAMS['hydro_month_sh'] = cp.as_int('hydro_month_sh')
PARAMS['temp_use_local_gradient'] = cp.as_bool('temp_use_local_gradient')
PARAMS['tstar_search_glacierwide'] = cp.as_bool('tstar_search_glacierwide')
k = 'temp_local_gradient_bounds'
PARAMS[k] = [float(vk) for vk in cp.as_list(k)]
k = 'tstar_search_window'
PARAMS[k] = [int(vk) for vk in cp.as_list(k)]
PARAMS['use_bias_for_run'] = cp.as_bool('use_bias_for_run')
# Inversion
k = 'use_shape_factor_for_inversion'
PARAMS[k] = cp[k]
# Flowline model
k = 'use_shape_factor_for_fluxbasedmodel'
PARAMS[k] = cp[k]
# Make sure we have a proper cache dir
from oggm.utils import download_oggm_files, get_demo_file
download_oggm_files()
# Delete non-floats
ltr = ['working_dir', 'dem_file', 'climate_file', 'use_tar_shapefiles',
'grid_dx_method', 'run_mb_calibration', 'compress_climate_netcdf',
'mp_processes', 'use_multiprocessing', 'baseline_y0', 'baseline_y1',
'temp_use_local_gradient', 'temp_local_gradient_bounds',
'topo_interp', 'use_compression', 'bed_shape', 'continue_on_error',
'use_multiple_flowlines', 'tstar_search_glacierwide',
'mpi_recv_buf_size', 'hydro_month_nh', 'clip_mu_star',
'tstar_search_window', 'use_bias_for_run', 'hydro_month_sh',
'use_intersects', 'filter_min_slope', 'clip_tidewater_border',
'auto_skip_task', 'correct_for_neg_flux', 'filter_for_neg_flux',
'rgi_version', 'dl_verify',
'use_shape_factor_for_inversion', 'use_rgi_area',
'use_shape_factor_for_fluxbasedmodel', 'baseline_climate']
for k in ltr:
cp.pop(k, None)
# Other params are floats
for k in cp:
PARAMS[k] = cp.as_float(k)
# Read-in the reference t* data - maybe it will be used, maybe not
fns = ['ref_tstars_rgi5_cru4', 'ref_tstars_rgi6_cru4',
'ref_tstars_rgi5_histalp', 'ref_tstars_rgi6_histalp']
for fn in fns:
PARAMS[fn] = pd.read_csv(get_demo_file('oggm_' + fn + '.csv'))
fpath = get_demo_file('oggm_' + fn + '_calib_params.json')
with open(fpath, 'r') as fp:
mbpar = json.load(fp)
PARAMS[fn+'_calib_params'] = mbpar
# Empty defaults
set_intersects_db()
IS_INITIALIZED = True
# Pre extract cru cl to avoid problems by multiproc
from oggm.utils import get_cru_cl_file
get_cru_cl_file()
# Read in the demo glaciers
file = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'data', 'demo_glaciers.csv')
DEMO_GLACIERS = pd.read_csv(file, index_col=0)
