def test_rgi_intersects(self):
# Make a fake RGI file
rgi_dir = os.path.join(self.dldir, 'rgi50')
utils.mkdir(rgi_dir)
make_fake_zipdir(os.path.join(rgi_dir, 'RGI_V5_Intersects'),
fakefile='Intersects_OGGM_Manifest.txt')
make_fake_zipdir(os.path.join(rgi_dir, 'RGI_V5_Intersects',
'11_rgi50_CentralEurope'),
fakefile='intersects_11_rgi50_CentralEurope.shp')
make_fake_zipdir(os.path.join(rgi_dir, 'RGI_V5_Intersects',
'00_rgi50_AllRegs'),
fakefile='intersects_rgi50_AllRegs.shp')
rgi_f = make_fake_zipdir(rgi_dir)
def down_check(url, cache_name=None, reset=False):
expected = ('https://cluster.klima.uni-bremen.de/~fmaussion/rgi/' +
'RGI_V5_Intersects.zip')
self.assertEqual(url, expected)
return rgi_f
with FakeDownloadManager('_progress_urlretrieve', down_check):
rgi = utils.get_rgi_intersects_dir()
utils.get_rgi_intersects_region_file('11', version='5')
utils.get_rgi_intersects_region_file('00', version='5')
assert os.path.isdir(rgi)
assert os.path.exists(os.path.join(rgi,
'Intersects_OGGM_Manifest.txt'))
# Make a fake RGI file
rgi_dir = os.path.join(self.dldir, 'rgi60')
utils.mkdir(rgi_dir)
make_fake_zipdir(os.path.join(rgi_dir, 'RGI_V6_Intersects'),
fakefile='Intersects_OGGM_Manifest.txt')
rgi_f = make_fake_zipdir(rgi_dir)
def down_check(url, cache_name=None, reset=False):
expected = ('https://cluster.klima.uni-bremen.de/~fmaussion/rgi/' +
'RGI_V6_Intersects.zip')
self.assertEqual(url, expected)
return rgi_f
with FakeDownloadManager('_progress_urlretrieve', down_check):
rgi = utils.get_rgi_intersects_dir(version='6')
assert os.path.isdir(rgi)
assert os.path.exists(os.path.join(rgi,
'Intersects_OGGM_Manifest.txt'))
from oggm.tests.funcs import get_test_dir
from oggm.core import gis, climate, centerlines, vascaling
# create temporary working directory
testdir = os.path.join(get_test_dir(), 'start_area')
if not os.path.exists(testdir):
os.makedirs(testdir)
shutil.rmtree(testdir)
os.makedirs(testdir)
# load default parameter file
cfg.initialize()
# set path to working directory
cfg.PATHS['working_dir'] = testdir
# load and set path to intersects
path = utils.get_rgi_intersects_region_file('11', version='6')
cfg.set_intersects_db(path)
# change some default parameters
cfg.PARAMS['border'] = 50
cfg.PARAMS['baseline_climate'] = 'CRU'
cfg.PARAMS['use_multiprocessing'] = True
# get RGI entity for Hintereisferner
rgi_id = 'RGI60-11.00897'
entity = utils.get_rgi_glacier_entities([rgi_id]).iloc[0]
# initialize the GlacierDirectory
gdir = oggm.GlacierDirectory(entity, base_dir=testdir)
# define the local grid and glacier mask
def compute_scaling_params(rgi_ids, path=None):
""" The routine computes scaling parameters by fitting a linear regression
to the volume/area and volume/length scatter in log-log space, using the
inversion volume, the RGI area and the longest centerline as "observations"
Thereby, the following two cases apply:
- compute only scaling constants, since scaling exponents have a physical
basis and should not be changed
- compute only scaling constants and scaling exponents
Returns parameters in a 2-level dictionary. The upper level differentiates
between the two cases, the lower level indicates the parameters.
Parameters
----------
rgi_ids: array-like
List of RGI IDs for which the equilibrium experiments are performed.
Returns
-------
Dictionary containing the computed parameters.
"""
log.info('Starting scaling parameter computation')
# compute RGI region and version from RGI IDs
# assuming all they are all the same
rgi_region = (rgi_ids[0].split('-')[-1]).split('.')[0]
rgi_version = (rgi_ids[0].split('-')[0])[-2:-1]
# load default parameter file
vascaling.initialize()
# get environmental variables for working and output directories
WORKING_DIR = os.environ["WORKDIR"]
OUTPUT_DIR = os.environ["OUTDIR"]
# create working directory
utils.mkdir(WORKING_DIR)
utils.mkdir(OUTPUT_DIR)
# set path to working directory
cfg.PATHS['working_dir'] = WORKING_DIR
# set RGI version and region
cfg.PARAMS['rgi_version'] = rgi_version
# define how many grid points to use around the glacier,
# if you expect the glacier to grow large use a larger border
cfg.PARAMS['border'] = 120
# we use HistAlp climate data
cfg.PARAMS['baseline_climate'] = 'HISTALP'
# set the mb hyper parameters accordingly
cfg.PARAMS['prcp_scaling_factor'] = 1.75
cfg.PARAMS['temp_melt'] = -1.75
# set minimum ice thickness to include in glacier length computation
# this reduces weird spikes in length records
cfg.PARAMS['min_ice_thick_for_length'] = 0.1
# read RGI entry for the glaciers as DataFrame
# containing the outline area as shapefile
rgidf = utils.get_rgi_glacier_entities(rgi_ids)
# get and set path to intersect shapefile
intersects_db = utils.get_rgi_intersects_region_file(region=rgi_region)
cfg.set_intersects_db(intersects_db)
# the bias is defined to be zero during the calibration process,
# which is why we don't use it here to reproduce the results
cfg.PARAMS['use_bias_for_run'] = True
# sort by area for more efficient parallel computing
rgidf = rgidf.sort_values('Area', ascending=False)
cfg.PARAMS['use_multiprocessing'] = True
# operational run, all glaciers should run
cfg.PARAMS['continue_on_error'] = True
# initialize the GlacierDirectory
gdirs = workflow.init_glacier_directories(rgidf, reset=False, force=True)
# run gis tasks
workflow.gis_prepro_tasks(gdirs)
# run climate tasks
workflow.execute_entity_task(climate.process_climate_data, gdirs)
# compute local t* and the corresponding mu*
workflow.execute_entity_task(climate.local_t_star, gdirs)
workflow.execute_entity_task(climate.mu_star_calibration, gdirs)
# run inversion tasks
workflow.inversion_tasks(gdirs)
# finalize preprocessing
workflow.execute_entity_task(flowline.init_present_time_glacier, gdirs)
# create empty dictionary
params = dict()
# compute scaling constants for given (fixed) slope
params['const_only'] = vascaling.get_scaling_constant(gdirs)
# compute scaling constants and scaling exponent via linear regression
params['const_expo'] = vascaling.get_scaling_constant_exponent(gdirs)
# store to file
if path:
if not isinstance(path, str):
# set default path and filename
path = os.path.join(OUTPUT_DIR, 'scaling_params.json')
json.dump(params, open(path, 'w'))
return params
def run_prepro_levels(rgi_version=None,
rgi_reg=None,
border=None,
output_folder='',
working_dir='',
dem_source='',
is_test=False,
demo=False,
test_rgidf=None,
test_intersects_file=None,
test_topofile=None,
test_crudir=None,
disable_mp=False,
timeout=0,
max_level=4,
logging_level='WORKFLOW'):
"""Does the actual job.
Parameters
----------
rgi_version : str
the RGI version to use (defaults to cfg.PARAMS)
rgi_reg : str
the RGI region to process
border : int
the number of pixels at the maps border
output_folder : str
path to the output folder (where to put the preprocessed tar files)
dem_source : str
which DEM source to use: default, SOURCE_NAME or ALL
working_dir : str
path to the OGGM working directory
is_test : bool
to test on a couple of glaciers only!
demo : bool
to run the prepro for the list of demo glaciers
test_rgidf : shapefile
for testing purposes only
test_intersects_file : shapefile
for testing purposes only
test_topofile : str
for testing purposes only
test_crudir : str
for testing purposes only
disable_mp : bool
disable multiprocessing
max_level : int
the maximum pre-processing level before stopping
logging_level : str
the logging level to use (DEBUG, INFO, WARNING, WORKFLOW)
"""
# TODO: temporarily silence Fiona deprecation warnings
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
# Input check
if max_level not in [1, 2, 3, 4]:
raise InvalidParamsError('max_level should be one of [1, 2, 3, 4]')
# Time
start = time.time()
def _time_log():
# Log util
m, s = divmod(time.time() - start, 60)
h, m = divmod(m, 60)
log.workflow('OGGM prepro_levels is done! Time needed: '
'{:02d}:{:02d}:{:02d}'.format(int(h), int(m), int(s)))
# Initialize OGGM and set up the run parameters
cfg.initialize(logging_level=logging_level)
# Local paths
utils.mkdir(working_dir)
cfg.PATHS['working_dir'] = working_dir
# Use multiprocessing?
cfg.PARAMS['use_multiprocessing'] = not disable_mp
# How many grid points around the glacier?
# Make it large if you expect your glaciers to grow large
cfg.PARAMS['border'] = border
# Set to True for operational runs
cfg.PARAMS['continue_on_error'] = True
# Timeout
cfg.PARAMS['task_timeout'] = timeout
# For statistics
climate_periods = [1920, 1960, 2000]
if rgi_version is None:
rgi_version = cfg.PARAMS['rgi_version']
rgi_dir_name = 'RGI{}'.format(rgi_version)
border_dir_name = 'b_{:03d}'.format(border)
base_dir = os.path.join(output_folder, rgi_dir_name, border_dir_name)
# Add a package version file
utils.mkdir(base_dir)
opath = os.path.join(base_dir, 'package_versions.txt')
with open(opath, 'w') as vfile:
vfile.write(utils.show_versions(logger=log))
if demo:
rgidf = utils.get_rgi_glacier_entities(cfg.DATA['demo_glaciers'].index)
elif test_rgidf is None:
# Get the RGI file
rgidf = gpd.read_file(
utils.get_rgi_region_file(rgi_reg, version=rgi_version))
# We use intersects
rgif = utils.get_rgi_intersects_region_file(rgi_reg,
version=rgi_version)
cfg.set_intersects_db(rgif)
else:
rgidf = test_rgidf
cfg.set_intersects_db(test_intersects_file)
if is_test:
# Just for fun
rgidf = rgidf.sample(4)
# Sort for more efficient parallel computing
rgidf = rgidf.sort_values('Area', ascending=False)
log.workflow('Starting prepro run for RGI reg: {} '
'and border: {}'.format(rgi_reg, border))
log.workflow('Number of glaciers: {}'.format(len(rgidf)))
# Input
if test_topofile:
cfg.PATHS['dem_file'] = test_topofile
# L1 - initialize working directories
# Which DEM source?
if dem_source.upper() == 'ALL':
# This is the complex one, just do the job an leave
log.workflow('Running prepro on ALL sources')
for i, s in enumerate(utils.DEM_SOURCES):
rs = i == 0
rgidf['DEM_SOURCE'] = s
log.workflow('Running prepro on sources: {}'.format(s))
gdirs = workflow.init_glacier_regions(rgidf, reset=rs, force=rs)
workflow.execute_entity_task(_rename_dem_folder, gdirs, source=s)
# Compress all in output directory
l_base_dir = os.path.join(base_dir, 'L1')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=l_base_dir)
utils.base_dir_to_tar(l_base_dir)
_time_log()
return
if dem_source:
# Force a given source
rgidf['DEM_SOURCE'] = dem_source.upper()
# L1 - go
gdirs = workflow.init_glacier_regions(rgidf, reset=True, force=True)
# Glacier stats
sum_dir = os.path.join(base_dir, 'L1', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# L1 OK - compress all in output directory
l_base_dir = os.path.join(base_dir, 'L1')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=l_base_dir)
utils.base_dir_to_tar(l_base_dir)
if max_level == 1:
_time_log()
return
# L2 - Tasks
# Pre-download other files just in case
if test_crudir is None:
_ = utils.get_cru_file(var='tmp')
_ = utils.get_cru_file(var='pre')
else:
cfg.PATHS['cru_dir'] = test_crudir
workflow.execute_entity_task(tasks.process_cru_data, gdirs)
# Glacier stats
sum_dir = os.path.join(base_dir, 'L2', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# L2 OK - compress all in output directory
l_base_dir = os.path.join(base_dir, 'L2')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=l_base_dir)
utils.base_dir_to_tar(l_base_dir)
if max_level == 2:
_time_log()
return
# L3 - Tasks
task_list = [
tasks.glacier_masks, tasks.compute_centerlines,
tasks.initialize_flowlines, tasks.compute_downstream_line,
tasks.compute_downstream_bedshape, tasks.catchment_area,
tasks.catchment_intersections, tasks.catchment_width_geom,
tasks.catchment_width_correction, tasks.local_t_star,
tasks.mu_star_calibration, tasks.prepare_for_inversion,
tasks.mass_conservation_inversion, tasks.filter_inversion_output,
tasks.init_present_time_glacier
]
for task in task_list:
workflow.execute_entity_task(task, gdirs)
# Glacier stats
sum_dir = os.path.join(base_dir, 'L3', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
opath = os.path.join(sum_dir, 'climate_statistics_{}.csv'.format(rgi_reg))
utils.compile_climate_statistics(gdirs,
add_climate_period=climate_periods,
path=opath)
# L3 OK - compress all in output directory
l_base_dir = os.path.join(base_dir, 'L3')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=l_base_dir)
utils.base_dir_to_tar(l_base_dir)
if max_level == 3:
_time_log()
return
# L4 - No tasks: add some stats for consistency and make the dirs small
sum_dir = os.path.join(base_dir, 'L4', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# Copy mini data to new dir
base_dir = os.path.join(base_dir, 'L4')
mini_gdirs = workflow.execute_entity_task(tasks.copy_to_basedir,
gdirs,
base_dir=base_dir)
# L4 OK - compress all in output directory
workflow.execute_entity_task(utils.gdir_to_tar, mini_gdirs, delete=True)
utils.base_dir_to_tar(base_dir)
_time_log()
# How many grid points around the glacier?
cfg.PARAMS['border'] = 200
# Set to True for operational runs
cfg.PARAMS['continue_on_error'] = True
cfg.PARAMS['run_mb_calibration'] = False
cfg.PARAMS['optimize_inversion_params'] = False
cfg.PARAMS['dl_verify'] = True
# add to BASENAMES
_doc = 'contains observed and searched glacier from synthetic experiment to find intial state'
cfg.BASENAMES['synthetic_experiment'] = ('synthetic_experiment.pkl', _doc)
# We use intersects
db = utils.get_rgi_intersects_region_file(version='61', region=REGION)
cfg.set_intersects_db(db)
# RGI file
path = utils.get_rgi_region_file(REGION, version='61')
rgidf = gpd.read_file(path)
rgidf = rgidf.sort_values('Area', ascending=False)
# exclude non-landterminating glaciers
rgidf = rgidf[rgidf.TermType == 0]
rgidf = rgidf[rgidf.Connect != 2]
wgms = utils.get_ref_mb_glaciers_candidates()
# Keep only the wgms reference glaciers
rgidf = rgidf.loc[rgidf.RGIId.isin(wgms)]
utils.mkdir(cfg.PATHS['working_dir'], reset=True)
df = pd.read_csv(g2ti.index_file, index_col=0)
df = df.loc[[rid for rid in df.index if ('-19.' not in rid) and ('-05.' not in rid)]]
for exp, fa in zip([1, 2, 3], [1.7, 1.3, 1.0]):
sel_ids = df.loc[df['p{}'.format(exp)] == 1].index
# For testing
sel_ids = sel_ids[[14, 15, 16]]
rgidf = utils.get_rgi_glacier_entities(sel_ids, version=version)
# Add intersects
db = utils.get_rgi_intersects_region_file(version='61', rgi_ids=sel_ids)
cfg.set_intersects_db(db)
# Sort for more efficient parallel computing
rgidf = rgidf.sort_values('Area', ascending=False)
log.info('Starting OGGM run')
log.info('Number of glaciers: {}'.format(len(rgidf)))
# Go - initialize working directories
gdirs = workflow.init_glacier_regions(rgidf)
# Preprocessing tasks
task_list = [
tasks.glacier_masks,
tasks.compute_centerlines,
def run_benchmark(rgi_version=None, rgi_reg=None, border=None,
output_folder='', working_dir='', is_test=False,
test_rgidf=None, test_intersects_file=None,
test_topofile=None, test_crudir=None):
"""Does the actual job.
Parameters
----------
rgi_version : str
the RGI version to use (defaults to cfg.PARAMS)
rgi_reg : str
the RGI region to process
border : int
the number of pixels at the maps border
output_folder : str
path to the output folder (where to put the preprocessed tar files)
working_dir : str
path to the OGGM working directory
is_test : bool
to test on a couple of glaciers only!
test_rgidf : shapefile
for testing purposes only
test_intersects_file : shapefile
for testing purposes only
test_topofile : str
for testing purposes only
test_crudir : str
for testing purposes only
"""
# TODO: temporarily silence Fiona deprecation warnings
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
# Module logger
log = logging.getLogger(__name__)
# Initialize OGGM and set up the run parameters
cfg.initialize(logging_level='WORKFLOW')
# Local paths
utils.mkdir(working_dir)
cfg.PATHS['working_dir'] = working_dir
# Use multiprocessing?
cfg.PARAMS['use_multiprocessing'] = True
# How many grid points around the glacier?
# Make it large if you expect your glaciers to grow large
cfg.PARAMS['border'] = border
# Set to True for operational runs
cfg.PARAMS['continue_on_error'] = True
# For statistics
odf = pd.DataFrame()
if rgi_version is None:
rgi_version = cfg.PARAMS['rgi_version']
base_dir = os.path.join(output_folder)
# Add a package version file
utils.mkdir(base_dir)
opath = os.path.join(base_dir, 'package_versions.txt')
with open(opath, 'w') as vfile:
vfile.write(utils.show_versions(logger=log))
# Read RGI
start = time.time()
if test_rgidf is None:
# Get the RGI file
rgidf = gpd.read_file(utils.get_rgi_region_file(rgi_reg,
version=rgi_version))
# We use intersects
rgif = utils.get_rgi_intersects_region_file(rgi_reg,
version=rgi_version)
cfg.set_intersects_db(rgif)
else:
rgidf = test_rgidf
cfg.set_intersects_db(test_intersects_file)
if is_test:
# Just for fun
rgidf = rgidf.sample(2)
_add_time_to_df(odf, 'Read RGI', time.time()-start)
# Sort for more efficient parallel computing
rgidf = rgidf.sort_values('Area', ascending=False)
log.workflow('Starting prepro run for RGI reg: {} '
'and border: {}'.format(rgi_reg, border))
log.workflow('Number of glaciers: {}'.format(len(rgidf)))
# Input
if test_topofile:
cfg.PATHS['dem_file'] = test_topofile
# Initialize working directories
start = time.time()
gdirs = workflow.init_glacier_regions(rgidf, reset=True, force=True)
_add_time_to_df(odf, 'init_glacier_regions', time.time()-start)
# Pre-download other files just in case
if test_crudir is None:
_ = utils.get_cru_file(var='tmp')
_ = utils.get_cru_file(var='pre')
else:
cfg.PATHS['cru_dir'] = test_crudir
# Tasks
task_list = [
tasks.process_cru_data,
tasks.glacier_masks,
tasks.compute_centerlines,
tasks.initialize_flowlines,
tasks.compute_downstream_line,
tasks.compute_downstream_bedshape,
tasks.catchment_area,
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
tasks.local_t_star,
tasks.mu_star_calibration,
tasks.prepare_for_inversion,
tasks.mass_conservation_inversion,
tasks.filter_inversion_output,
tasks.init_present_time_glacier,
]
for task in task_list:
start = time.time()
workflow.execute_entity_task(task, gdirs)
_add_time_to_df(odf, task.__name__, time.time()-start)
# Runs
start = time.time()
workflow.execute_entity_task(tasks.run_random_climate, gdirs,
nyears=250, bias=0, seed=0,
output_filesuffix='_tstar')
_add_time_to_df(odf, 'run_random_climate_tstar_250', time.time()-start)
start = time.time()
workflow.execute_entity_task(tasks.run_random_climate, gdirs,
nyears=250, y0=1995, seed=0,
output_filesuffix='_commit')
_add_time_to_df(odf, 'run_random_climate_commit_250', time.time()-start)
# Compile results
start = time.time()
utils.compile_glacier_statistics(gdirs)
_add_time_to_df(odf, 'compile_glacier_statistics', time.time()-start)
start = time.time()
utils.compile_climate_statistics(gdirs,
add_climate_period=[1920, 1960, 2000])
_add_time_to_df(odf, 'compile_climate_statistics', time.time()-start)
start = time.time()
utils.compile_run_output(gdirs, filesuffix='_tstar')
_add_time_to_df(odf, 'compile_run_output_tstar', time.time()-start)
start = time.time()
utils.compile_run_output(gdirs, filesuffix='_commit')
_add_time_to_df(odf, 'compile_run_output_commit', time.time()-start)
# Log
opath = os.path.join(base_dir, 'benchmarks_b{:03d}.csv'.format(border))
odf.index.name = 'Task'
odf.to_csv(opath)
log.workflow('OGGM benchmarks is done!')
def climate_run_fl(rgi_ids,
path=True,
temp_biases=[0, +0.5, -0.5],
use_bias_for_run=False,
suffixes=['_bias_zero', '_bias_p', '_bias_n'],
tstar=None,
nyears=None,
**kwargs):
"""Computes 'only' the massbalance in analogy to the `equilibrium_run_...`
routines, without running the evolution (flowline) model.
Dataset containing yearly values of specific mass balance is returned.
Parameters
----------
rgi_ids: array-like
List of RGI IDs for which the equilibrium experiments are performed.
path: bool or str, optional, default=True
If a path is given (or True), the resulting dataset is stored to file.
temp_biases: array-like, optional, default=(0, +0.5, -0.5)
List of temperature biases (float, in degC) for the mass balance model.
suffixes: array-like, optional, default=['_normal', '_bias_p', '_bias_n']
Descriptive suffixes corresponding to the given temperature biases.
tstar: float
'Equilibrium year' used for the mass balance calibration.
nyears: int, optional, default=None
Number of years for which to compute the random mass balance
kwargs:
Additional key word arguments for massbalance model.
Returns
-------
Dataset containing yearly values of specific massbalance.
"""
# assert correct output file suffixes for temp biases
if len(temp_biases) != len(suffixes):
raise RuntimeError("Each given temperature bias must have its "
"corresponding suffix")
# compute RGI region and version from RGI IDs
# assuming all they are all the same
rgi_region = (rgi_ids[0].split('-')[-1]).split('.')[0]
rgi_version = (rgi_ids[0].split('-')[0])[-2:]
# load default parameter file
cfg.initialize()
# create working directory
wdir = '/Users/oberrauch/work/master/working_directories/'
wdir += 'test_cluster'
if not os.path.exists(wdir):
os.makedirs(wdir)
# shutil.rmtree(wdir)
# os.makedirs(wdir)
# set path to working directory
cfg.PATHS['working_dir'] = wdir
# set RGI verion and region
cfg.PARAMS['rgi_version'] = rgi_version
# define how many grid points to use around the glacier,
# if you expect the glacier to grow large use a larger border
cfg.PARAMS['border'] = 120
# we use HistAlp climate data
cfg.PARAMS['baseline_climate'] = 'HISTALP'
# set the mb hyper parameters accordingly
cfg.PARAMS['prcp_scaling_factor'] = 1.75
cfg.PARAMS['temp_melt'] = -1.75
# the bias is defined to be zero during the calibration process,
# which is why we don't use it here to reproduce the results
cfg.PARAMS['use_bias_for_run'] = use_bias_for_run
# operational run, all glaciers should run
cfg.PARAMS['continue_on_error'] = True
# read RGI entry for the glaciers as DataFrame
# containing the outline area as shapefile
rgidf = utils.get_rgi_glacier_entities(rgi_ids)
# get and set path to intersect shapefile
intersects_db = utils.get_rgi_intersects_region_file(region=rgi_region)
cfg.set_intersects_db(intersects_db)
# initialize the GlacierDirectory
gdirs = workflow.init_glacier_directories(rgidf, reset=False, force=True)
# run gis tasks
workflow.gis_prepro_tasks(gdirs)
# run climate tasks
workflow.execute_entity_task(climate.process_climate_data, gdirs)
workflow.execute_entity_task(climate.local_t_star,
gdirs,
tstar=tstar,
bias=0)
workflow.execute_entity_task(climate.mu_star_calibration, gdirs)
# run inversion tasks
workflow.inversion_tasks(gdirs)
# finalize preprocessing
workflow.execute_entity_task(flowline.init_present_time_glacier, gdirs)
# use t* as center year, even if specified differently
kwargs['y0'] = tstar
# run for 3000 years if not specified otherwise
if nyears is None:
nyears = 10000
years = np.arange(0, nyears + 1)
# create dataset
ds = list()
# run RandomMassBalance model centered around t*, once without
# temperature bias and once with positive and negative temperature bias
# of 0.5 °C each.
for gdir in gdirs:
# set random seed to get reproducible results
kwargs.setdefault('seed', 12)
kwargs.setdefault('halfsize', 15)
kwargs.setdefault('mb_model_class', flowline.RandomMassBalance)
kwargs.setdefault('filename', 'climate_historical')
kwargs.setdefault('input_filesuffix', '')
kwargs.setdefault('unique_samples', False)
ds_ = list()
fls = gdir.read_pickle('model_flowlines')
for suffix, temp_bias in zip(suffixes, temp_biases):
# instance mass balance model
mb_mod = flowline.MultipleFlowlineMassBalance(gdir, **kwargs)
if temp_bias is not None:
# add given temperature bias to mass balance model
mb_mod.temp_bias = temp_bias
# create empty container
spec_mb = list()
# iterate over all years
for yr in years:
spec_mb.append(mb_mod.get_specific_mb(fls=fls, year=yr))
# add to dataset
da = xr.DataArray(spec_mb, dims=('year'), coords={'year': years})
ds_.append(xr.Dataset({'spec_mb': da}))
ds_ = xr.concat(ds_, pd.Index(temp_biases, name='temp_bias'))
ds_.coords['rgi_id'] = gdir.rgi_id
ds.append(ds_)
ds = xr.concat(ds, 'rgi_id')
# store datasets
if path:
if path is True:
path = os.path.join(cfg.PATHS['working_dir'], 'mb_output_fl.nc')
ds.to_netcdf(path)
# ds_normal.to_netcdf(path[1])
# return ds, ds_normal
return ds
# Initialize OGGM and set up the default run parameters
cfg.initialize()
# Set to True for operational runs
cfg.PARAMS['continue_on_error'] = False
# Working directory
cfg.PATHS['working_dir'] = '/home/mowglie/disk/G2TI/vas_region'
geom_dir = g2ti.geometry_dir
flist = glob.glob(os.path.join(geom_dir, 'RGI60-{}'.format(region), '*'))
# For testing
flist = flist[:10]
cfg.set_intersects_db(utils.get_rgi_intersects_region_file(region=region,
version=version))
gdirs = g2tasks.parallel_define(flist)
# Preprocessing tasks
execute_entity_task(g2tasks.g2ti_masks, gdirs)
execute_entity_task(g2tasks.distribute_thickness_vas, gdirs,
vas_c=0.034,
slope_factor=0.7,
dis_factor=0.2,
write_tiff=True)
# Compile output
log.info('Compiling output')
utils.glacier_characteristics(gdirs)
# Log
def equilibrium_run_fl(rgi_ids,
use_random_mb=True,
path=True,
temp_biases=(0, +0.5, -0.5),
use_bias_for_run=False,
suffixes=['_bias_zero', '_bias_p', '_bias_n'],
store_individual_glaciers=True,
store_mean_sum=True,
tstar=None,
**kwargs):
""" The routine runs all steps for the equilibrium experiments using the
flowline model. For details see docstring of `sensitivity_run_vas`.
Parameters
----------
rgi_ids: array-like
List of RGI IDs for which the equilibrium experiments are performed.
use_random_mb: bool, optional, default=True
Choose between random massbalance model and constant massbalance model.
path: bool or str, optional, default=True
If a path is given (or True), the resulting dataset is stored to file.
temp_biases: array-like, optional, default=(0, +0.5, -0.5)
List of temperature biases (float, in degC) for the mass balance model.
suffixes: array-like, optional, default=['_normal', '_bias_p', '_bias_n']
Descriptive suffixes corresponding to the given temperature biases.
tstar: float
'Equilibrium year' used for the mass balance calibration.
kwargs:
Additional key word arguments for the `run_random_climate` or
`run_constant_climate` routines of the vascaling module.
Returns
-------
Dataset containing yearly values of all glacier geometries.
"""
# assert correct output file suffixes for temp biases
if len(temp_biases) != len(suffixes):
raise RuntimeError("Each given temperature bias must have its "
"corresponding suffix")
# compute RGI region and version from RGI IDs
# assuming all they are all the same
rgi_region = (rgi_ids[0].split('-')[-1]).split('.')[0]
rgi_version = (rgi_ids[0].split('-')[0])[-2:]
# load default parameter file
cfg.initialize()
# create working directory
WORKING_DIR = os.environ["WORKDIR"]
utils.mkdir(WORKING_DIR)
# set path to working directory
cfg.PATHS['working_dir'] = WORKING_DIR
# set RGI version and region
cfg.PARAMS['rgi_version'] = rgi_version
# define how many grid points to use around the glacier,
# if you expect the glacier to grow large use a larger border
cfg.PARAMS['border'] = 120
# we use HistAlp climate data
cfg.PARAMS['baseline_climate'] = 'HISTALP'
# set the mb hyper parameters accordingly
cfg.PARAMS['prcp_scaling_factor'] = 1.75
cfg.PARAMS['temp_melt'] = -1.75
# the bias is defined to be zero during the calibration process,
# which is why we don't use it here to reproduce the results
cfg.PARAMS['use_bias_for_run'] = use_bias_for_run
# read RGI entry for the glaciers as DataFrame
# containing the outline area as shapefile
rgidf = utils.get_rgi_glacier_entities(rgi_ids)
# get and set path to intersect shapefile
intersects_db = utils.get_rgi_intersects_region_file(region=rgi_region)
cfg.set_intersects_db(intersects_db)
# sort by area for more efficient parallel computing
rgidf = rgidf.sort_values('Area', ascending=False)
cfg.PARAMS['use_multiprocessing'] = True
# operational run, all glaciers should run
cfg.PARAMS['continue_on_error'] = True
# initialize the GlacierDirectory
gdirs = workflow.init_glacier_directories(rgidf, reset=False, force=True)
# run gis tasks
workflow.gis_prepro_tasks(gdirs)
# run climate tasks
workflow.execute_entity_task(climate.process_climate_data, gdirs)
ref_df = pd.read_csv(
utils.get_demo_file('oggm_ref_tstars_rgi6_histalp.csv'))
workflow.execute_entity_task(climate.local_t_star,
gdirs,
ref_df=ref_df,
tstar=tstar,
bias=0)
workflow.execute_entity_task(climate.mu_star_calibration, gdirs)
# run inversion tasks
workflow.inversion_tasks(gdirs)
# finalize preprocessing
workflow.execute_entity_task(flowline.init_present_time_glacier, gdirs)
# use t* as center year, even if specified differently
kwargs['y0'] = tstar
# run for 3000 years if not specified otherwise
kwargs.setdefault('nyears', 3000)
# disregard glaciers exceeding their domain boundaries
# to not dirsupt the entire run
kwargs.setdefault('check_for_boundaries', True)
if use_random_mb:
# set random seed to get reproducible results
kwargs.setdefault('seed', 12)
# run RandomMassBalance model centered around t*, once without
# temperature bias and once with positive and negative temperature bias
# of 0.5 °C each.
for suffix, temp_bias in zip(suffixes, temp_biases):
workflow.execute_entity_task(
flowline.run_random_climate,
gdirs,
temperature_bias=temp_bias,
output_filesuffix=suffix,
**kwargs,
)
else:
# run RandomMassBalance model centered around t*, once without
# temperature bias and once with positive and negative temperature bias
# of 0.5 °C each.
for suffix, temp_bias in zip(suffixes, temp_biases):
workflow.execute_entity_task(
flowline.run_constant_climate,
gdirs,
temperature_bias=temp_bias,
output_filesuffix=suffix,
**kwargs,
)
ds = list()
for suffix, temp_bias in zip(suffixes, temp_biases):
# compile the output for each run and store to file
ds_ = utils.compile_run_output(np.atleast_1d(gdirs),
input_filesuffix=suffix,
path=False)
ds.append(ds_)
# concat into one dataset with temperature bias as coordinate
if ds:
ds = xr.concat(ds, pd.Index(temp_biases, name='temp_bias'))
# add model type as coordinate
ds.coords['model'] = 'fl'
# add mb model type as coordinate
ds.coords['mb_model'] = 'random' if use_random_mb else 'constant'
# fill NaN values (which happen for vanished glaciers) with zero
ds = ds.fillna(0)
if store_individual_glaciers:
if store_mean_sum:
# compute mean and sum over all glaciers
ds_mean = ds.mean(dim='rgi_id')
ds_mean.coords['rgi_id'] = 'mean'
ds_sum = ds.sum(dim='rgi_id')
ds_sum.coords['rgi_id'] = 'sum'
# add to dataset
ds = xr.concat([ds, ds_mean, ds_sum], dim='rgi_id')
else:
pass
else:
# compute mean and sum over all glaciers
ds_mean = ds.mean(dim='rgi_id')
ds_mean.coords['rgi_id'] = 'mean'
ds_sum = ds.sum(dim='rgi_id')
ds_sum.coords['rgi_id'] = 'sum'
# add to dataset
ds = xr.concat([ds_mean, ds_sum], dim='rgi_id')
# normalize glacier geometries (length/area/volume) with start value
ds_normal = normalize_ds_with_start(ds)
# add coordinate to distinguish between normalized and absolute values
ds.coords['normalized'] = int(False)
ds_normal.coords['normalized'] = int(True)
# combine datasets
ds = xr.concat([ds, ds_normal], 'normalized')
# store datasets
if path:
if path is True:
mb = 'random' if use_random_mb else 'constant'
path = os.path.join(cfg.PATHS['working_dir'],
'run_output_{}_fl.nc'.format(mb))
ds.to_netcdf(path)
return ds
def climate_run_vas(rgi_ids, path=True, temp_biases=[0, +0.5, -0.5],
suffixes=['_bias_zero', '_bias_p', '_bias_n'],
use_bias_for_run=False, use_default_tstar=True,
tstar=None, nyears=None, **kwargs):
"""Computes 'only' the massbalance in analogy to the `equilibrium_run_...`
routines, without running the (volume/area scaling) evolution model.
Dataset containing yearly values of specific mass balance is returned.
Note: the task is not parallelized, hence it can take long if many glaciers
are given. TODO: could/should be fixed sometime...
TODO: add logging information
Parameters
----------
rgi_ids: array-like
List of RGI IDs for which the equilibrium experiments are performed.
path: bool or str, optional, default=True
If a path is given (or True), the resulting dataset is stored to file.
temp_biases: array-like, optional, default=(0, +0.5, -0.5)
List of temperature biases (float, in degC) for the mass balance model.
suffixes: array-like, optional, default=['_normal', '_bias_p', '_bias_n']
Descriptive suffixes corresponding to the given temperature biases.
use_bias_for_run: bool, optional, default=False
Flag deciding whether or not the mass balance residual is used
tstar: float, optional, default=None
'Equilibrium year' used for the mass balance calibration. Using the
`ref_tstars.csv` table if not supplied.
use_default_tstar : bool, optional, default=True
Flag deciding whether or not to use the default ref_tstar.csv list. If
`False`the `oggm_ref_tstars_rgi6_histalp.csv` reference table is used.
nyears: int, optional, default=None
Number of years for which to compute the random mass balance
kwargs:
Additional key word arguments for massbalance model.
Returns
-------
Dataset containing yearly values of specific massbalance.
"""
# assert correct output file suffixes for temp biases
if len(temp_biases) != len(suffixes):
raise RuntimeError("Each given temperature bias must have its "
"corresponding suffix")
# compute RGI region and version from RGI IDs
# assuming all they are all the same
rgi_region = (rgi_ids[0].split('-')[-1]).split('.')[0]
rgi_version = (rgi_ids[0].split('-')[0])[-2:-1]
# load default parameter file
cfg.initialize()
# get environmental variables for working and output directories
WORKING_DIR = os.environ["WORKDIR"]
OUTPUT_DIR = os.environ["OUTDIR"]
# create working directory
utils.mkdir(WORKING_DIR)
# set path to working directory
cfg.PATHS['working_dir'] = WORKING_DIR
# set RGI version and region
cfg.PARAMS['rgi_version'] = rgi_version
# define how many grid points to use around the glacier,
# if you expect the glacier to grow large use a larger border
cfg.PARAMS['border'] = 120
# we use HistAlp climate data
cfg.PARAMS['baseline_climate'] = 'HISTALP'
# set the mb hyper parameters accordingly
cfg.PARAMS['prcp_scaling_factor'] = 2.5
cfg.PARAMS['temp_melt'] = -0.5
# the bias is defined to be zero during the calibration process,
# which is why we don't use it here to reproduce the results
cfg.PARAMS['use_bias_for_run'] = use_bias_for_run
# operational run, all glaciers should run
cfg.PARAMS['continue_on_error'] = False
# read RGI entry for the glaciers as DataFrame
# containing the outline area as shapefile
rgidf = utils.get_rgi_glacier_entities(rgi_ids)
# get and set path to intersect shapefile
intersects_db = utils.get_rgi_intersects_region_file(region=rgi_region)
cfg.set_intersects_db(intersects_db)
# initialize the GlacierDirectory
gdirs = workflow.init_glacier_directories(rgidf, reset=False, force=True)
# define the local grid and glacier mask
workflow.execute_entity_task(gis.define_glacier_region, gdirs)
workflow.execute_entity_task(gis.glacier_masks, gdirs)
# process the given climate file
workflow.execute_entity_task(climate.process_climate_data, gdirs)
# compute local t* and the corresponding mu*
if tstar or use_default_tstar:
# compute mustar from given tstar
workflow.execute_entity_task(vascaling.local_t_star, gdirs, tstar=tstar, bias=0)
else:
# compute mustar from the reference table for the flowline model
# RGI v6 and HISTALP baseline climate
ref_df = pd.read_csv(utils.get_demo_file('oggm_ref_tstars_rgi6_histalp.csv'))
workflow.execute_entity_task(vascaling.local_t_star, gdirs, ref_df=ref_df)
# use t* as center year, even if specified differently
kwargs['y0'] = tstar
# run for 10'000 years if not specified otherwise
if nyears is None:
nyears = 1e4
years = np.arange(0, nyears + 1)
# create dataset
ds = list()
# run RandomMassBalance model centered around t*, once without
# temperature bias and once with positive and negative temperature bias
# of 0.5 deg C each.
for gdir in gdirs:
# set random seed to get reproducible results
kwargs.setdefault('halfsize', 15)
kwargs.setdefault('filename', 'climate_historical')
kwargs.setdefault('input_filesuffix', '')
ds_ = list()
for suffix, temp_bias in zip(suffixes, temp_biases):
# instance mass balance model
try:
mb_mod = vascaling.ConstantVASMassBalance(gdir, **kwargs)
except:
# continue with the next glacier or raise exception
# depending on `continue_on_error` flag
if cfg.PARAMS['continue_on_error']:
continue
else:
raise
if temp_bias is not None:
# add given temperature bias to mass balance model
mb_mod.temp_bias = temp_bias
# where to store the model output
diag_path = gdir.get_filepath('model_diagnostics',
filesuffix='_vas',
delete=True)
# get minimum and maximum glacier elevation
min_hgt, max_hgt = vascaling.get_min_max_elevation(gdir)
# create empty container
spec_mb = list()
# iterate over all years
for yr in years:
spec_mb.append(mb_mod.get_specific_mb(min_hgt, max_hgt, yr))
# add to dataset
da = xr.DataArray(spec_mb, dims=('year'), coords={'year': years})
ds_.append(xr.Dataset({'spec_mb': da}))
if ds_:
ds_ = xr.concat(ds_, pd.Index(temp_biases, name='temp_bias'))
ds_.coords['rgi_id'] = gdir.rgi_id
ds.append(ds_)
if ds:
# combine output from single glaciers into one dataset
ds = xr.concat(ds, 'rgi_id')
# store datasets
if path:
if path is True:
path = os.path.join(OUTPUT_DIR, 'mb_output_vas.nc')
ds.to_netcdf(path)
# return ds, ds_normal
return ds
def run_prepro_levels(rgi_version=None,
rgi_reg=None,
border=None,
output_folder='',
working_dir='',
dem_source='',
is_test=False,
test_ids=None,
demo=False,
test_rgidf=None,
test_intersects_file=None,
test_topofile=None,
disable_mp=False,
params_file=None,
elev_bands=False,
match_geodetic_mb=False,
centerlines_only=False,
add_consensus=False,
max_level=5,
logging_level='WORKFLOW',
disable_dl_verify=False):
"""Does the actual job.
Parameters
----------
rgi_version : str
the RGI version to use (defaults to cfg.PARAMS)
rgi_reg : str
the RGI region to process
border : int
the number of pixels at the maps border
output_folder : str
path to the output folder (where to put the preprocessed tar files)
dem_source : str
which DEM source to use: default, SOURCE_NAME or ALL
working_dir : str
path to the OGGM working directory
params_file : str
path to the OGGM parameter file (to override defaults)
is_test : bool
to test on a couple of glaciers only!
test_ids : list
if is_test: list of ids to process
demo : bool
to run the prepro for the list of demo glaciers
test_rgidf : shapefile
for testing purposes only
test_intersects_file : shapefile
for testing purposes only
test_topofile : str
for testing purposes only
test_crudir : str
for testing purposes only
disable_mp : bool
disable multiprocessing
elev_bands : bool
compute all flowlines based on the Huss&Hock 2015 method instead
of the OGGM default, which is a mix of elev_bands and centerlines.
centerlines_only : bool
compute all flowlines based on the OGGM centerline(s) method instead
of the OGGM default, which is a mix of elev_bands and centerlines.
match_geodetic_mb : bool
match the regional mass-balance estimates at the regional level
(currently Hugonnet et al., 2020).
add_consensus : bool
adds (reprojects) the consensus estimates thickness to the glacier
directories. With elev_bands=True, the data will also be binned.
max_level : int
the maximum pre-processing level before stopping
logging_level : str
the logging level to use (DEBUG, INFO, WARNING, WORKFLOW)
disable_dl_verify : bool
disable the hash verification of OGGM downloads
"""
# TODO: temporarily silence Fiona and other deprecation warnings
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
# Input check
if max_level not in [1, 2, 3, 4, 5]:
raise InvalidParamsError('max_level should be one of [1, 2, 3, 4, 5]')
# Time
start = time.time()
def _time_log():
# Log util
m, s = divmod(time.time() - start, 60)
h, m = divmod(m, 60)
log.workflow('OGGM prepro_levels is done! Time needed: '
'{:02d}:{:02d}:{:02d}'.format(int(h), int(m), int(s)))
# Config Override Params
params = {}
# Local paths
utils.mkdir(working_dir)
params['working_dir'] = working_dir
# Initialize OGGM and set up the run parameters
cfg.initialize(file=params_file,
params=params,
logging_level=logging_level,
future=True)
# Use multiprocessing?
cfg.PARAMS['use_multiprocessing'] = not disable_mp
# How many grid points around the glacier?
# Make it large if you expect your glaciers to grow large
cfg.PARAMS['border'] = border
# Set to True for operational runs
cfg.PARAMS['continue_on_error'] = True
# Check for the integrity of the files OGGM downloads at run time
# For large files (e.g. using a 1 tif DEM like ALASKA) calculating the hash
# takes a long time, so deactivating this can make sense
cfg.PARAMS['dl_verify'] = not disable_dl_verify
# Log the parameters
msg = '# OGGM Run parameters:'
for k, v in cfg.PARAMS.items():
if type(v) in [pd.DataFrame, dict]:
continue
msg += '\n {}: {}'.format(k, v)
log.workflow(msg)
if rgi_version is None:
rgi_version = cfg.PARAMS['rgi_version']
output_base_dir = os.path.join(output_folder, 'RGI{}'.format(rgi_version),
'b_{:03d}'.format(border))
# Add a package version file
utils.mkdir(output_base_dir)
opath = os.path.join(output_base_dir, 'package_versions.txt')
with open(opath, 'w') as vfile:
vfile.write(utils.show_versions(logger=log))
if demo:
rgidf = utils.get_rgi_glacier_entities(cfg.DATA['demo_glaciers'].index)
elif test_rgidf is None:
# Get the RGI file
rgidf = gpd.read_file(
utils.get_rgi_region_file(rgi_reg, version=rgi_version))
# We use intersects
rgif = utils.get_rgi_intersects_region_file(rgi_reg,
version=rgi_version)
cfg.set_intersects_db(rgif)
# Some RGI input quality checks - this is based on visual checks
# of large glaciers in the RGI
ids_to_ice_cap = [
'RGI60-05.10315', # huge Greenland ice cap
'RGI60-03.01466', # strange thing next to Devon
'RGI60-09.00918', # Academy of sciences Ice cap
'RGI60-09.00969',
'RGI60-09.00958',
'RGI60-09.00957',
]
rgidf.loc[rgidf.RGIId.isin(ids_to_ice_cap), 'Form'] = '1'
# In AA almost all large ice bodies are actually ice caps
if rgi_reg == '19':
rgidf.loc[rgidf.Area > 100, 'Form'] = '1'
# For greenland we omit connectivity level 2
if rgi_reg == '05':
rgidf = rgidf.loc[rgidf['Connect'] != 2]
else:
rgidf = test_rgidf
cfg.set_intersects_db(test_intersects_file)
if is_test:
if test_ids is not None:
rgidf = rgidf.loc[rgidf.RGIId.isin(test_ids)]
else:
rgidf = rgidf.sample(4)
log.workflow('Starting prepro run for RGI reg: {} '
'and border: {}'.format(rgi_reg, border))
log.workflow('Number of glaciers: {}'.format(len(rgidf)))
# L0 - go
gdirs = workflow.init_glacier_directories(rgidf, reset=True, force=True)
# Glacier stats
sum_dir = os.path.join(output_base_dir, 'L0', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# L0 OK - compress all in output directory
log.workflow('L0 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L0')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 0:
_time_log()
return
# L1 - Add dem files
if test_topofile:
cfg.PATHS['dem_file'] = test_topofile
# Which DEM source?
if dem_source.upper() == 'ALL':
# This is the complex one, just do the job and leave
log.workflow('Running prepro on ALL sources')
for i, s in enumerate(utils.DEM_SOURCES):
rs = i == 0
log.workflow('Running prepro on sources: {}'.format(s))
gdirs = workflow.init_glacier_directories(rgidf,
reset=rs,
force=rs)
workflow.execute_entity_task(tasks.define_glacier_region,
gdirs,
source=s)
workflow.execute_entity_task(_rename_dem_folder, gdirs, source=s)
# make a GeoTiff mask of the glacier, choose any source
workflow.execute_entity_task(gis.rasterio_glacier_mask,
gdirs,
source='ALL')
# Compress all in output directory
level_base_dir = os.path.join(output_base_dir, 'L1')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
_time_log()
return
# Force a given source
source = dem_source.upper() if dem_source else None
# L1 - go
workflow.execute_entity_task(tasks.define_glacier_region,
gdirs,
source=source)
# Glacier stats
sum_dir = os.path.join(output_base_dir, 'L1', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# L1 OK - compress all in output directory
log.workflow('L1 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L1')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 1:
_time_log()
return
# L2 - Tasks
# Check which glaciers will be processed as what
if elev_bands:
gdirs_band = gdirs
gdirs_cent = []
elif centerlines_only:
gdirs_band = []
gdirs_cent = gdirs
else:
# Default is to mix
# Curated list of large (> 50 km2) glaciers that don't run
# (CFL error) mostly because the centerlines are crap
# This is a really temporary fix until we have some better
# solution here
ids_to_bands = [
'RGI60-01.13696', 'RGI60-03.01710', 'RGI60-01.13635',
'RGI60-01.14443', 'RGI60-03.01678', 'RGI60-03.03274',
'RGI60-01.17566', 'RGI60-03.02849', 'RGI60-01.16201',
'RGI60-01.14683', 'RGI60-07.01506', 'RGI60-07.01559',
'RGI60-03.02687', 'RGI60-17.00172', 'RGI60-01.23649',
'RGI60-09.00077', 'RGI60-03.00994', 'RGI60-01.26738',
'RGI60-03.00283', 'RGI60-01.16121', 'RGI60-01.27108',
'RGI60-09.00132', 'RGI60-13.43483', 'RGI60-09.00069',
'RGI60-14.04404', 'RGI60-17.01218', 'RGI60-17.15877',
'RGI60-13.30888', 'RGI60-17.13796', 'RGI60-17.15825',
'RGI60-01.09783'
]
if rgi_reg == '19':
gdirs_band = gdirs
gdirs_cent = []
else:
gdirs_band = []
gdirs_cent = []
for gdir in gdirs:
if gdir.is_icecap or gdir.rgi_id in ids_to_bands:
gdirs_band.append(gdir)
else:
gdirs_cent.append(gdir)
log.workflow('Start flowline processing with: '
'N centerline type: {}, '
'N elev bands type: {}.'
''.format(len(gdirs_cent), len(gdirs_band)))
# HH2015 method
workflow.execute_entity_task(tasks.simple_glacier_masks, gdirs_band)
# Centerlines OGGM
workflow.execute_entity_task(tasks.glacier_masks, gdirs_cent)
if add_consensus:
from oggm.shop.bedtopo import add_consensus_thickness
workflow.execute_entity_task(add_consensus_thickness, gdirs_band)
workflow.execute_entity_task(add_consensus_thickness, gdirs_cent)
# Elev bands with var data
vn = 'consensus_ice_thickness'
workflow.execute_entity_task(tasks.elevation_band_flowline,
gdirs_band,
bin_variables=vn)
workflow.execute_entity_task(tasks.fixed_dx_elevation_band_flowline,
gdirs_band,
bin_variables=vn)
else:
# HH2015 method without it
task_list = [
tasks.elevation_band_flowline,
tasks.fixed_dx_elevation_band_flowline,
]
for task in task_list:
workflow.execute_entity_task(task, gdirs_band)
# HH2015 method
task_list = [
tasks.compute_downstream_line,
tasks.compute_downstream_bedshape,
]
for task in task_list:
workflow.execute_entity_task(task, gdirs_band)
# Centerlines OGGM
task_list = [
tasks.compute_centerlines,
tasks.initialize_flowlines,
tasks.compute_downstream_line,
tasks.compute_downstream_bedshape,
tasks.catchment_area,
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
]
for task in task_list:
workflow.execute_entity_task(task, gdirs_cent)
# Glacier stats
sum_dir = os.path.join(output_base_dir, 'L2', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# L2 OK - compress all in output directory
log.workflow('L2 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L2')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 2:
_time_log()
return
# L3 - Tasks
task_list = [
tasks.process_climate_data,
tasks.historical_climate_qc,
tasks.local_t_star,
tasks.mu_star_calibration,
]
for task in task_list:
workflow.execute_entity_task(task, gdirs)
# Inversion: we match the consensus
workflow.calibrate_inversion_from_consensus(gdirs,
apply_fs_on_mismatch=True,
error_on_mismatch=False)
# Do we want to match geodetic estimates?
# This affects only the bias so we can actually do this *after*
# the inversion, but we really want to take calving into account here
if match_geodetic_mb:
workflow.match_regional_geodetic_mb(gdirs, rgi_reg)
# We get ready for modelling
workflow.execute_entity_task(tasks.init_present_time_glacier, gdirs)
# Glacier stats
sum_dir = os.path.join(output_base_dir, 'L3', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
opath = os.path.join(sum_dir, 'climate_statistics_{}.csv'.format(rgi_reg))
utils.compile_climate_statistics(gdirs, path=opath)
opath = os.path.join(sum_dir,
'fixed_geometry_mass_balance_{}.csv'.format(rgi_reg))
utils.compile_fixed_geometry_mass_balance(gdirs, path=opath)
# L3 OK - compress all in output directory
log.workflow('L3 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L3')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 3:
_time_log()
return
# L4 - No tasks: add some stats for consistency and make the dirs small
sum_dir_L3 = sum_dir
sum_dir = os.path.join(output_base_dir, 'L4', 'summary')
utils.mkdir(sum_dir)
# Copy L3 files for consistency
for bn in [
'glacier_statistics', 'climate_statistics',
'fixed_geometry_mass_balance'
]:
ipath = os.path.join(sum_dir_L3, bn + '_{}.csv'.format(rgi_reg))
opath = os.path.join(sum_dir, bn + '_{}.csv'.format(rgi_reg))
shutil.copyfile(ipath, opath)
# Copy mini data to new dir
mini_base_dir = os.path.join(working_dir, 'mini_perglacier')
mini_gdirs = workflow.execute_entity_task(tasks.copy_to_basedir,
gdirs,
base_dir=mini_base_dir)
# L4 OK - compress all in output directory
log.workflow('L4 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L4')
workflow.execute_entity_task(utils.gdir_to_tar,
mini_gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 4:
_time_log()
return
# L5 - spinup run in mini gdirs
gdirs = mini_gdirs
# Get end date. The first gdir might have blown up, try some others
i = 0
while True:
if i >= len(gdirs):
raise RuntimeError('Found no valid glaciers!')
try:
y0 = gdirs[i].get_climate_info()['baseline_hydro_yr_0']
# One adds 1 because the run ends at the end of the year
ye = gdirs[i].get_climate_info()['baseline_hydro_yr_1'] + 1
break
except BaseException:
i += 1
# OK - run
workflow.execute_entity_task(tasks.run_from_climate_data,
gdirs,
min_ys=y0,
ye=ye,
output_filesuffix='_historical')
# Now compile the output
sum_dir = os.path.join(output_base_dir, 'L5', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir,
'historical_run_output_{}.nc'.format(rgi_reg))
utils.compile_run_output(gdirs, path=opath, input_filesuffix='_historical')
# Glacier statistics we recompute here for error analysis
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# Other stats for consistency
for bn in ['climate_statistics', 'fixed_geometry_mass_balance']:
ipath = os.path.join(sum_dir_L3, bn + '_{}.csv'.format(rgi_reg))
opath = os.path.join(sum_dir, bn + '_{}.csv'.format(rgi_reg))
shutil.copyfile(ipath, opath)
# Add the extended files
pf = os.path.join(sum_dir, 'historical_run_output_{}.nc'.format(rgi_reg))
mf = os.path.join(sum_dir,
'fixed_geometry_mass_balance_{}.csv'.format(rgi_reg))
# This is crucial - extending calving only with L3 data!!!
sf = os.path.join(sum_dir_L3, 'glacier_statistics_{}.csv'.format(rgi_reg))
opath = os.path.join(
sum_dir, 'historical_run_output_extended_{}.nc'.format(rgi_reg))
utils.extend_past_climate_run(past_run_file=pf,
fixed_geometry_mb_file=mf,
glacier_statistics_file=sf,
path=opath)
# L5 OK - compress all in output directory
log.workflow('L5 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L5')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
_time_log()
def run_prepro_levels(rgi_version=None,
rgi_reg=None,
border=None,
output_folder='',
working_dir='',
is_test=False,
demo=False,
test_rgidf=None,
test_intersects_file=None,
test_topofile=None,
test_crudir=None):
"""Does the actual job.
Parameters
----------
rgi_version : str
the RGI version to use (defaults to cfg.PARAMS)
rgi_reg : str
the RGI region to process
border : int
the number of pixels at the maps border
output_folder : str
path to the output folder (where to put the preprocessed tar files)
working_dir : str
path to the OGGM working directory
is_test : bool
to test on a couple of glaciers only!
demo : bool
to run the prepro for the list of demo glaciers
test_rgidf : shapefile
for testing purposes only
test_intersects_file : shapefile
for testing purposes only
test_topofile : str
for testing purposes only
test_crudir : str
for testing purposes only
"""
# TODO: temporarily silence Fiona deprecation warnings
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
# Module logger
log = logging.getLogger(__name__)
# Time
start = time.time()
# Initialize OGGM and set up the run parameters
cfg.initialize(logging_level='WORKFLOW')
# Local paths
utils.mkdir(working_dir)
cfg.PATHS['working_dir'] = working_dir
# Use multiprocessing?
cfg.PARAMS['use_multiprocessing'] = True
# How many grid points around the glacier?
# Make it large if you expect your glaciers to grow large
cfg.PARAMS['border'] = border
# Set to True for operational runs
cfg.PARAMS['continue_on_error'] = True
# For statistics
climate_periods = [1920, 1960, 2000]
if rgi_version is None:
rgi_version = cfg.PARAMS['rgi_version']
rgi_dir_name = 'RGI{}'.format(rgi_version)
border_dir_name = 'b_{:03d}'.format(border)
base_dir = os.path.join(output_folder, rgi_dir_name, border_dir_name)
# Add a package version file
utils.mkdir(base_dir)
opath = os.path.join(base_dir, 'package_versions.txt')
with open(opath, 'w') as vfile:
vfile.write(utils.show_versions(logger=log))
if demo:
rgidf = utils.get_rgi_glacier_entities(cfg.DATA['demo_glaciers'].index)
elif test_rgidf is None:
# Get the RGI file
rgidf = gpd.read_file(
utils.get_rgi_region_file(rgi_reg, version=rgi_version))
# We use intersects
rgif = utils.get_rgi_intersects_region_file(rgi_reg,
version=rgi_version)
cfg.set_intersects_db(rgif)
else:
rgidf = test_rgidf
cfg.set_intersects_db(test_intersects_file)
if is_test:
# Just for fun
rgidf = rgidf.sample(4)
# Sort for more efficient parallel computing
rgidf = rgidf.sort_values('Area', ascending=False)
log.workflow('Starting prepro run for RGI reg: {} '
'and border: {}'.format(rgi_reg, border))
log.workflow('Number of glaciers: {}'.format(len(rgidf)))
# Input
if test_topofile:
cfg.PATHS['dem_file'] = test_topofile
# L1 - initialize working directories
gdirs = workflow.init_glacier_regions(rgidf, reset=True, force=True)
# Glacier stats
sum_dir = os.path.join(base_dir, 'L1', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# L1 OK - compress all in output directory
l_base_dir = os.path.join(base_dir, 'L1')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=l_base_dir)
utils.base_dir_to_tar(l_base_dir)
# L2 - Tasks
# Pre-download other files just in case
if test_crudir is None:
_ = utils.get_cru_file(var='tmp')
_ = utils.get_cru_file(var='pre')
else:
cfg.PATHS['cru_dir'] = test_crudir
workflow.execute_entity_task(tasks.process_cru_data, gdirs)
# Glacier stats
sum_dir = os.path.join(base_dir, 'L2', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# L2 OK - compress all in output directory
l_base_dir = os.path.join(base_dir, 'L2')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=l_base_dir)
utils.base_dir_to_tar(l_base_dir)
# L3 - Tasks
task_list = [
tasks.glacier_masks,
tasks.compute_centerlines,
tasks.initialize_flowlines,
tasks.compute_downstream_line,
tasks.compute_downstream_bedshape,
tasks.catchment_area,
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
tasks.local_t_star,
tasks.mu_star_calibration,
tasks.prepare_for_inversion,
tasks.mass_conservation_inversion,
tasks.filter_inversion_output,
]
for task in task_list:
workflow.execute_entity_task(task, gdirs)
# Glacier stats
sum_dir = os.path.join(base_dir, 'L3', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
opath = os.path.join(sum_dir, 'climate_statistics_{}.csv'.format(rgi_reg))
utils.compile_climate_statistics(gdirs,
add_climate_period=climate_periods,
path=opath)
# L3 OK - compress all in output directory
l_base_dir = os.path.join(base_dir, 'L3')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=l_base_dir)
utils.base_dir_to_tar(l_base_dir)
# L4 - Tasks
workflow.execute_entity_task(tasks.init_present_time_glacier, gdirs)
# Glacier stats
sum_dir = os.path.join(base_dir, 'L4', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# Copy mini data to new dir
base_dir = os.path.join(base_dir, 'L4')
mini_gdirs = workflow.execute_entity_task(tasks.copy_to_basedir,
gdirs,
base_dir=base_dir)
# L4 OK - compress all in output directory
workflow.execute_entity_task(utils.gdir_to_tar, mini_gdirs, delete=True)
utils.base_dir_to_tar(base_dir)
# Log
m, s = divmod(time.time() - start, 60)
h, m = divmod(m, 60)
log.workflow('OGGM prepro_levels is done! Time needed: '
'{:02d}:{:02d}:{:02d}'.format(int(h), int(m), int(s)))
def run_cmip():
"""
"""
# Initialize OGGM and set up the default run parameters
vascaling.initialize(logging_level='DEBUG')
rgi_version = '62'
cfg.PARAMS['border'] = 80
# CLUSTER paths
wdir = os.environ.get('WORKDIR', '')
utils.mkdir(wdir)
cfg.PATHS['working_dir'] = wdir
outdir = os.environ.get('OUTDIR', '')
utils.mkdir(outdir)
# define the baseline climate CRU or HISTALP
cfg.PARAMS['baseline_climate'] = 'CRU'
# set the mb hyper parameters accordingly
cfg.PARAMS['prcp_scaling_factor'] = 3
cfg.PARAMS['temp_melt'] = 0
cfg.PARAMS['temp_all_solid'] = 4
cfg.PARAMS['prcp_default_gradient'] = 4e-4
cfg.PARAMS['run_mb_calibration'] = False
# set minimum ice thickness to include in glacier length computation
# this reduces weird spikes in length records
cfg.PARAMS['min_ice_thick_for_length'] = 0.1
# the bias is defined to be zero during the calibration process,
# which is why we don't use it here to reproduce the results
cfg.PARAMS['use_bias_for_run'] = True
# read RGI entry for the glaciers as DataFrame
# containing the outline area as shapefile
# RGI glaciers
rgi_reg = os.environ.get('RGI_REG', '')
if rgi_reg not in ['{:02d}'.format(r) for r in range(1, 20)]:
raise RuntimeError('Need an RGI Region')
rgi_ids = gpd.read_file(
utils.get_rgi_region_file(rgi_reg, version=rgi_version))
# For greenland we omit connectivity level 2
if rgi_reg == '05':
rgi_ids = rgi_ids.loc[rgi_ids['Connect'] != 2]
# get and set path to intersect shapefile
intersects_db = utils.get_rgi_intersects_region_file(region=rgi_reg)
cfg.set_intersects_db(intersects_db)
# operational run, all glaciers should run
cfg.PARAMS['continue_on_error'] = True
# Module logger
log = logging.getLogger(__name__)
log.workflow('Starting run for RGI reg {}'.format(rgi_reg))
# Go - get the pre-processed glacier directories
base_url = 'https://cluster.klima.uni-bremen.de/' \
'~moberrauch/prepro_vas_paper/'
gdirs = workflow.init_glacier_directories(rgi_ids, from_prepro_level=3,
prepro_base_url=base_url,
prepro_rgi_version=rgi_version)
# read gcm list
gcms = pd.read_csv('/home/www/oggm/cmip6/all_gcm_list.csv', index_col=0)
# iterate over all specified GCMs
for gcm in sys.argv[1:]:
# iterate over all SSPs (Shared Socioeconomic Pathways)
df1 = gcms.loc[gcms.gcm == gcm]
for ssp in df1.ssp.unique():
df2 = df1.loc[df1.ssp == ssp]
assert len(df2) == 2
# get temperature projections
ft = df2.loc[df2['var'] == 'tas'].iloc[0]
# get precipitation projections
fp = df2.loc[df2['var'] == 'pr'].iloc[0].path
rid = ft.fname.replace('_r1i1p1f1_tas.nc', '')
ft = ft.path
log.workflow('Starting run for {}'.format(rid))
workflow.execute_entity_task(gcm_climate.process_cmip_data, gdirs,
# recognize the climate file for later
filesuffix='_' + rid,
# temperature projections
fpath_temp=ft,
# precip projections
fpath_precip=fp,
year_range=('1981', '2020'))
workflow.execute_entity_task(vascaling.run_from_climate_data,
gdirs,
# use gcm_data, not climate_historical
climate_filename='gcm_data',
# use a different scenario
climate_input_filesuffix='_' + rid,
# this is important! Start from 2019
init_model_filesuffix='_historical',
# recognize the run for later
output_filesuffix=rid,
return_value=False)
gcm_dir = os.path.join(outdir, 'RGI' + rgi_reg, gcm)
utils.mkdir(gcm_dir)
utils.compile_run_output(gdirs, input_filesuffix=rid,
path=os.path.join(gcm_dir, rid + '.nc'))
log.workflow('OGGM Done')
def run_prepro_levels(rgi_version=None,
rgi_reg=None,
border=None,
output_folder='',
working_dir='',
dem_source='',
is_test=False,
test_ids=None,
demo=False,
test_rgidf=None,
test_intersects_file=None,
test_topofile=None,
disable_mp=False,
params_file=None,
elev_bands=False,
match_regional_geodetic_mb=False,
match_geodetic_mb_per_glacier=False,
evolution_model='fl_sia',
centerlines_only=False,
override_params=None,
add_consensus=False,
start_level=None,
start_base_url=None,
max_level=5,
ref_tstars_base_url='',
logging_level='WORKFLOW',
disable_dl_verify=False,
dynamic_spinup=False,
continue_on_error=True):
"""Generate the preprocessed OGGM glacier directories for this OGGM version
Parameters
----------
rgi_version : str
the RGI version to use (defaults to cfg.PARAMS)
rgi_reg : str
the RGI region to process
border : int
the number of pixels at the maps border
output_folder : str
path to the output folder (where to put the preprocessed tar files)
dem_source : str
which DEM source to use: default, SOURCE_NAME or ALL
working_dir : str
path to the OGGM working directory
ref_tstars_base_url : str
url where to find the pre-calibrated reference tstar list.
Required as of v1.4.
params_file : str
path to the OGGM parameter file (to override defaults)
is_test : bool
to test on a couple of glaciers only!
test_ids : list
if is_test: list of ids to process
demo : bool
to run the prepro for the list of demo glaciers
test_rgidf : shapefile
for testing purposes only
test_intersects_file : shapefile
for testing purposes only
test_topofile : str
for testing purposes only
test_crudir : str
for testing purposes only
disable_mp : bool
disable multiprocessing
elev_bands : bool
compute all flowlines based on the Huss&Hock 2015 method instead
of the OGGM default, which is a mix of elev_bands and centerlines.
centerlines_only : bool
compute all flowlines based on the OGGM centerline(s) method instead
of the OGGM default, which is a mix of elev_bands and centerlines.
match_regional_geodetic_mb : str
match the regional mass-balance estimates at the regional level
('hugonnet': Hugonnet et al., 2020 or 'zemp': Zemp et al., 2019).
match_geodetic_mb_per_glacier : str
match the mass-balance estimates at the glacier level
(currently only 'hugonnet': Hugonnet et al., 2020).
evolution_model : str
which geometry evolution model to use: `fl_sia` (default),
or `massredis` (mass redistribution curve).
add_consensus : bool
adds (reprojects) the consensus estimates thickness to the glacier
directories. With elev_bands=True, the data will also be binned.
start_level : int
the pre-processed level to start from (default is to start from
scratch). If set, you'll need to indicate start_base_url as well.
start_base_url : str
the pre-processed base-url to fetch the data from.
max_level : int
the maximum pre-processing level before stopping
logging_level : str
the logging level to use (DEBUG, INFO, WARNING, WORKFLOW)
override_params : dict
a dict of parameters to override.
disable_dl_verify : bool
disable the hash verification of OGGM downloads
dynamic_spinup: str
include a dynamic spinup matching 'area' OR 'volume' at the RGI-date
"""
# Input check
if max_level not in [1, 2, 3, 4, 5]:
raise InvalidParamsError('max_level should be one of [1, 2, 3, 4, 5]')
if start_level is not None:
if start_level not in [0, 1, 2]:
raise InvalidParamsError('start_level should be one of [0, 1, 2]')
if start_level > 0 and start_base_url is None:
raise InvalidParamsError('With start_level, please also indicate '
'start_base_url')
else:
start_level = 0
if match_regional_geodetic_mb and match_geodetic_mb_per_glacier:
raise InvalidParamsError(
'match_regional_geodetic_mb incompatible with '
'match_geodetic_mb_per_glacier!')
if match_geodetic_mb_per_glacier and match_geodetic_mb_per_glacier != 'hugonnet':
raise InvalidParamsError('Currently only `hugonnet` is available for '
'match_geodetic_mb_per_glacier.')
if evolution_model not in ['fl_sia', 'massredis']:
raise InvalidParamsError('evolution_model should be one of '
"['fl_sia', 'massredis'].")
if dynamic_spinup and dynamic_spinup not in ['area', 'volume']:
raise InvalidParamsError(f"Dynamic spinup option '{dynamic_spinup}' "
"not supported")
if dynamic_spinup and evolution_model == 'massredis':
raise InvalidParamsError("Dynamic spinup is not working/tested"
"with massredis!")
# Time
start = time.time()
def _time_log():
# Log util
m, s = divmod(time.time() - start, 60)
h, m = divmod(m, 60)
log.workflow('OGGM prepro_levels is done! Time needed: '
'{:02d}:{:02d}:{:02d}'.format(int(h), int(m), int(s)))
# Local paths
if override_params is None:
override_params = {}
utils.mkdir(working_dir)
override_params['working_dir'] = working_dir
# Initialize OGGM and set up the run parameters
cfg.initialize(file=params_file,
params=override_params,
logging_level=logging_level,
future=True)
if match_geodetic_mb_per_glacier and (cfg.PARAMS['hydro_month_nh'] != 1 or
cfg.PARAMS['hydro_month_sh'] != 1):
raise InvalidParamsError('We recommend to set hydro_month_nh and sh '
'to 1 for the geodetic MB calibration per '
'glacier.')
# Use multiprocessing?
cfg.PARAMS['use_multiprocessing'] = not disable_mp
# How many grid points around the glacier?
# Make it large if you expect your glaciers to grow large
cfg.PARAMS['border'] = border
# Set to True for operational runs
cfg.PARAMS['continue_on_error'] = continue_on_error
# Check for the integrity of the files OGGM downloads at run time
# For large files (e.g. using a 1 tif DEM like ALASKA) calculating the hash
# takes a long time, so deactivating this can make sense
cfg.PARAMS['dl_verify'] = not disable_dl_verify
# Other things that make sense
cfg.PARAMS['store_model_geometry'] = True
# Log the parameters
msg = '# OGGM Run parameters:'
for k, v in cfg.PARAMS.items():
if type(v) in [pd.DataFrame, dict]:
continue
msg += '\n {}: {}'.format(k, v)
log.workflow(msg)
if rgi_version is None:
rgi_version = cfg.PARAMS['rgi_version']
output_base_dir = os.path.join(output_folder, 'RGI{}'.format(rgi_version),
'b_{:03d}'.format(border))
# Add a package version file
utils.mkdir(output_base_dir)
opath = os.path.join(output_base_dir, 'package_versions.txt')
with open(opath, 'w') as vfile:
vfile.write(utils.show_versions(logger=log))
if demo:
rgidf = utils.get_rgi_glacier_entities(cfg.DATA['demo_glaciers'].index)
elif test_rgidf is None:
# Get the RGI file
rgidf = gpd.read_file(
utils.get_rgi_region_file(rgi_reg, version=rgi_version))
# We use intersects
rgif = utils.get_rgi_intersects_region_file(rgi_reg,
version=rgi_version)
cfg.set_intersects_db(rgif)
# Some RGI input quality checks - this is based on visual checks
# of large glaciers in the RGI
ids_to_ice_cap = [
'RGI60-05.10315', # huge Greenland ice cap
'RGI60-03.01466', # strange thing next to Devon
'RGI60-09.00918', # Academy of sciences Ice cap
'RGI60-09.00969',
'RGI60-09.00958',
'RGI60-09.00957',
]
rgidf.loc[rgidf.RGIId.isin(ids_to_ice_cap), 'Form'] = '1'
# In AA almost all large ice bodies are actually ice caps
if rgi_reg == '19':
rgidf.loc[rgidf.Area > 100, 'Form'] = '1'
# For greenland we omit connectivity level 2
if rgi_reg == '05':
rgidf = rgidf.loc[rgidf['Connect'] != 2]
else:
rgidf = test_rgidf
cfg.set_intersects_db(test_intersects_file)
if is_test:
if test_ids is not None:
rgidf = rgidf.loc[rgidf.RGIId.isin(test_ids)]
else:
rgidf = rgidf.sample(4)
if max_level > 2:
# Also use ref tstars
utils.apply_test_ref_tstars()
if max_level > 2 and ref_tstars_base_url:
workflow.download_ref_tstars(base_url=ref_tstars_base_url)
log.workflow('Starting prepro run for RGI reg: {} '
'and border: {}'.format(rgi_reg, border))
log.workflow('Number of glaciers: {}'.format(len(rgidf)))
# L0 - go
if start_level == 0:
gdirs = workflow.init_glacier_directories(rgidf,
reset=True,
force=True)
# Glacier stats
sum_dir = os.path.join(output_base_dir, 'L0', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir,
'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# L0 OK - compress all in output directory
log.workflow('L0 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L0')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 0:
_time_log()
return
else:
gdirs = workflow.init_glacier_directories(
rgidf,
reset=True,
force=True,
from_prepro_level=start_level,
prepro_border=border,
prepro_rgi_version=rgi_version,
prepro_base_url=start_base_url)
# L1 - Add dem files
if start_level == 0:
if test_topofile:
cfg.PATHS['dem_file'] = test_topofile
# Which DEM source?
if dem_source.upper() == 'ALL':
# This is the complex one, just do the job and leave
log.workflow('Running prepro on ALL sources')
for i, s in enumerate(utils.DEM_SOURCES):
rs = i == 0
log.workflow('Running prepro on sources: {}'.format(s))
gdirs = workflow.init_glacier_directories(rgidf,
reset=rs,
force=rs)
workflow.execute_entity_task(tasks.define_glacier_region,
gdirs,
source=s)
workflow.execute_entity_task(_rename_dem_folder,
gdirs,
source=s)
# make a GeoTiff mask of the glacier, choose any source
workflow.execute_entity_task(gis.rasterio_glacier_mask,
gdirs,
source='ALL')
# Compress all in output directory
level_base_dir = os.path.join(output_base_dir, 'L1')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
_time_log()
return
# Force a given source
source = dem_source.upper() if dem_source else None
# L1 - go
workflow.execute_entity_task(tasks.define_glacier_region,
gdirs,
source=source)
# Glacier stats
sum_dir = os.path.join(output_base_dir, 'L1', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir,
'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# L1 OK - compress all in output directory
log.workflow('L1 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L1')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 1:
_time_log()
return
# L2 - Tasks
if start_level <= 1:
# Check which glaciers will be processed as what
if elev_bands:
gdirs_band = gdirs
gdirs_cent = []
elif centerlines_only:
gdirs_band = []
gdirs_cent = gdirs
else:
# Default is to centerlines_only, but it used to be a mix
# (e.g. bands for ice caps, etc)
# I still keep this logic here in case we want to mix again
gdirs_band = []
gdirs_cent = gdirs
log.workflow('Start flowline processing with: '
'N centerline type: {}, '
'N elev bands type: {}.'
''.format(len(gdirs_cent), len(gdirs_band)))
# HH2015 method
workflow.execute_entity_task(tasks.simple_glacier_masks, gdirs_band)
# Centerlines OGGM
workflow.execute_entity_task(tasks.glacier_masks, gdirs_cent)
if add_consensus:
from oggm.shop.bedtopo import add_consensus_thickness
workflow.execute_entity_task(add_consensus_thickness, gdirs_band)
workflow.execute_entity_task(add_consensus_thickness, gdirs_cent)
# Elev bands with var data
vn = 'consensus_ice_thickness'
workflow.execute_entity_task(tasks.elevation_band_flowline,
gdirs_band,
bin_variables=vn)
workflow.execute_entity_task(
tasks.fixed_dx_elevation_band_flowline,
gdirs_band,
bin_variables=vn)
else:
# HH2015 method without it
task_list = [
tasks.elevation_band_flowline,
tasks.fixed_dx_elevation_band_flowline,
]
for task in task_list:
workflow.execute_entity_task(task, gdirs_band)
# Centerlines OGGM
task_list = [
tasks.compute_centerlines,
tasks.initialize_flowlines,
tasks.catchment_area,
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
]
for task in task_list:
workflow.execute_entity_task(task, gdirs_cent)
# Same for all glaciers
if border >= 20:
task_list = [
tasks.compute_downstream_line,
tasks.compute_downstream_bedshape,
]
for task in task_list:
workflow.execute_entity_task(task, gdirs)
else:
log.workflow('L2: for map border values < 20, wont compute '
'downstream lines.')
# Glacier stats
sum_dir = os.path.join(output_base_dir, 'L2', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir,
'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# And for level 2: shapes
if len(gdirs_cent) > 0:
opath = os.path.join(sum_dir, 'centerlines_{}.shp'.format(rgi_reg))
utils.write_centerlines_to_shape(gdirs_cent,
to_tar=True,
path=opath)
# L2 OK - compress all in output directory
log.workflow('L2 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L2')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 2:
_time_log()
return
# L3 - Tasks
sum_dir = os.path.join(output_base_dir, 'L3', 'summary')
utils.mkdir(sum_dir)
# Climate
workflow.execute_entity_task(tasks.process_climate_data, gdirs)
if cfg.PARAMS['climate_qc_months'] > 0:
workflow.execute_entity_task(tasks.historical_climate_qc, gdirs)
if match_geodetic_mb_per_glacier:
utils.get_geodetic_mb_dataframe() # Small optim to avoid concurrency
workflow.execute_entity_task(
tasks.mu_star_calibration_from_geodetic_mb, gdirs)
workflow.execute_entity_task(tasks.apparent_mb_from_any_mb, gdirs)
else:
workflow.execute_entity_task(tasks.local_t_star, gdirs)
workflow.execute_entity_task(tasks.mu_star_calibration, gdirs)
# Inversion: we match the consensus
filter = border >= 20
workflow.calibrate_inversion_from_consensus(gdirs,
apply_fs_on_mismatch=True,
error_on_mismatch=False,
filter_inversion_output=filter)
# Do we want to match geodetic estimates?
# This affects only the bias so we can actually do this *after*
# the inversion, but we really want to take calving into account here
if match_regional_geodetic_mb:
opath = os.path.join(
sum_dir, 'fixed_geometry_mass_balance_'
'before_match_{}.csv'.format(rgi_reg))
utils.compile_fixed_geometry_mass_balance(gdirs, path=opath)
workflow.match_regional_geodetic_mb(gdirs,
rgi_reg=rgi_reg,
dataset=match_regional_geodetic_mb)
# We get ready for modelling
if border >= 20:
workflow.execute_entity_task(tasks.init_present_time_glacier, gdirs)
else:
log.workflow(
'L3: for map border values < 20, wont initialize glaciers '
'for the run.')
# Glacier stats
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
opath = os.path.join(sum_dir, 'climate_statistics_{}.csv'.format(rgi_reg))
utils.compile_climate_statistics(gdirs, path=opath)
opath = os.path.join(sum_dir,
'fixed_geometry_mass_balance_{}.csv'.format(rgi_reg))
utils.compile_fixed_geometry_mass_balance(gdirs, path=opath)
# L3 OK - compress all in output directory
log.workflow('L3 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L3')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 3:
_time_log()
return
if border < 20:
log.workflow('L3: for map border values < 20, wont compute L4 and L5.')
_time_log()
return
# L4 - No tasks: add some stats for consistency and make the dirs small
sum_dir_L3 = sum_dir
sum_dir = os.path.join(output_base_dir, 'L4', 'summary')
utils.mkdir(sum_dir)
# Copy L3 files for consistency
for bn in [
'glacier_statistics', 'climate_statistics',
'fixed_geometry_mass_balance'
]:
ipath = os.path.join(sum_dir_L3, bn + '_{}.csv'.format(rgi_reg))
opath = os.path.join(sum_dir, bn + '_{}.csv'.format(rgi_reg))
shutil.copyfile(ipath, opath)
# Copy mini data to new dir
mini_base_dir = os.path.join(working_dir, 'mini_perglacier',
'RGI{}'.format(rgi_version),
'b_{:03d}'.format(border))
mini_gdirs = workflow.execute_entity_task(tasks.copy_to_basedir,
gdirs,
base_dir=mini_base_dir)
# L4 OK - compress all in output directory
log.workflow('L4 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L4')
workflow.execute_entity_task(utils.gdir_to_tar,
mini_gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 4:
_time_log()
return
# L5 - spinup run in mini gdirs
gdirs = mini_gdirs
# Get end date. The first gdir might have blown up, try some others
i = 0
while True:
if i >= len(gdirs):
raise RuntimeError('Found no valid glaciers!')
try:
y0 = gdirs[i].get_climate_info()['baseline_hydro_yr_0']
# One adds 1 because the run ends at the end of the year
ye = gdirs[i].get_climate_info()['baseline_hydro_yr_1'] + 1
break
except BaseException:
i += 1
# Which model?
if evolution_model == 'massredis':
from oggm.core.flowline import MassRedistributionCurveModel
evolution_model = MassRedistributionCurveModel
else:
from oggm.core.flowline import FluxBasedModel
evolution_model = FluxBasedModel
# OK - run
if dynamic_spinup:
workflow.execute_entity_task(
tasks.run_dynamic_spinup,
gdirs,
evolution_model=evolution_model,
minimise_for=dynamic_spinup,
precision_percent=1,
output_filesuffix='_dynamic_spinup',
)
workflow.execute_entity_task(tasks.run_from_climate_data,
gdirs,
min_ys=y0,
ye=ye,
evolution_model=evolution_model,
init_model_filesuffix='_dynamic_spinup',
output_filesuffix='_hist_spin')
workflow.execute_entity_task(tasks.merge_consecutive_run_outputs,
gdirs,
input_filesuffix_1='_dynamic_spinup',
input_filesuffix_2='_hist_spin',
output_filesuffix='_historical_spinup',
delete_input=True)
workflow.execute_entity_task(tasks.run_from_climate_data,
gdirs,
min_ys=y0,
ye=ye,
evolution_model=evolution_model,
output_filesuffix='_historical')
# Now compile the output
sum_dir = os.path.join(output_base_dir, 'L5', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, f'historical_run_output_{rgi_reg}.nc')
utils.compile_run_output(gdirs, path=opath, input_filesuffix='_historical')
if dynamic_spinup:
opath = os.path.join(sum_dir,
f'historical_spinup_run_output_{rgi_reg}.nc')
utils.compile_run_output(gdirs,
path=opath,
input_filesuffix='_historical_spinup')
# Glacier statistics we recompute here for error analysis
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# Other stats for consistency
for bn in ['climate_statistics', 'fixed_geometry_mass_balance']:
ipath = os.path.join(sum_dir_L3, bn + '_{}.csv'.format(rgi_reg))
opath = os.path.join(sum_dir, bn + '_{}.csv'.format(rgi_reg))
shutil.copyfile(ipath, opath)
# Add the extended files
pf = os.path.join(sum_dir, 'historical_run_output_{}.nc'.format(rgi_reg))
mf = os.path.join(sum_dir,
'fixed_geometry_mass_balance_{}.csv'.format(rgi_reg))
# This is crucial - extending calving only possible with L3 data!!!
sf = os.path.join(sum_dir_L3, 'glacier_statistics_{}.csv'.format(rgi_reg))
opath = os.path.join(
sum_dir, 'historical_run_output_extended_{}.nc'.format(rgi_reg))
utils.extend_past_climate_run(past_run_file=pf,
fixed_geometry_mb_file=mf,
glacier_statistics_file=sf,
path=opath)
# L5 OK - compress all in output directory
log.workflow('L5 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L5')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
_time_log()
# Set to True for operational runs
cfg.PARAMS['continue_on_error'] = False
# test directory
idir = g2ti.geometry_dir
cfg.PATHS['working_dir'] = '/home/mowglie/disk/G2TI/vas_plot'
utils.mkdir(cfg.PATHS['working_dir'], reset=True)
df = pd.read_csv(g2ti.index_file)
df = df.loc[df.n_pts > 49] # 582 glaciers
rgi_ids = df.RGIId.values
flist = [idir + r[:8] + '/' + r for r in rgi_ids]
cfg.set_intersects_db(
utils.get_rgi_intersects_region_file(version=version, rgi_ids=rgi_ids))
gdirs = g2tasks.parallel_define(flist)
# Preprocessing tasks
task_list = [
g2tasks.g2ti_masks,
]
for task in task_list:
execute_entity_task(task, gdirs)
from g2ti import plots
from functools import partial
func = partial(plots.plot_domain_with_gtd, autosave=True)
execute_entity_task(func, gdirs)
# Log
def equilibrium_run_vas(rgi_ids,
use_random_mb=True,
path=True,
temp_biases=(0, +0.5, -0.5),
use_bias_for_run=False,
suffixes=('_bias_zero', '_bias_p', '_bias_n'),
tstar=None,
vas_c_length_m=None,
vas_c_area_m2=None,
**kwargs):
""" The routine runs all steps for the equilibrium experiments using the
volume/area scaling model:
- OGGM preprocessing, including initialization, GIS tasks, climate tasks and
massbalance tasks.
- Run model for all glaciers with constant (or random) massbalance model
over 3000 years (default value).
- Process the model output dataset(s), i.e. normalization, average/sum, ...
The final dataset containing all results is returned. Given a path is is
also stored to file.
Parameters
----------
rgi_ids: array-like
List of RGI IDs for which the equilibrium experiments are performed.
use_random_mb: bool, optional, default=True
Choose between random massbalance model and constant massbalance model.
path: bool or str, optional, default=True
If a path is given (or True), the resulting dataset is stored to file.
temp_biases: array-like, optional, default=(0, +0.5, -0.5)
List of temperature biases (float, in degC) for the mass balance model.
suffixes: array-like, optional, default=['_normal', '_bias_p', '_bias_n']
Descriptive suffixes corresponding to the given temperature biases.
tstar: float, optional, default=None
'Equilibrium year' used for the mass balance calibration.
vas_c_length_m: float, optional, default=None
Scaling constant for volume/length scaling
vas_c_area_m2: float, optional, default=None
Scaling constant for volume/area scaling
kwargs:
Additional key word arguments for the `run_random_climate` or
`run_constant_climate` routines of the vascaling module.
Returns
-------
Dataset containing yearly values of all glacier geometries.
"""
# assert correct output file suffixes for temp biases
if len(temp_biases) != len(suffixes):
raise RuntimeError("Each given temperature bias must have its "
"corresponding suffix")
# OGGM preprocessing
# ------------------
# compute RGI region and version from RGI IDs
# assuming all they are all the same
rgi_region = (rgi_ids[0].split('-')[-1]).split('.')[0]
rgi_version = (rgi_ids[0].split('-')[0])[-2:]
# load default parameter file
cfg.initialize()
# create working directory
wdir = '/Users/oberrauch/work/master/working_directories/'
wdir += 'test_cluster'
if not os.path.exists(wdir):
os.makedirs(wdir)
# shutil.rmtree(wdir)
# os.makedirs(wdir)
# set path to working directory
cfg.PATHS['working_dir'] = wdir
# set RGI verion and region
cfg.PARAMS['rgi_version'] = rgi_version
# define how many grid points to use around the glacier,
# if you expect the glacier to grow large use a larger border
cfg.PARAMS['border'] = 120
# we use HistAlp climate data
cfg.PARAMS['baseline_climate'] = 'HISTALP'
# set the mb hyper parameters accordingly
cfg.PARAMS['prcp_scaling_factor'] = 1.75
cfg.PARAMS['temp_melt'] = -1.75
# change scaling constants for lenght and area
if vas_c_length_m:
cfg.PARAMS['vas_c_length_m'] = vas_c_length_m
if vas_c_area_m2:
cfg.PARAMS['vas_c_area_m2'] = vas_c_area_m2
# the bias is defined to be zero during the calibration process,
# which is why we don't use it here to reproduce the results
cfg.PARAMS['use_bias_for_run'] = use_bias_for_run
# read RGI entry for the glaciers as DataFrame
# containing the outline area as shapefile
rgidf = utils.get_rgi_glacier_entities(rgi_ids)
# get and set path to intersect shapefile
intersects_db = utils.get_rgi_intersects_region_file(region=rgi_region)
cfg.set_intersects_db(intersects_db)
# sort by area for more efficient parallel computing
rgidf = rgidf.sort_values('Area', ascending=False)
cfg.PARAMS['use_multiprocessing'] = True
# operational run, all glaciers should run
cfg.PARAMS['continue_on_error'] = True
# initialize the GlacierDirectory
gdirs = workflow.init_glacier_directories(rgidf, reset=False, force=True)
# define the local grid and glacier mask
workflow.execute_entity_task(gis.define_glacier_region, gdirs)
workflow.execute_entity_task(gis.glacier_masks, gdirs)
# process the given climate file
workflow.execute_entity_task(climate.process_climate_data, gdirs)
# compute local t* and the corresponding mu*
workflow.execute_entity_task(vascaling.local_t_star,
gdirs,
tstar=tstar,
bias=0)
# Run model with constant/random mass balance model
# -------------------------------------------------
# use t* as center year, even if specified differently
kwargs['y0'] = tstar
# run for 3000 years if not specified otherwise
kwargs.setdefault('nyears', 3000)
if use_random_mb:
# set random seed to get reproducible results
kwargs.setdefault('seed', 12)
# run RandomMassBalance model centered around t*, once without
# temperature bias and once with positive and negative temperature bias
# of 0.5 °C each (per default).
for suffix, temp_bias in zip(suffixes, temp_biases):
workflow.execute_entity_task(vascaling.run_random_climate,
gdirs,
temperature_bias=temp_bias,
output_filesuffix=suffix,
**kwargs)
else:
# run ConstantMassBalance model centered around t*, once without
# temperature bias and once with positive and negative temperature bias
# of 0.5 °C each (per default).
for suffix, temp_bias in zip(suffixes, temp_biases):
workflow.execute_entity_task(vascaling.run_constant_climate,
gdirs,
temperature_bias=temp_bias,
output_filesuffix=suffix,
**kwargs)
# Process output dataset(s)
# -------------------------
# create empty container
ds = list()
# iterate over all temperature biases/suffixes
for suffix in suffixes:
# compile the output for each run
ds_ = utils.compile_run_output(np.atleast_1d(gdirs),
input_filesuffix=suffix,
path=False)
# add to container
ds.append(ds_)
# concat the single output datasets into one,
# with temperature bias as coordinate
ds = xr.concat(ds, pd.Index(temp_biases, name='temp_bias'))
# add model type as coordinate
ds.coords['model'] = 'vas'
# add mb model type as coordinate
ds.coords['mb_model'] = 'random' if use_random_mb else 'constant'
# compute mean and sum over all glaciers
ds_mean = ds.mean(dim='rgi_id')
ds_mean.coords['rgi_id'] = 'mean'
ds_sum = ds.sum(dim='rgi_id')
ds_sum.coords['rgi_id'] = 'sum'
# add to dataset
ds = xr.concat([ds, ds_mean, ds_sum], dim='rgi_id')
# normalize glacier geometries (length/area/volume) with start value
ds_normal = normalize_ds_with_start(ds)
# add coordinate to distinguish between normalized and absolute values
ds.coords['normalized'] = int(False)
ds_normal.coords['normalized'] = int(True)
# combine datasets
ds = xr.concat([ds, ds_normal], 'normalized')
# store datasets
if path:
if path is True:
mb = 'random' if use_random_mb else 'constant'
path = os.path.join(cfg.PATHS['working_dir'],
'run_output_{}_vas.nc'.format(mb))
ds.to_netcdf(path)
return ds
def sensitivity_run_vas_old(rgi_ids, use_random_mb=False, use_mean=True,
path=True, temp_bias=0, tstar=None,
sensitivity_params=[[(4.5507, 0.191), 1]], suffixes=[''],
**kwargs):
""" The routine runs all steps for the equilibrium experiments using the
volume/area scaling model (cf. `equilibrium_run_vas`) but for only one
given temperature bias. However, it is possible to supply a list of
sensitivity parameters (the scaling constants, and time scale factor) to
alter the model behavior.
- OGGM preprocessing, including initialization, GIS tasks, climate tasks and
massbalance tasks.
- Run model for all glaciers with constant (or random) massbalance model
over 3000 years (default value).
- Process the model output dataset(s), i.e. normalization, average/sum, ...
The final dataset containing all results is returned. Given a path is is
also stored to file.
Parameters
----------
rgi_ids: array-like
List of RGI IDs for which the equilibrium experiments are performed.
use_random_mb: bool, optional, default=True
Choose between random massbalance model and constant massbalance model.
use_mean: bool, optional, default=True
Choose between the mean or summation over all glaciers
path: bool or str, optional, default=True
If a path is given (or True), the resulting dataset is stored to file.
temp_bias: float, optional, default=0
Temperature bias (degC) for the mass balance model.
sensitivity_params: multi-dimensional array-like, optional,
default=[[(4.5507, 0.191), 1]]
list containing the parameters which are to be varied in the following
order: float tuple with length and area scaling constant, float as time
scale factor
suffixes: array-like, optional, default=['']
Descriptive suffixes corresponding to the given sensitivity params
tstar: float, optional, default=None
'Equilibrium year' used for the mass balance calibration.
kwargs:
Additional key word arguments for the `run_random_climate` or
`run_constant_climate` routines of the vascaling module.
Returns
-------
Dataset containing yearly values of all glacier geometries.
"""
# assert correct output file suffixes for temp biases
if len(sensitivity_params) != len(suffixes):
raise RuntimeError("Each given parameter set must have its "
"corresponding suffix")
# OGGM preprocessing
# ------------------
# compute RGI region and version from RGI IDs
# assuming all they are all the same
rgi_region = (rgi_ids[0].split('-')[-1]).split('.')[0]
rgi_version = (rgi_ids[0].split('-')[0])[-2:]
# load default parameter file
cfg.initialize()
# create working directory
wdir = '/Users/oberrauch/work/master/working_directories/'
wdir += 'sensitivity_vas_wdir'
if not os.path.exists(wdir):
os.makedirs(wdir)
# set path to working directory
cfg.PATHS['working_dir'] = wdir
# set RGI verion and region
cfg.PARAMS['rgi_version'] = rgi_version
# define how many grid points to use around the glacier,
# if you expect the glacier to grow large use a larger border
cfg.PARAMS['border'] = 80
# we use HistAlp climate data
cfg.PARAMS['baseline_climate'] = 'HISTALP'
# set the mb hyper parameters accordingly
cfg.PARAMS['prcp_scaling_factor'] = 1.75
cfg.PARAMS['temp_melt'] = -1.75
# the bias is defined to be zero during the calibration process,
# which is why we don't use it here to reproduce the results
cfg.PARAMS['use_bias_for_run'] = False
# read RGI entry for the glaciers as DataFrame
# containing the outline area as shapefile
rgidf = utils.get_rgi_glacier_entities(rgi_ids)
# get and set path to intersect shapefile
intersects_db = utils.get_rgi_intersects_region_file(region=rgi_region)
cfg.set_intersects_db(intersects_db)
# initialize the GlacierDirectory
gdirs = workflow.init_glacier_regions(rgidf)
# define the local grid and glacier mask
workflow.execute_entity_task(gis.glacier_masks, gdirs)
# process the given climate file
workflow.execute_entity_task(climate.process_histalp_data, gdirs)
# compute local t* and the corresponding mu*
workflow.execute_entity_task(vascaling.local_t_star, gdirs,
tstar=tstar, bias=0)
# Run model with constant/random mass balance model
# -------------------------------------------------
# use t* as center year, even if specified differently
kwargs['y0'] = tstar
# run for 3000 years if not specified otherwise
kwargs.setdefault('nyears', 3000)
if use_random_mb:
# set random seed to get reproducible results
kwargs.setdefault('seed', 12)
# run RandomMassBalance model centered around t* for each given
# parameter set
for suffix, params in zip(suffixes, sensitivity_params):
cfg.PARAMS['vas_c_length_m'] = params[0]
cfg.PARAMS['vas_c_area_m2'] = params[1]
kwargs['time_scale_factor'] = params[2]
workflow.execute_entity_task(vascaling.run_random_climate, gdirs,
temperature_bias=temp_bias,
output_filesuffix=suffix, **kwargs)
else:
# run ConstantMassBalance model centered around t* for each given
# parameter set
for suffix, params in zip(suffixes, sensitivity_params):
cfg.PARAMS['vas_c_length_m'] = params[0][0]
cfg.PARAMS['vas_c_area_m2'] = params[0][1]
kwargs['time_scale_factor'] = params[1]
workflow.execute_entity_task(vascaling.run_constant_climate, gdirs,
temperature_bias=temp_bias,
output_filesuffix=suffix, **kwargs)
# Process output dataset(s)
# -------------------------
# create empty container
ds = list()
# iterate over all temperature biases/suffixes
for suffix, params in zip(suffixes, sensitivity_params):
# compile the output for each run
ds_ = utils.compile_run_output(np.atleast_1d(gdirs),
filesuffix=suffix, path=False)
# add sensitivity parameters as coordinates
ds_.coords['length_scaling_const'] = params[0][0]
ds_.coords['area_scaling_const'] = params[0][1]
ds_.coords['time_scale_factor'] = params[1]
# add to container
ds.append(ds_)
# concat the single output datasets into one, using 'sensitivity_params'
# as name fot the new concatenate dimension
ds = xr.combine_nested(ds, )
# add model type as coordinate
ds.coords['model'] = 'vas'
# add mb model type as coordinate
ds.coords['mb_model'] = 'random' if use_random_mb else 'constant'
# normalize glacier geometries (length/area/volume) with start value
if use_mean:
# compute average over all glaciers
ds_normal = normalize_ds_with_start(ds).mean(dim='rgi_id')
ds = ds.mean(dim='rgi_id')
else:
# compute sum over all glaciers
ds_normal = normalize_ds_with_start(ds.sum(dim='rgi_id'))
ds = ds.sum(dim='rgi_id')
# add coordinate to distinguish between normalized and absolute values
ds.coords['normalized'] = False
ds_normal.coords['normalized'] = True
# combine datasets
ds = xr.concat([ds, ds_normal], 'normalized')
# store datasets
if path:
if path is True:
path = list()
mb = 'random' if use_random_mb else 'constant'
path.append(os.path.join(cfg.PATHS['working_dir'],
'run_output_{}_vas.nc'.format(mb)))
# path.append(os.path.join(cfg.PATHS['working_dir'],
# 'run_output_{}_vas.nc'.format(mb)))
# path.append(os.path.join(cfg.PATHS['working_dir'],
# 'normalized_output_{}_vas.nc'.format(mb)))
ds.to_netcdf(path)
# ds_normal.to_netcdf(path[1])
# return ds, ds_normal
return ds
def run_benchmark(rgi_version=None,
rgi_reg=None,
border=None,
output_folder='',
working_dir='',
is_test=False,
test_rgidf=None,
test_intersects_file=None,
test_topofile=None):
"""Does the actual job.
Parameters
----------
rgi_version : str
the RGI version to use (defaults to cfg.PARAMS)
rgi_reg : str
the RGI region to process
border : int
the number of pixels at the maps border
output_folder : str
path to the output folder (where to put the preprocessed tar files)
working_dir : str
path to the OGGM working directory
is_test : bool
to test on a couple of glaciers only!
test_rgidf : shapefile
for testing purposes only
test_intersects_file : shapefile
for testing purposes only
test_topofile : str
for testing purposes only
"""
# TODO: temporarily silence Fiona deprecation warnings
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
# Module logger
log = logging.getLogger(__name__)
# Initialize OGGM and set up the run parameters
cfg.initialize(logging_level='WORKFLOW')
# Local paths
utils.mkdir(working_dir)
cfg.PATHS['working_dir'] = working_dir
# Use multiprocessing?
cfg.PARAMS['use_multiprocessing'] = True
# How many grid points around the glacier?
# Make it large if you expect your glaciers to grow large
cfg.PARAMS['border'] = border
# Set to True for operational runs
cfg.PARAMS['continue_on_error'] = True
# For statistics
odf = pd.DataFrame()
if rgi_version is None:
rgi_version = cfg.PARAMS['rgi_version']
base_dir = os.path.join(output_folder)
# Add a package version file
utils.mkdir(base_dir)
opath = os.path.join(base_dir, 'package_versions.txt')
with open(opath, 'w') as vfile:
vfile.write(utils.show_versions(logger=log))
# Read RGI
start = time.time()
if test_rgidf is None:
# Get the RGI file
rgidf = gpd.read_file(
utils.get_rgi_region_file(rgi_reg, version=rgi_version))
# We use intersects
rgif = utils.get_rgi_intersects_region_file(rgi_reg,
version=rgi_version)
cfg.set_intersects_db(rgif)
else:
rgidf = test_rgidf
cfg.set_intersects_db(test_intersects_file)
if is_test:
# Just for fun
rgidf = rgidf.sample(2)
_add_time_to_df(odf, 'Read RGI', time.time() - start)
# Sort for more efficient parallel computing
rgidf = rgidf.sort_values('Area', ascending=False)
log.workflow('Starting prepro run for RGI reg: {} '
'and border: {}'.format(rgi_reg, border))
log.workflow('Number of glaciers: {}'.format(len(rgidf)))
# Input
if test_topofile:
cfg.PATHS['dem_file'] = test_topofile
# Initialize working directories
start = time.time()
gdirs = workflow.init_glacier_directories(rgidf, reset=True, force=True)
_add_time_to_df(odf, 'init_glacier_directories', time.time() - start)
# Tasks
task_list = [
tasks.define_glacier_region,
tasks.process_cru_data,
tasks.glacier_masks,
tasks.compute_centerlines,
tasks.initialize_flowlines,
tasks.compute_downstream_line,
tasks.compute_downstream_bedshape,
tasks.catchment_area,
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
tasks.local_t_star,
tasks.mu_star_calibration,
tasks.prepare_for_inversion,
tasks.mass_conservation_inversion,
tasks.filter_inversion_output,
tasks.init_present_time_glacier,
]
for task in task_list:
start = time.time()
workflow.execute_entity_task(task, gdirs)
_add_time_to_df(odf, task.__name__, time.time() - start)
# Runs
start = time.time()
workflow.execute_entity_task(tasks.run_random_climate,
gdirs,
nyears=250,
bias=0,
seed=0,
output_filesuffix='_tstar')
_add_time_to_df(odf, 'run_random_climate_tstar_250', time.time() - start)
start = time.time()
workflow.execute_entity_task(tasks.run_random_climate,
gdirs,
nyears=250,
y0=1995,
seed=0,
output_filesuffix='_commit')
_add_time_to_df(odf, 'run_random_climate_commit_250', time.time() - start)
# Compile results
start = time.time()
utils.compile_glacier_statistics(gdirs)
_add_time_to_df(odf, 'compile_glacier_statistics', time.time() - start)
start = time.time()
utils.compile_climate_statistics(gdirs,
add_climate_period=[1920, 1960, 2000])
_add_time_to_df(odf, 'compile_climate_statistics', time.time() - start)
start = time.time()
utils.compile_run_output(gdirs, input_filesuffix='_tstar')
_add_time_to_df(odf, 'compile_run_output_tstar', time.time() - start)
start = time.time()
utils.compile_run_output(gdirs, input_filesuffix='_commit')
_add_time_to_df(odf, 'compile_run_output_commit', time.time() - start)
# Log
opath = os.path.join(base_dir, 'benchmarks_b{:03d}.csv'.format(border))
odf.index.name = 'Task'
odf.to_csv(opath)
log.workflow('OGGM benchmarks is done!')
def run_prepro_levels(rgi_version=None, rgi_reg=None, border=None,
output_folder='', working_dir='', is_test=False,
demo=False, test_rgidf=None, test_intersects_file=None,
test_topofile=None, test_crudir=None):
"""Does the actual job.
Parameters
----------
rgi_version : str
the RGI version to use (defaults to cfg.PARAMS)
rgi_reg : str
the RGI region to process
border : int
the number of pixels at the maps border
output_folder : str
path to the output folder (where to put the preprocessed tar files)
working_dir : str
path to the OGGM working directory
is_test : bool
to test on a couple of glaciers only!
demo : bool
to run the prepro for the list of demo glaciers
test_rgidf : shapefile
for testing purposes only
test_intersects_file : shapefile
for testing purposes only
test_topofile : str
for testing purposes only
test_crudir : str
for testing purposes only
"""
# TODO: temporarily silence Fiona deprecation warnings
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
# Module logger
log = logging.getLogger(__name__)
# Time
start = time.time()
# Initialize OGGM and set up the run parameters
cfg.initialize(logging_level='WORKFLOW')
# Local paths
utils.mkdir(working_dir)
cfg.PATHS['working_dir'] = working_dir
# Use multiprocessing?
cfg.PARAMS['use_multiprocessing'] = True
# How many grid points around the glacier?
# Make it large if you expect your glaciers to grow large
cfg.PARAMS['border'] = border
# Set to True for operational runs
cfg.PARAMS['continue_on_error'] = True
# For statistics
climate_periods = [1920, 1960, 2000]
if rgi_version is None:
rgi_version = cfg.PARAMS['rgi_version']
rgi_dir_name = 'RGI{}'.format(rgi_version)
border_dir_name = 'b_{:03d}'.format(border)
base_dir = os.path.join(output_folder, rgi_dir_name, border_dir_name)
# Add a package version file
utils.mkdir(base_dir)
opath = os.path.join(base_dir, 'package_versions.txt')
with open(opath, 'w') as vfile:
vfile.write(utils.show_versions(logger=log))
if demo:
rgidf = utils.get_rgi_glacier_entities(cfg.DEMO_GLACIERS.index)
elif test_rgidf is None:
# Get the RGI file
rgidf = gpd.read_file(utils.get_rgi_region_file(rgi_reg,
version=rgi_version))
# We use intersects
rgif = utils.get_rgi_intersects_region_file(rgi_reg,
version=rgi_version)
cfg.set_intersects_db(rgif)
else:
rgidf = test_rgidf
cfg.set_intersects_db(test_intersects_file)
if is_test:
# Just for fun
rgidf = rgidf.sample(4)
# Sort for more efficient parallel computing
rgidf = rgidf.sort_values('Area', ascending=False)
log.workflow('Starting prepro run for RGI reg: {} '
'and border: {}'.format(rgi_reg, border))
log.workflow('Number of glaciers: {}'.format(len(rgidf)))
# Input
if test_topofile:
cfg.PATHS['dem_file'] = test_topofile
# L1 - initialize working directories
gdirs = workflow.init_glacier_regions(rgidf, reset=True, force=True)
# Glacier stats
sum_dir = os.path.join(base_dir, 'L1', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# L1 OK - compress all in output directory
l_base_dir = os.path.join(base_dir, 'L1')
workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False,
base_dir=l_base_dir)
utils.base_dir_to_tar(l_base_dir)
# L2 - Tasks
# Pre-download other files just in case
if test_crudir is None:
_ = utils.get_cru_file(var='tmp')
_ = utils.get_cru_file(var='pre')
else:
cfg.PATHS['cru_dir'] = test_crudir
workflow.execute_entity_task(tasks.process_cru_data, gdirs)
# Glacier stats
sum_dir = os.path.join(base_dir, 'L2', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# L2 OK - compress all in output directory
l_base_dir = os.path.join(base_dir, 'L2')
workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False,
base_dir=l_base_dir)
utils.base_dir_to_tar(l_base_dir)
# L3 - Tasks
task_list = [
tasks.glacier_masks,
tasks.compute_centerlines,
tasks.initialize_flowlines,
tasks.compute_downstream_line,
tasks.compute_downstream_bedshape,
tasks.catchment_area,
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
tasks.local_t_star,
tasks.mu_star_calibration,
tasks.prepare_for_inversion,
tasks.mass_conservation_inversion,
tasks.filter_inversion_output,
]
for task in task_list:
workflow.execute_entity_task(task, gdirs)
# Glacier stats
sum_dir = os.path.join(base_dir, 'L3', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
opath = os.path.join(sum_dir, 'climate_statistics_{}.csv'.format(rgi_reg))
utils.compile_climate_statistics(gdirs, add_climate_period=climate_periods,
path=opath)
# L3 OK - compress all in output directory
l_base_dir = os.path.join(base_dir, 'L3')
workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False,
base_dir=l_base_dir)
utils.base_dir_to_tar(l_base_dir)
# L4 - Tasks
workflow.execute_entity_task(tasks.init_present_time_glacier, gdirs)
# Glacier stats
sum_dir = os.path.join(base_dir, 'L4', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# Copy mini data to new dir
base_dir = os.path.join(base_dir, 'L4')
mini_gdirs = workflow.execute_entity_task(tasks.copy_to_basedir, gdirs,
base_dir=base_dir)
# L4 OK - compress all in output directory
workflow.execute_entity_task(utils.gdir_to_tar, mini_gdirs, delete=True)
utils.base_dir_to_tar(base_dir)
# Log
m, s = divmod(time.time() - start, 60)
h, m = divmod(m, 60)
log.workflow('OGGM prepro_levels is done! Time needed: '
'{:02d}:{:02d}:{:02d}'.format(int(h), int(m), int(s)))
def setup_cache(self):
setattr(full_workflow.setup_cache, "timeout", 360)
utils.mkdir(self.testdir, reset=True)
self.cfg_init()
# Pre-download other files which will be needed later
utils.get_cru_cl_file()
utils.get_cru_file(var='tmp')
utils.get_cru_file(var='pre')
# Get the RGI glaciers for the run.
rgi_list = ['RGI60-01.10299', 'RGI60-11.00897', 'RGI60-18.02342']
rgidf = utils.get_rgi_glacier_entities(rgi_list)
# We use intersects
db = utils.get_rgi_intersects_region_file(version='61',
rgi_ids=rgi_list)
cfg.set_intersects_db(db)
# Sort for more efficient parallel computing
rgidf = rgidf.sort_values('Area', ascending=False)
# Go - initialize working directories
gdirs = workflow.init_glacier_regions(rgidf)
# Preprocessing tasks
task_list = [
tasks.glacier_masks,
tasks.compute_centerlines,
tasks.initialize_flowlines,
tasks.compute_downstream_line,
tasks.compute_downstream_bedshape,
tasks.catchment_area,
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
]
for task in task_list:
execute_entity_task(task, gdirs)
# Climate tasks -- only data IO and tstar interpolation!
execute_entity_task(tasks.process_cru_data, gdirs)
execute_entity_task(tasks.local_mustar, gdirs)
execute_entity_task(tasks.apparent_mb, gdirs)
# Inversion tasks
execute_entity_task(tasks.prepare_for_inversion, gdirs)
# We use the default parameters for this run
execute_entity_task(tasks.mass_conservation_inversion, gdirs)
execute_entity_task(tasks.filter_inversion_output, gdirs)
# Final preparation for the run
execute_entity_task(tasks.init_present_time_glacier, gdirs)
# Random climate representative for the tstar climate, without bias
# In an ideal world this would imply that the glaciers remain stable,
# but it doesn't have to be so
execute_entity_task(tasks.run_constant_climate,
gdirs,
bias=0,
nyears=100,
output_filesuffix='_tstar')
execute_entity_task(tasks.run_constant_climate,
gdirs,
y0=1990,
nyears=100,
output_filesuffix='_pd')
# Compile output
utils.glacier_characteristics(gdirs)
utils.compile_run_output(gdirs, filesuffix='_tstar')
utils.compile_run_output(gdirs, filesuffix='_pd')
utils.compile_climate_input(gdirs)
return gdirs
cfg.PATHS['working_dir'] = WORKING_DIR
# Use multiprocessing?
cfg.PARAMS['use_multiprocessing'] = True
# Here we override some of the default parameters
# How many grid points around the glacier?
# Make it large if you expect your glaciers to grow large
cfg.PARAMS['border'] = 100
# Set to True for operational runs
cfg.PARAMS['continue_on_error'] = False
# We use intersects
# Here we use the global file but there are regional files too (faster)
cfg.set_intersects_db(utils.get_rgi_intersects_region_file('00', version='5'))
# Pre-download other files which will be needed later
utils.get_cru_cl_file()
utils.get_cru_file(var='tmp')
utils.get_cru_file(var='pre')
# Download the RGI file for the run
# We us a set of four glaciers here but this could be an entire RGI region,
# or any glacier list you'd like to model
dl = 'https://cluster.klima.uni-bremen.de/~fmaussion/misc/RGI_example_glaciers.zip'
with zipfile.ZipFile(utils.file_downloader(dl)) as zf:
zf.extractall(WORKING_DIR)
rgidf = salem.read_shapefile(
path.join(WORKING_DIR, 'RGI_example_glaciers', 'RGI_example_glaciers.shp'))
cfg.PARAMS['use_bias_for_run'] = True
# read RGI entry for the glaciers as DataFrame
# containing the outline area as shapefile
rgi_reg = os.environ.get('OGGM_RGI_REG', '')
if rgi_reg not in ['{:02d}'.format(r) for r in range(1, 20)]:
raise RuntimeError('Need an RGI Region')
rgi_ids = gpd.read_file(
utils.get_rgi_region_file(rgi_reg, version=rgi_version))
# For greenland we omit connectivity level 2
if rgi_reg == '05':
rgi_ids = rgi_ids.loc[rgi_ids['Connect'] != 2]
# get and set path to intersect shapefile
intersects_db = utils.get_rgi_intersects_region_file(region=rgi_reg)
cfg.set_intersects_db(intersects_db)
# operational run, all glaciers should run
cfg.PARAMS['use_multiprocessing'] = True
cfg.PARAMS['continue_on_error'] = True
# Module logger
log = logging.getLogger(__name__)
log.workflow('Starting run for RGI reg {}'.format(rgi_reg))
# Go - get the pre-processed glacier directories
base_url = "https://cluster.klima.uni-bremen.de/~oggm/gdirs/oggm_v1.4/" \
"L3-L5_files/CRU/elev_bands/qc3/pcp2.5/match_geod"
gdirs = workflow.init_glacier_directories(rgi_ids, from_prepro_level=3,
prepro_base_url=base_url,
def configure(workdir, glclist, baselineclimate='HISTALP', resetwd=False):
global MERGEDICT
global GLCDICT
global ADDITIONAL_REFERENCE_GLACIERS
# Initialize OGGM
cfg.initialize()
cfg.PATHS['working_dir'] = workdir
# Local working directory (where OGGM will write its output)
utils.mkdir(workdir, reset=resetwd)
# Use multiprocessing?
cfg.PARAMS['use_multiprocessing'] = True
# Set to True for operational runs
cfg.PARAMS['continue_on_error'] = False
# We use intersects
cfg.PARAMS['use_intersects'] = True
rgif = utils.get_rgi_intersects_region_file('11', version='60')
cfg.set_intersects_db(rgif)
cfg.PARAMS['use_rgi_area'] = True
# set negative flux filtering to false. should be standard soon
cfg.PARAMS['filter_for_neg_flux'] = False
cfg.PARAMS['correct_for_neg_flux'] = True
# here in relic we want to run the mb calibration every time
cfg.PARAMS['run_mb_calibration'] = True
# glacier length
cfg.PARAMS['min_ice_thick_for_length'] = 1.0
cfg.PARAMS['glacier_length_method'] = 'consecutive'
# check if we want to merge a glacier
mglclist = []
for glc in glclist:
mglc = merge_pair_dict(glc)
if mglc is not None:
mglclist += mglc[0]
# How many grid points around the glacier?
# Make it large if you expect your glaciers to grow large
cfg.PARAMS['border'] = 160
gdirs = workflow.init_glacier_regions(glclist + mglclist,
from_prepro_level=3)
# and we want to use all glaciers for the MB calibration
refids = get_ref_mb_glaciers_candidates()
# right now we only do Alpine glaciers
refids = [rid for rid in refids if '-11.' in rid]
# but do leave out the actual glaciers
refids = [rid for rid in refids if rid not in glclist + mglclist]
# I SAID ALPS, NOT PYRENEES
refids.remove('RGI60-11.03232')
refids.remove('RGI60-11.03209')
refids.remove('RGI60-11.03241')
# initialize the reference glaciers with a small border
ref_gdirs = workflow.init_glacier_regions(rgidf=refids,
from_prepro_level=3,
prepro_border=10)
# save these ids for later
ADDITIONAL_REFERENCE_GLACIERS = refids
# climate
if baselineclimate == 'CRU':
cfg.PARAMS['prcp_scaling_factor'] = 2.5
cfg.PARAMS['temp_melt'] = -1.0
if baselineclimate == 'HISTALP':
cfg.PARAMS['baseline_climate'] = baselineclimate
# and set standard histalp values
cfg.PARAMS['prcp_scaling_factor'] = 1.75
cfg.PARAMS['temp_melt'] = -1.75
# run histalp climate on all glaciers!
execute_entity_task(tasks.process_histalp_data,
gdirs + ref_gdirs,
y0=1849)
# TODO: if I do use custom climate stuff like histalp_annual_mean:
# ->>>> look back at commits before 1.10.2019
return gdirs
def seek_start_area(rgi_id,
name,
show=False,
path='',
ref=np.NaN,
adjust_term_elev=False,
legend=True,
instant_geometry_change=False):
""" Set up an VAS model from scratch and run/test the start area seeking
tasks. The result is a plot showing the modeled glacier area evolution for
different start values. The plots can be displayed and/or stored to file.
Parameters
----------
rgi_id: string
RGI ID denoting the glacier on which to perform the tasks
name: string
Name og glacier, since it is not always given (or correct) in RGI
show: bool, optional, default=False
Flag deciding whether or not to show the created plots.
path: string, optional, default=''
Path under which the modeled area plot should be stored.
ref: float, optional, default=np.NaN
Historic (1851) reference area with which a reference model run is
performed.
"""
# Initialization and load default parameter file
vascaling.initialize()
# compute RGI region and version from RGI IDs
# assuming they all are all the same
rgi_region = (rgi_id.split('-')[-1]).split('.')[0]
rgi_version = (rgi_id.split('-')[0])[-2:-1]
# specify working directory and output directory
working_dir = os.path.abspath('../working_directories/start_area/')
# output_dir = os.path.abspath('./vas_run_output')
output_dir = os.path.abspath('../data/vas_run_output')
# create working directory
utils.mkdir(working_dir, reset=False)
utils.mkdir(output_dir)
# set path to working directory
cfg.PATHS['working_dir'] = working_dir
# set RGI version and region
cfg.PARAMS['rgi_version'] = rgi_version
# define how many grid points to use around the glacier,
# if you expect the glacier to grow large use a larger border
cfg.PARAMS['border'] = 20
# we use HistAlp climate data
cfg.PARAMS['baseline_climate'] = 'HISTALP'
# set the mb hyper parameters accordingly
cfg.PARAMS['prcp_scaling_factor'] = 1.75
cfg.PARAMS['temp_melt'] = -1.75
cfg.PARAMS['run_mb_calibration'] = False
# the bias is defined to be zero during the calibration process,
# which is why we don't use it here to reproduce the results
cfg.PARAMS['use_bias_for_run'] = True
# get/downlaod the rgi entity including the outline shapefile
rgi_df = utils.get_rgi_glacier_entities([rgi_id])
# set name, if not delivered with RGI
if rgi_df.loc[int(rgi_id[-5:]) - 1, 'Name'] is None:
rgi_df.loc[int(rgi_id[-5:]) - 1, 'Name'] = name
# get and set path to intersect shapefile
intersects_db = utils.get_rgi_intersects_region_file(region=rgi_region)
cfg.set_intersects_db(intersects_db)
# initialize the GlacierDirectory
gdir = workflow.init_glacier_directories(rgi_df)[0]
# # DEM and GIS tasks
# # get the path to the DEM file (will download if necessary)
# dem = utils.get_topo_file(gdir.cenlon, gdir.cenlat)
# print('DEM source: {}, path to DEM file: {}'.format(dem[1], dem[0][0]))
# # set path in config file
# cfg.PATHS['dem_file'] = dem[0][0]
# cfg.PARAMS['border'] = 10
# cfg.PARAMS['use_intersects'] = False
# run GIS tasks
gis.define_glacier_region(gdir)
gis.glacier_masks(gdir)
# process climate data
climate.process_climate_data(gdir)
# compute local t* and the corresponding mu*
vascaling.local_t_star(gdir)
# create mass balance model
mb_mod = vascaling.VAScalingMassBalance(gdir)
# look at specific mass balance over climate data period
min_hgt, max_hgt = vascaling.get_min_max_elevation(gdir)
y0 = 1851
y1 = 2014
# run scalar minimization
minimize_res = vascaling.find_start_area(
gdir,
adjust_term_elev=adjust_term_elev,
instant_geometry_change=instant_geometry_change)
# print(minimize_res)
# stop script if minimization was not successful
if minimize_res.status and False:
sys.exit(minimize_res.status)
# instance glacier with today's values
model_ref = vascaling.VAScalingModel(year_0=gdir.rgi_date,
area_m2_0=gdir.rgi_area_m2,
min_hgt=min_hgt,
max_hgt=max_hgt,
mb_model=mb_mod)
# instance guessed starting areas
num = 9
area_guess = np.linspace(1e6,
np.floor(gdir.rgi_area_m2 * 2),
num,
endpoint=True)
# create empty containers
area_list = list()
volume_list = list()
spec_mb_list = list()
# iterate over all starting areas
for area_ in area_guess:
# instance iteration model
model_guess = vascaling.VAScalingModel(year_0=gdir.rgi_date,
area_m2_0=gdir.rgi_area_m2,
min_hgt=min_hgt,
max_hgt=max_hgt,
mb_model=mb_mod)
# set new starting values
model_guess.create_start_glacier(area_,
y0,
adjust_term_elev=adjust_term_elev)
# run model and save years and area
best_guess_ds = model_guess.run_until_and_store(
year_end=model_ref.year,
instant_geometry_change=instant_geometry_change)
# create series and store in container
area_list.append(best_guess_ds.area_m2.to_dataframe()['area_m2'])
volume_list.append(best_guess_ds.volume_m3.to_dataframe()['volume_m3'])
spec_mb_list.append(best_guess_ds.spec_mb.to_dataframe()['spec_mb'])
# create DataFrame
area_df = pd.DataFrame(
area_list, index=['{:.2f}'.format(a / 1e6) for a in area_guess])
area_df.index.name = 'Start Area [km$^2$]'
volume_df = pd.DataFrame(
volume_list, index=['{:.2f}'.format(a / 1e6) for a in area_guess])
volume_df.index.name = 'Start Area [km$^2$]'
# set up model with resulted starting area
model = vascaling.VAScalingModel(year_0=model_ref.year_0,
area_m2_0=model_ref.area_m2_0,
min_hgt=model_ref.min_hgt_0,
max_hgt=model_ref.max_hgt,
mb_model=model_ref.mb_model)
model.create_start_glacier(minimize_res.x,
y0,
adjust_term_elev=adjust_term_elev)
# run model with best guess initial area
best_guess_ds = model.run_until_and_store(
year_end=model_ref.year,
instant_geometry_change=instant_geometry_change)
# run model with historic reference area
if ref:
model.reset()
model.create_start_glacier(ref * 1e6,
y0,
adjust_term_elev=adjust_term_elev)
ref_ds = model.run_until_and_store(
year_end=model_ref.year,
instant_geometry_change=instant_geometry_change)
# create figure and add axes
fig = plt.figure(figsize=[5, 5])
ax = fig.add_axes([0.125, 0.075, 0.85, 0.9])
# plot model output
ax = (area_df / 1e6).T.plot(legend=False, colormap='Spectral', ax=ax)
# plot best guess
ax.plot(
best_guess_ds.time,
best_guess_ds.area_m2 / 1e6,
color='k',
ls='--',
lw=1.2,
label=
f'{best_guess_ds.area_m2.isel(time=0).values/1e6:.2f} km$^2$ (best result)'
)
# plot reference
if ref:
ax.plot(
ref_ds.time,
ref_ds.area_m2 / 1e6,
color='k',
ls='-.',
lw=1.2,
label=
f'{ref_ds.area_m2.isel(time=0).values/1e6:.2f} km$^2$ (1850 ref.)')
# plot 2003 reference line
ax.axhline(
model_ref.area_m2_0 / 1e6,
c='k',
ls=':',
label=f'{model_ref.area_m2_0/1e6:.2f} km$^2$ ({gdir.rgi_date} obs.)')
# add legend
if legend:
handels, labels = ax.get_legend_handles_labels()
labels[:-3] = [r'{} km$^2$'.format(l) for l in labels[:-3]]
leg = ax.legend(handels, labels, loc='upper right', ncol=2)
# leg.set_title('Start area $A_0$', prop={'size': 12})
# replot best guess estimate and reference (in case it lies below another
# guess)
ax.plot(best_guess_ds.time,
best_guess_ds.area_m2 / 1e6,
color='k',
ls='--',
lw=1.2)
if ref:
ax.plot(ref_ds.time, ref_ds.area_m2 / 1e6, color='k', ls='-.', lw=1.2)
# labels, title
ax.set_xlim([best_guess_ds.time.values[0], best_guess_ds.time.values[-1]])
ax.set_xlabel('')
ax.set_ylabel('Glacier area [km$^2$]')
# save figure to file
if path:
fig.savefig(path)
# show plot
if show:
plt.show()
plt.clf()
# plot and store volume evolution
(volume_df / 1e9).T.plot(legend=False, colormap='viridis')
plt.gcf().savefig(path[:-4] + '_volume.pdf')
def run_cmip():
# Initialize OGGM and set up the default run parameters
vascaling.initialize(logging_level='WORKFLOW')
rgi_version = '62'
cfg.PARAMS['border'] = 80
# CLUSTER paths
wdir = os.environ.get('WORKDIR', '')
cfg.PATHS['working_dir'] = wdir
outdir = os.environ.get('OUTDIR', '')
# define the baseline climate CRU or HISTALP
cfg.PARAMS['baseline_climate'] = 'CRU'
# set the mb hyper parameters accordingly
cfg.PARAMS['prcp_scaling_factor'] = 3
cfg.PARAMS['temp_melt'] = 0
cfg.PARAMS['temp_all_solid'] = 4
cfg.PARAMS['run_mb_calibration'] = False
# set minimum ice thickness to include in glacier length computation
# this reduces weird spikes in length records
cfg.PARAMS['min_ice_thick_for_length'] = 0.1
# the bias is defined to be zero during the calibration process,
# which is why we don't use it here to reproduce the results
cfg.PARAMS['use_bias_for_run'] = True
# read RGI entry for the glaciers as DataFrame
# containing the outline area as shapefile
# RGI glaciers
rgi_reg = os.environ.get('OGGM_RGI_REG', '')
if rgi_reg not in ['{:02d}'.format(r) for r in range(1, 20)]:
raise RuntimeError('Need an RGI Region')
rgi_ids = gpd.read_file(
utils.get_rgi_region_file(rgi_reg, version=rgi_version))
# get and set path to intersect shapefile
intersects_db = utils.get_rgi_intersects_region_file(region=rgi_reg)
cfg.set_intersects_db(intersects_db)
# operational run, all glaciers should run
cfg.PARAMS['continue_on_error'] = True
# Module logger
log = logging.getLogger(__name__)
log.workflow('Starting run for RGI reg {}'.format(rgi_reg))
# Go - get the pre-processed glacier directories
# base_url = 'https://cluster.klima.uni-bremen.de/~oggm/gdirs/oggm_v1.4/' \
# 'L3-L5_files/RGIV62_fleb_qc3_CRU_pcp2.5'
prepro_dir = '/home/users/moberrauch/run_output/vas_prepro/'
gdirs = workflow.init_glacier_directories(rgi_ids, from_tar=prepro_dir)
# # run vascaling climate tasks
# workflow.execute_entity_task(vascaling.local_t_star, gdirs)
# # adjust mass balance residual with geodetic observations
# vascaling.match_regional_geodetic_mb(gdirs=gdirs, rgi_reg=rgi_reg)
# # prepare historic "spinup"
# workflow.execute_entity_task(vascaling.run_from_climate_data, gdirs,
# ys=2003, ye=2020,
# output_filesuffix='_historical')
# read gcm list
gcms = pd.read_csv('/home/www/oggm/cmip6/all_gcm_list.csv', index_col=0)
# iterate over all specified gcms
for gcm in sys.argv[1:]:
# iterate over all SSPs (Shared Socioeconomic Pathways)
df1 = gcms.loc[gcms.gcm == gcm]
for ssp in df1.ssp.unique():
df2 = df1.loc[df1.ssp == ssp]
assert len(df2) == 2
# get temperature projections
ft = df2.loc[df2['var'] == 'tas'].iloc[0]
# get precipitation projections
fp = df2.loc[df2['var'] == 'pr'].iloc[0].path
rid = ft.fname.replace('_r1i1p1f1_tas.nc', '')
ft = ft.path
log.workflow('Starting run for {}'.format(rid))
workflow.execute_entity_task(
gcm_climate.process_cmip_data,
gdirs,
filesuffix='_' + rid,
# recognize the climate file for later
fpath_temp=ft,
# temperature projections
fpath_precip=fp, # precip projections
year_range=('1981', '2020'))
workflow.execute_entity_task(vascaling.run_from_climate_data,
gdirs,
climate_filename='gcm_data',
climate_input_filesuffix='_' + rid,
init_model_filesuffix='_historical',
output_filesuffix=rid,
return_value=False)
gcm_dir = os.path.join(outdir, 'RGI' + rgi_reg, gcm)
utils.mkdir(gcm_dir)
utils.compile_run_output(gdirs,
input_filesuffix=rid,
path=os.path.join(gcm_dir, rid + '.nc'))
log.workflow('OGGM Done')
cfg.PATHS['working_dir'] = WORKING_DIR
# Use multiprocessing
cfg.PARAMS['use_multiprocessing'] = True
# We make the border 80 so we can use the Columbia itmix DEM
cfg.PARAMS['border'] = 20
# Set to True for operational runs
cfg.PARAMS['continue_on_error'] = True
#For tidewater we set this to False
cfg.PARAMS['correct_for_neg_flux'] = False
cfg.PARAMS['filter_for_neg_flux'] = False
# We will needs this set to true for the calving loop
cfg.PARAMS['allow_negative_mustar'] = True
# We use intersects
path = utils.get_rgi_intersects_region_file(rgi_region, version=rgi_version)
cfg.set_intersects_db(path)
# RGI file, 4 Marine-terminating glaciers were merged with their respective
# branches, this is why we use our own version of RGI v6
rgidf = gpd.read_file(RGI_FILE)
# Pre-download other files which will be needed later
_ = utils.get_cru_file(var='tmp')
p = utils.get_cru_file(var='pre')
print('CRU file: ' + p)
# Some controllers maybe this is not necessary
RUN_GIS_mask = True
RUN_GIS_PREPRO = True # run GIS pre-processing tasks (before climate)
RUN_CLIMATE_PREPRO = True # run climate pre-processing tasks
def sensitivity_run_vas(rgi_ids,
use_random_mb=False,
use_mean=False,
path=True,
temp_bias=0,
tstar=None,
use_default_tstar=True,
use_bias_for_run=True,
scaling_params=[(4.5507, 0.191, 2.2, 1.375)],
time_scale_factors=[1],
suffixes=['_default'],
**kwargs):
""" The routine runs all steps for the equilibrium experiments using the
volume/area scaling model (cf. `equilibrium_run_vas`) but for only one
given temperature bias. However, it is possible to supply a list of
sensitivity parameters (the scaling constants, and time scale factor) to
alter the model behavior.
- OGGM preprocessing, including initialization, GIS tasks, climate tasks and
massbalance tasks.
- Run model for all glaciers with constant (or random) massbalance model
over 3000 years (default value).
- Process the model output dataset(s), i.e. normalization, average/sum, ...
The final dataset containing all results is returned. Given a path is is
also stored to file.
Parameters
----------
rgi_ids: array-like
List of RGI IDs for which the equilibrium experiments are performed.
use_random_mb: bool, optional, default=True
Choose between random massbalance model and constant massbalance model.
use_mean: bool, optional, default=False
Choose between the mean or summation over all glaciers
path: bool or str, optional, default=True
If a path is given (or True), the resulting dataset is stored to file.
temp_bias: float, optional, default=0
Temperature bias (degC) for the mass balance model.
sensitivity_params: multi-dimensional array-like, optional,
default=[(4.5507, 0.191, 2.2, 1.375)]
List containing the scaling constants and scaling exponents for length
and area scaling as tuples, e.g., (c_l, c_a, q, gamma)
suffixes: array-like, optional, default=['_default']
Descriptive suffixes corresponding to the given sensitivity params
tstar: float, optional, default=None
'Equilibrium year' used for the mass balance calibration.
use_default_tstar: bool, optional, default=True
Flag deciding whether or not to compute mustar from given from reference
table. Overridden by a given tstar.
use_bias_for_run: bool, optional, default=True
Flag deciding whether or not to use the mass balance residual.
kwargs:
Additional key word arguments for the `run_random_climate` or
`run_constant_climate` routines of the vascaling module.
Returns
-------
Dataset containing yearly values of all glacier geometries.
"""
# assert correct output file suffixes for temp biases
if len(scaling_params) * len(time_scale_factors) != len(suffixes):
raise RuntimeError("Each given combination of scaling parameters and "
"time scale factor must have its corresponding"
"suffix.")
# OGGM preprocessing
# ------------------
# compute RGI region and version from RGI IDs
# assuming all they are all the same
rgi_region = (rgi_ids[0].split('-')[-1]).split('.')[0]
rgi_version = (rgi_ids[0].split('-')[0])[-2:-1]
# load default parameter file
vascaling.initialize()
# get environmental variables for working and output directories
WORKING_DIR = os.environ["WORKDIR"]
OUTPUT_DIR = os.environ["OUTDIR"]
# create working directory
utils.mkdir(WORKING_DIR)
utils.mkdir(OUTPUT_DIR)
# set path to working directory
cfg.PATHS['working_dir'] = WORKING_DIR
# set RGI version and region
cfg.PARAMS['rgi_version'] = rgi_version
# define how many grid points to use around the glacier,
# if you expect the glacier to grow large use a larger border
cfg.PARAMS['border'] = 120
# we use HistAlp climate data
cfg.PARAMS['baseline_climate'] = 'HISTALP'
# set the mb hyper parameters accordingly
cfg.PARAMS['prcp_scaling_factor'] = 2.5
cfg.PARAMS['temp_melt'] = -0.5
# the bias is defined to be zero during the calibration process,
# which is why we don't use it here to reproduce the results
cfg.PARAMS['use_bias_for_run'] = use_bias_for_run
# set minimum ice thickness to include in glacier length computation
# this reduces weird spikes in length records
cfg.PARAMS['min_ice_thick_for_length'] = 0.1
# read RGI entry for the glaciers as DataFrame
# containing the outline area as shapefile
rgidf = utils.get_rgi_glacier_entities(rgi_ids)
# get and set path to intersect shapefile
intersects_db = utils.get_rgi_intersects_region_file(region=rgi_region)
cfg.set_intersects_db(intersects_db)
# sort by area for more efficient parallel computing
rgidf = rgidf.sort_values('Area', ascending=False)
cfg.PARAMS['use_multiprocessing'] = True
# operational run, all glaciers should run
cfg.PARAMS['continue_on_error'] = True
# initialize the GlacierDirectory
gdirs = workflow.init_glacier_directories(rgidf, reset=False, force=True)
# define the local grid and glacier mask
workflow.execute_entity_task(gis.define_glacier_region, gdirs)
workflow.execute_entity_task(gis.glacier_masks, gdirs)
# process the given climate file
workflow.execute_entity_task(climate.process_climate_data, gdirs)
# compute local t* and the corresponding mu*
if tstar or use_default_tstar:
# compute mustar from given tstar or reference table
workflow.execute_entity_task(vascaling.local_t_star,
gdirs,
tstar=tstar,
bias=0)
else:
# compute mustar from the reference table for the flowline model
# RGI v6 and HISTALP baseline climate
ref_df = pd.read_csv(
utils.get_demo_file('oggm_ref_tstars_rgi6_histalp.csv'))
workflow.execute_entity_task(vascaling.local_t_star,
gdirs,
ref_df=ref_df)
# Run model with constant/random mass balance model
# -------------------------------------------------
# use t* as center year, even if specified differently
kwargs['y0'] = tstar
# run for 3000 years if not specified otherwise
kwargs.setdefault('nyears', 1000)
# limit parameters to 3 decimal points
scaling_params = recursive_round(scaling_params, 3)
time_scale_factors = recursive_round(time_scale_factors, 3)
# assure that scaling params are handled as tuples (pairs)
scaling_params_list = np.zeros(len(scaling_params), dtype=object)
scaling_params_list[:] = scaling_params
# combine scaling constants, scaling exponents and time scale factor
# into one iterable array
sensitivity_params = np.array(
np.meshgrid(scaling_params_list, time_scale_factors)).T
sensitivity_params = (sensitivity_params.reshape(
-1, sensitivity_params.shape[-1]))
if use_random_mb:
# set random seed to get reproducible results
kwargs.setdefault('seed', 12)
# run RandomMassBalance model centered around t* for each given
# parameter set
for suffix, params in zip(suffixes, sensitivity_params):
cfg.PARAMS['vas_c_length_m'] = params[0][0]
cfg.PARAMS['vas_c_area_m2'] = params[0][1]
cfg.PARAMS['vas_q_length'] = params[0][2]
cfg.PARAMS['vas_gamma_area'] = params[0][3]
kwargs['time_scale_factor'] = params[1]
workflow.execute_entity_task(vascaling.run_random_climate,
gdirs,
temperature_bias=temp_bias,
output_filesuffix=suffix,
**kwargs)
else:
# run ConstantMassBalance model centered around t* for each given
# parameter set
for suffix, params in zip(suffixes, sensitivity_params):
cfg.PARAMS['vas_c_length_m'] = params[0][0]
cfg.PARAMS['vas_c_area_m2'] = params[0][1]
cfg.PARAMS['vas_q_length'] = params[0][2]
cfg.PARAMS['vas_gamma_area'] = params[0][3]
kwargs['time_scale_factor'] = params[1]
workflow.execute_entity_task(vascaling.run_constant_climate,
gdirs,
temperature_bias=temp_bias,
output_filesuffix=suffix,
**kwargs)
# Process output dataset(s)
# -------------------------
# create empty container
ds = list()
# iterate over all scaling constants
for i, params in enumerate(scaling_params):
# create empty container
ds_ = list()
# iterate over all time scale factor
for j, factor in enumerate(time_scale_factors):
# compile the output for each run
pos = j + len(time_scale_factors) * i
ds__ = utils.compile_run_output(np.atleast_1d(gdirs),
filesuffix=suffixes[pos],
path=False)
# add time scale factor as coordinate
ds__.coords['time_scale_factor'] = factor
# add to container
ds_.append(ds__)
# concatenate using time scale factor as concat dimension
ds_ = xr.concat(ds_, dim='time_scale_factor')
# add scaling constants as coordinate
params_list = np.zeros(len([params]), dtype=object)
params_list[:] = [params]
ds_ = ds_.expand_dims(dim={'scaling_params': params_list})
# add to container
ds.append(ds_)
# concatenate using scaling constants as concat dimension
ds = xr.concat(ds, dim='scaling_params')
# add model type as coordinate
ds.coords['model'] = 'vas'
# add mb model type as coordinate
ds.coords['mb_model'] = 'random' if use_random_mb else 'constant'
# normalize glacier geometries (length/area/volume) with start value
if use_mean:
# compute average over all glaciers
ds_normal = normalize_ds_with_start(ds).mean(dim='rgi_id')
ds = ds.mean(dim='rgi_id')
else:
# compute sum over all glaciers
ds_normal = normalize_ds_with_start(ds.sum(dim='rgi_id'))
ds = ds.sum(dim='rgi_id')
# add coordinate to distinguish between normalized and absolute values
ds.coords['normalized'] = False
ds_normal.coords['normalized'] = True
# combine datasets
ds = xr.concat([ds, ds_normal], 'normalized')
# store datasets
if path:
if not isinstance(path, str):
# set default path and filename
mb = 'random' if use_random_mb else 'constant'
path = os.path.join(OUTPUT_DIR, f'run_output_{mb}_vas.nc')
# write to file
log.info(f'Writing run output to {path}')
pickle.dump(ds, open(path, mode='wb'))
# return ds, ds_normal
return ds
