def test_ice_cap():
testdir = os.path.join(cfg.PATHS['test_dir'], 'tmp_icecap')
utils.mkdir(testdir)
cfg.initialize()
cfg.PATHS['dem_file'] = get_demo_file('dem_RGI50-05.08389.tif')
cfg.PARAMS['border'] = 20
cfg.set_divides_db(get_demo_file('divides_RGI50-05.08389.shp'))
hef_file = get_demo_file('RGI50-05.08389.shp')
entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0]
gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=True)
gis.define_glacier_region(gdir, entity=entity)
gis.glacier_masks(gdir)
centerlines.compute_centerlines(gdir)
centerlines.compute_downstream_lines(gdir)
geometry.initialize_flowlines(gdir)
# We should have five groups
lines = gdir.read_pickle('downstream_lines', div_id=0)
assert len(np.unique(lines.group))==5
# This just checks that it works
geometry.catchment_area(gdir)
geometry.catchment_intersections(gdir)
geometry.catchment_width_geom(gdir)
geometry.catchment_width_correction(gdir)
fig, ax = plt.subplots()
graphics.plot_catchment_width(gdir, ax=ax, add_intersects=True,
add_touches=True)
fig.tight_layout()
return fig
def test_nodivide_corrected():
# test directory
testdir = os.path.join(cfg.PATHS['test_dir'], 'tmp_nodiv')
if not os.path.exists(testdir):
os.makedirs(testdir)
# Init
cfg.initialize()
cfg.set_divides_db()
cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif')
cfg.PATHS['climate_file'] = get_demo_file('histalp_merged_hef.nc')
cfg.PARAMS['border'] = 40
hef_file = get_demo_file('Hintereisferner_RGI5.shp')
entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0]
gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=True)
gis.define_glacier_region(gdir, entity=entity)
gis.glacier_masks(gdir)
centerlines.compute_centerlines(gdir)
geometry.initialize_flowlines(gdir)
geometry.catchment_area(gdir)
geometry.catchment_intersections(gdir)
geometry.catchment_width_geom(gdir)
geometry.catchment_width_correction(gdir)
fig, ax = plt.subplots()
graphics.plot_catchment_width(gdir, ax=ax, corrected=True,
add_intersects=True, add_touches=True)
fig.tight_layout()
shutil.rmtree(testdir)
return fig
def test_nodivide():
# test directory
testdir = TESTDIR_BASE + '_nodiv'
if not os.path.exists(testdir):
os.makedirs(testdir)
# Init
cfg.initialize()
cfg.set_divides_db()
cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif')
cfg.PATHS['climate_file'] = get_demo_file('histalp_merged_hef.nc')
cfg.PARAMS['border'] = 40
# loop because for some reason indexing wont work
hef_file = get_demo_file('Hintereisferner.shp')
rgidf = gpd.GeoDataFrame.from_file(hef_file)
for index, entity in rgidf.iterrows():
gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=True)
gis.define_glacier_region(gdir, entity=entity)
gis.glacier_masks(gdir)
centerlines.compute_centerlines(gdir)
graphics.plot_centerlines(gdir)
def setUp(self):
N = 3
cfg.initialize()
self.glen_a = 2.4e-24 # Modern style Glen parameter A
self.aglen_old = (N + 2) * 1.9e-24 / 2. # outdated value
self.fd = 2. * self.glen_a / (N + 2.) # equivalent to glen_a
self.fs = 0 # set slidin
self.fs_old = 5.7e-20 # outdated value
def test_download_cru(self):
cfg.initialize()
tmp = cfg.PATHS['cru_dir']
cfg.PATHS['cru_dir'] = TEST_DIR
of = utils.get_cru_file('tmp')
self.assertTrue(os.path.exists(of))
cfg.PATHS['cru_dir'] = tmp
def setUp(self):
# test directory
self.testdir = os.path.join(current_dir, "tmp")
if not os.path.exists(self.testdir):
os.makedirs(self.testdir)
self.clean_dir()
# Init
cfg.initialize()
cfg.PATHS["dem_file"] = get_demo_file("hef_srtm.tif")
def setUp(self):
# test directory
self.testdir = os.path.join(current_dir, 'tmp')
if not os.path.exists(self.testdir):
os.makedirs(self.testdir)
self.clean_dir()
# Init
cfg.initialize()
cfg.set_divides_db(get_demo_file('HEF_divided.shp'))
cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif')
def test_download_rgi(self):
cfg.initialize()
tmp = cfg.PATHS['rgi_dir']
cfg.PATHS['rgi_dir'] = TEST_DIR
of = utils.get_rgi_dir()
of = os.path.join(of, '01_rgi50_Alaska', '01_rgi50_Alaska.shp')
self.assertTrue(os.path.exists(of))
cfg.PATHS['rgi_dir'] = tmp
def test_coxe():
testdir = os.path.join(cfg.PATHS['test_dir'], 'tmp_coxe')
utils.mkdir(testdir)
# Init
cfg.initialize()
cfg.PATHS['dem_file'] = get_demo_file('dem_RGI50-01.10299.tif')
cfg.PARAMS['border'] = 40
cfg.PARAMS['use_multiple_flowlines'] = False
hef_file = get_demo_file('rgi_RGI50-01.10299.shp')
entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0]
gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=True)
gis.define_glacier_region(gdir, entity=entity)
gis.glacier_masks(gdir)
centerlines.compute_centerlines(gdir)
centerlines.compute_downstream_lines(gdir)
geometry.initialize_flowlines(gdir)
# Just check if the rest runs
centerlines.compute_downstream_bedshape(gdir)
geometry.catchment_area(gdir)
geometry.catchment_intersections(gdir)
geometry.catchment_width_geom(gdir)
geometry.catchment_width_correction(gdir)
climate.apparent_mb_from_linear_mb(gdir)
inversion.prepare_for_inversion(gdir)
inversion.volume_inversion(gdir, use_cfg_params={'glen_a': cfg.A,
'fs': 0})
inversion.filter_inversion_output(gdir)
flowline.init_present_time_glacier(gdir)
fls = gdir.read_pickle('model_flowlines')
p = gdir.read_pickle('linear_mb_params')
mb_mod = massbalance.LinearMassBalanceModel(ela_h=p['ela_h'],
grad=p['grad'])
mb_mod.temp_bias = -0.3
model = flowline.FluxBasedModel(fls, mb_model=mb_mod, y0=0,
is_tidewater=True)
# run
model.run_until(200)
assert model.calving_m3_since_y0 > 0
fig, ax = plt.subplots()
graphics.plot_modeloutput_map(gdir, ax=ax, model=model)
fig.tight_layout()
return fig
def setUp(self):
# test directory
self.testdir = os.path.join(current_dir, "tmp")
if not os.path.exists(self.testdir):
os.makedirs(self.testdir)
self.clean_dir()
# Init
cfg.initialize()
cfg.PATHS["dem_file"] = get_demo_file("hef_srtm.tif")
cfg.PATHS["climate_file"] = get_demo_file("histalp_merged_hef.nc")
cfg.PARAMS["border"] = 10
def setUp(self):
# test directory
self.testdir = os.path.join(current_dir, "tmp_grindel")
self.clean_dir()
# Init
cfg.initialize()
cfg.PARAMS["use_multiple_flowlines"] = False
# not crop
cfg.PARAMS["max_thick_to_width_ratio"] = 10
cfg.PARAMS["max_shape_param"] = 10
cfg.PARAMS["section_smoothing"] = 0.0
def setUp(self):
# test directory
self.testdir = os.path.join(current_dir, 'tmp')
if not os.path.exists(self.testdir):
os.makedirs(self.testdir)
self.clean_dir()
# Init
cfg.initialize()
cfg.set_divides_db(get_demo_file('divides_workflow.shp'))
cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif')
cfg.PATHS['climate_file'] = get_demo_file('histalp_merged_hef.nc')
cfg.PARAMS['border'] = 10
def up_to_climate(reset=False):
"""Run the tasks you want."""
# test directory
if not os.path.exists(TEST_DIR):
os.makedirs(TEST_DIR)
if reset:
clean_dir(TEST_DIR)
if not os.path.exists(CLI_LOGF):
with open(CLI_LOGF, 'wb') as f:
pickle.dump('none', f)
# Init
cfg.initialize()
# Use multiprocessing
cfg.PARAMS['use_multiprocessing'] = use_multiprocessing()
# Working dir
cfg.PATHS['working_dir'] = TEST_DIR
cfg.PATHS['dem_file'] = get_demo_file('srtm_oetztal.tif')
cfg.set_intersects_db(get_demo_file('rgi_intersect_oetztal.shp'))
# Read in the RGI file
rgi_file = get_demo_file('rgi_oetztal.shp')
rgidf = gpd.read_file(rgi_file)
# Be sure data is downloaded
utils.get_cru_cl_file()
# Params
cfg.PARAMS['border'] = 70
cfg.PARAMS['tstar_search_window'] = [1902, 0]
cfg.PARAMS['run_mb_calibration'] = True
# Go
gdirs = workflow.init_glacier_regions(rgidf)
try:
tasks.catchment_width_correction(gdirs[0])
except Exception:
reset = True
if reset:
# First preprocessing tasks
workflow.gis_prepro_tasks(gdirs)
return gdirs
def setUp(self):
# test directory
self.testdir = os.path.join(get_test_dir(), 'tmp')
if not os.path.exists(self.testdir):
os.makedirs(self.testdir)
self.clean_dir()
self.rgi_file = get_demo_file('rgi_RGI50-01.10299.shp')
# Init
cfg.initialize()
cfg.PARAMS['use_intersects'] = False
cfg.PATHS['dem_file'] = get_demo_file('dem_RGI50-01.10299.tif')
cfg.PARAMS['border'] = 40
def setUp(self):
# test directory
self.testdir = os.path.join(get_test_dir(), 'tmp')
if not os.path.exists(self.testdir):
os.makedirs(self.testdir)
self.clean_dir()
# Init
cfg.initialize()
cfg.PARAMS['use_intersects'] = False
cfg.PATHS['working_dir'] = self.testdir
cfg.PATHS['dem_file'] = get_demo_file('dem_SouthGlacier.tif')
cfg.PATHS['cru_dir'] = os.path.dirname(cfg.PATHS['dem_file'])
cfg.PARAMS['border'] = 10
def test_chhota_shigri():
testdir = os.path.join(cfg.PATHS['test_dir'], 'tmp_chhota')
utils.mkdir(testdir)
# Init
cfg.initialize()
cfg.PATHS['dem_file'] = get_demo_file('dem_chhota_shigri.tif')
cfg.PARAMS['border'] = 60
cfg.set_divides_db(get_demo_file('divides_RGI50-14.15990.shp'))
hef_file = get_demo_file('RGI50-14.15990.shp')
entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0]
gdir = oggm.GlacierDirectory(entity, base_dir=testdir)
gis.define_glacier_region(gdir, entity=entity)
gis.glacier_masks(gdir)
centerlines.compute_centerlines(gdir)
centerlines.compute_downstream_lines(gdir)
geometry.initialize_flowlines(gdir)
# We should have two groups
lines = gdir.read_pickle('downstream_lines', div_id=0)
assert len(np.unique(lines.group)) == 2
# Just check if the rest runs
centerlines.compute_downstream_bedshape(gdir)
geometry.catchment_area(gdir)
geometry.catchment_intersections(gdir)
geometry.catchment_width_geom(gdir)
geometry.catchment_width_correction(gdir)
climate.apparent_mb_from_linear_mb(gdir)
inversion.prepare_for_inversion(gdir)
inversion.volume_inversion(gdir, use_cfg_params={'glen_a': cfg.A,
'fs': 0})
inversion.filter_inversion_output(gdir)
flowline.init_present_time_glacier(gdir)
fls = gdir.read_pickle('model_flowlines')
for fl in fls:
fl.thick = np.clip(fl.thick, 100, 1000)
model = flowline.FlowlineModel(fls)
fig, ax = plt.subplots()
graphics.plot_modeloutput_map(gdir, ax=ax, model=model)
fig.tight_layout()
return fig
def test_download_demo_files(self):
f = utils.get_demo_file('Hintereisferner.shp')
self.assertTrue(os.path.exists(f))
sh = salem.read_shapefile(f)
self.assertTrue(hasattr(sh, 'geometry'))
# Data files
cfg.initialize()
lf, df = utils.get_wgms_files()
self.assertTrue(os.path.exists(lf))
lf = utils.get_glathida_file()
self.assertTrue(os.path.exists(lf))
def test_nodivide():
# test directory
testdir = TESTDIR_BASE + "_nodiv"
if not os.path.exists(testdir):
os.makedirs(testdir)
# Init
cfg.initialize()
cfg.set_divides_db()
cfg.PATHS["dem_file"] = get_demo_file("hef_srtm.tif")
cfg.PATHS["climate_file"] = get_demo_file("histalp_merged_hef.nc")
cfg.PARAMS["border"] = 40
hef_file = get_demo_file("Hintereisferner.shp")
entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0]
gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=True)
gis.define_glacier_region(gdir, entity=entity)
gis.glacier_masks(gdir)
centerlines.compute_centerlines(gdir)
graphics.plot_centerlines(gdir)
def init_hef(reset=False, border=40):
# test directory
testdir = os.path.join(get_test_dir(), 'tmp_border{}'.format(border))
if not os.path.exists(testdir):
os.makedirs(testdir)
reset = True
# Init
cfg.initialize()
cfg.set_intersects_db(get_demo_file('rgi_intersect_oetztal.shp'))
cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif')
cfg.PATHS['climate_file'] = get_demo_file('histalp_merged_hef.nc')
cfg.PARAMS['baseline_climate'] = ''
cfg.PATHS['working_dir'] = testdir
cfg.PARAMS['border'] = border
hef_file = get_demo_file('Hintereisferner_RGI5.shp')
entity = gpd.read_file(hef_file).iloc[0]
gdir = oggm.GlacierDirectory(entity, reset=reset)
if not gdir.has_file('inversion_params'):
reset = True
gdir = oggm.GlacierDirectory(entity, reset=reset)
if not reset:
return gdir
gis.define_glacier_region(gdir, entity=entity)
execute_entity_task(gis.glacier_masks, [gdir])
execute_entity_task(centerlines.compute_centerlines, [gdir])
centerlines.initialize_flowlines(gdir)
centerlines.compute_downstream_line(gdir)
centerlines.compute_downstream_bedshape(gdir)
centerlines.catchment_area(gdir)
centerlines.catchment_intersections(gdir)
centerlines.catchment_width_geom(gdir)
centerlines.catchment_width_correction(gdir)
climate.process_custom_climate_data(gdir)
mbdf = gdir.get_ref_mb_data()['ANNUAL_BALANCE']
res = climate.t_star_from_refmb(gdir, mbdf=mbdf)
climate.local_t_star(gdir, tstar=res['t_star'], bias=res['bias'])
climate.mu_star_calibration(gdir)
inversion.prepare_for_inversion(gdir, add_debug_var=True)
ref_v = 0.573 * 1e9
glen_n = cfg.PARAMS['glen_n']
def to_optimize(x):
# For backwards compat
_fd = 1.9e-24 * x[0]
glen_a = (glen_n+2) * _fd / 2.
fs = 5.7e-20 * x[1]
v, _ = inversion.mass_conservation_inversion(gdir, fs=fs,
glen_a=glen_a)
return (v - ref_v)**2
out = optimization.minimize(to_optimize, [1, 1],
bounds=((0.01, 10), (0.01, 10)),
tol=1e-4)['x']
_fd = 1.9e-24 * out[0]
glen_a = (glen_n+2) * _fd / 2.
fs = 5.7e-20 * out[1]
v, _ = inversion.mass_conservation_inversion(gdir, fs=fs,
glen_a=glen_a,
write=True)
d = dict(fs=fs, glen_a=glen_a)
d['factor_glen_a'] = out[0]
d['factor_fs'] = out[1]
gdir.write_pickle(d, 'inversion_params')
# filter
inversion.filter_inversion_output(gdir)
inversion.distribute_thickness_interp(gdir, varname_suffix='_interp')
inversion.distribute_thickness_per_altitude(gdir, varname_suffix='_alt')
flowline.init_present_time_glacier(gdir)
return gdir
def run_prepro_levels(rgi_version=None, rgi_reg=None, border=None,
output_folder='', working_dir='', is_test=False,
demo=False, test_rgidf=None, test_intersects_file=None,
test_topofile=None, test_crudir=None):
"""Does the actual job.
Parameters
----------
rgi_version : str
the RGI version to use (defaults to cfg.PARAMS)
rgi_reg : str
the RGI region to process
border : int
the number of pixels at the maps border
output_folder : str
path to the output folder (where to put the preprocessed tar files)
working_dir : str
path to the OGGM working directory
is_test : bool
to test on a couple of glaciers only!
demo : bool
to run the prepro for the list of demo glaciers
test_rgidf : shapefile
for testing purposes only
test_intersects_file : shapefile
for testing purposes only
test_topofile : str
for testing purposes only
test_crudir : str
for testing purposes only
"""
# TODO: temporarily silence Fiona deprecation warnings
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
# Module logger
log = logging.getLogger(__name__)
# Time
start = time.time()
# Initialize OGGM and set up the run parameters
cfg.initialize(logging_level='WORKFLOW')
# Local paths
utils.mkdir(working_dir)
cfg.PATHS['working_dir'] = working_dir
# Use multiprocessing?
cfg.PARAMS['use_multiprocessing'] = True
# How many grid points around the glacier?
# Make it large if you expect your glaciers to grow large
cfg.PARAMS['border'] = border
# Set to True for operational runs
cfg.PARAMS['continue_on_error'] = True
# For statistics
climate_periods = [1920, 1960, 2000]
if rgi_version is None:
rgi_version = cfg.PARAMS['rgi_version']
rgi_dir_name = 'RGI{}'.format(rgi_version)
border_dir_name = 'b_{:03d}'.format(border)
base_dir = os.path.join(output_folder, rgi_dir_name, border_dir_name)
# Add a package version file
utils.mkdir(base_dir)
opath = os.path.join(base_dir, 'package_versions.txt')
with open(opath, 'w') as vfile:
vfile.write(utils.show_versions(logger=log))
if demo:
rgidf = utils.get_rgi_glacier_entities(cfg.DEMO_GLACIERS.index)
elif test_rgidf is None:
# Get the RGI file
rgidf = gpd.read_file(utils.get_rgi_region_file(rgi_reg,
version=rgi_version))
# We use intersects
rgif = utils.get_rgi_intersects_region_file(rgi_reg,
version=rgi_version)
cfg.set_intersects_db(rgif)
else:
rgidf = test_rgidf
cfg.set_intersects_db(test_intersects_file)
if is_test:
# Just for fun
rgidf = rgidf.sample(4)
# Sort for more efficient parallel computing
rgidf = rgidf.sort_values('Area', ascending=False)
log.workflow('Starting prepro run for RGI reg: {} '
'and border: {}'.format(rgi_reg, border))
log.workflow('Number of glaciers: {}'.format(len(rgidf)))
# Input
if test_topofile:
cfg.PATHS['dem_file'] = test_topofile
# L1 - initialize working directories
gdirs = workflow.init_glacier_regions(rgidf, reset=True, force=True)
# Glacier stats
sum_dir = os.path.join(base_dir, 'L1', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# L1 OK - compress all in output directory
l_base_dir = os.path.join(base_dir, 'L1')
workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False,
base_dir=l_base_dir)
utils.base_dir_to_tar(l_base_dir)
# L2 - Tasks
# Pre-download other files just in case
if test_crudir is None:
_ = utils.get_cru_file(var='tmp')
_ = utils.get_cru_file(var='pre')
else:
cfg.PATHS['cru_dir'] = test_crudir
workflow.execute_entity_task(tasks.process_cru_data, gdirs)
# Glacier stats
sum_dir = os.path.join(base_dir, 'L2', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# L2 OK - compress all in output directory
l_base_dir = os.path.join(base_dir, 'L2')
workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False,
base_dir=l_base_dir)
utils.base_dir_to_tar(l_base_dir)
# L3 - Tasks
task_list = [
tasks.glacier_masks,
tasks.compute_centerlines,
tasks.initialize_flowlines,
tasks.compute_downstream_line,
tasks.compute_downstream_bedshape,
tasks.catchment_area,
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
tasks.local_t_star,
tasks.mu_star_calibration,
tasks.prepare_for_inversion,
tasks.mass_conservation_inversion,
tasks.filter_inversion_output,
]
for task in task_list:
workflow.execute_entity_task(task, gdirs)
# Glacier stats
sum_dir = os.path.join(base_dir, 'L3', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
opath = os.path.join(sum_dir, 'climate_statistics_{}.csv'.format(rgi_reg))
utils.compile_climate_statistics(gdirs, add_climate_period=climate_periods,
path=opath)
# L3 OK - compress all in output directory
l_base_dir = os.path.join(base_dir, 'L3')
workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False,
base_dir=l_base_dir)
utils.base_dir_to_tar(l_base_dir)
# L4 - Tasks
workflow.execute_entity_task(tasks.init_present_time_glacier, gdirs)
# Glacier stats
sum_dir = os.path.join(base_dir, 'L4', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# Copy mini data to new dir
base_dir = os.path.join(base_dir, 'L4')
mini_gdirs = workflow.execute_entity_task(tasks.copy_to_basedir, gdirs,
base_dir=base_dir)
# L4 OK - compress all in output directory
workflow.execute_entity_task(utils.gdir_to_tar, mini_gdirs, delete=True)
utils.base_dir_to_tar(base_dir)
# Log
m, s = divmod(time.time() - start, 60)
h, m = divmod(m, 60)
log.workflow('OGGM prepro_levels is done! Time needed: '
'{:02d}:{:02d}:{:02d}'.format(int(h), int(m), int(s)))
def setUp(self):
cfg.initialize()
def setUp(self):
cfg.initialize()
cfg.PATHS['dl_cache_dir'] = os.path.join(TEST_DIR, 'dl_cache')
cfg.PATHS['working_dir'] = os.path.join(TEST_DIR, 'wd')
cfg.PATHS['tmp_dir'] = os.path.join(TEST_DIR, 'extract')
self.reset_dir()
import geopandas as gpd
import shapely.geometry as shpg
# Locals
import oggm.cfg as cfg
from oggm import utils, workflow, tasks
# For timing the run
import time
start = time.time()
# Module logger
log = logging.getLogger(__name__)
# Initialize OGGM and set up the default run parameters
cfg.initialize(logging_level='WORKFLOW')
rgi_version = '61'
rgi_region = '11' # Region Central Europe
# Here we override some of the default parameters
# How many grid points around the glacier?
# Make it large if you expect your glaciers to grow large:
# here, 80 is more than enough
cfg.PARAMS['border'] = 80
# Local working directory (where OGGM will write its output)
WORKING_DIR = utils.gettempdir('OGGM_Rofental')
utils.mkdir(WORKING_DIR, reset=True)
cfg.PATHS['working_dir'] = WORKING_DIR
# RGI file
from oggm.workflow import execute_entity_task
from oggm import utils
from oggm.core import inversion
import pandas as pd
# Time
import time
start = time.time()
# Regions: Alaska
rgi_region = '01'
# Initialize OGGM and set up the run parameters
# ---------------------------------------------
cfg.initialize()
rgi_version = '6'
SLURM_WORKDIR = os.environ["WORKDIR"]
# Local paths (where to write output and where to download input)
WORKING_DIR = SLURM_WORKDIR
MAIN_PATH = os.path.expanduser('~/cryo_calving_2019/')
RGI_FILE = os.path.join(
MAIN_PATH, 'input_data/01_rgi60_Alaska_modify/01_rgi60_Alaska.shp')
Columbia_prepro = os.path.join(
MAIN_PATH,
'output_data/1_Columbia_Glacier_runs/1_preprocessing/per_glacier/')
def setUp(self):
cfg.initialize()
self.homedir = os.path.expanduser('~')
def sensitivity_run_vas(rgi_ids,
use_random_mb=False,
use_mean=False,
path=True,
temp_bias=0,
tstar=None,
use_default_tstar=True,
use_bias_for_run=True,
scaling_params=[(4.5507, 0.191, 2.2, 1.375)],
time_scale_factors=[1],
suffixes=['_default'],
**kwargs):
""" The routine runs all steps for the equilibrium experiments using the
volume/area scaling model (cf. `equilibrium_run_vas`) but for only one
given temperature bias. However, it is possible to supply a list of
sensitivity parameters (the scaling constants, and time scale factor) to
alter the model behavior.
- OGGM preprocessing, including initialization, GIS tasks, climate tasks and
massbalance tasks.
- Run model for all glaciers with constant (or random) massbalance model
over 3000 years (default value).
- Process the model output dataset(s), i.e. normalization, average/sum, ...
The final dataset containing all results is returned. Given a path is is
also stored to file.
Parameters
----------
rgi_ids: array-like
List of RGI IDs for which the equilibrium experiments are performed.
use_random_mb: bool, optional, default=True
Choose between random massbalance model and constant massbalance model.
use_mean: bool, optional, default=False
Choose between the mean or summation over all glaciers
path: bool or str, optional, default=True
If a path is given (or True), the resulting dataset is stored to file.
temp_bias: float, optional, default=0
Temperature bias (degC) for the mass balance model.
sensitivity_params: multi-dimensional array-like, optional,
default=[(4.5507, 0.191, 2.2, 1.375)]
List containing the scaling constants and scaling exponents for length
and area scaling as tuples, e.g., (c_l, c_a, q, gamma)
suffixes: array-like, optional, default=['_default']
Descriptive suffixes corresponding to the given sensitivity params
tstar: float, optional, default=None
'Equilibrium year' used for the mass balance calibration.
use_default_tstar: bool, optional, default=True
Flag deciding whether or not to compute mustar from given from reference
table. Overridden by a given tstar.
use_bias_for_run: bool, optional, default=True
Flag deciding whether or not to use the mass balance residual.
kwargs:
Additional key word arguments for the `run_random_climate` or
`run_constant_climate` routines of the vascaling module.
Returns
-------
Dataset containing yearly values of all glacier geometries.
"""
# assert correct output file suffixes for temp biases
if len(scaling_params) * len(time_scale_factors) != len(suffixes):
raise RuntimeError("Each given combination of scaling parameters and "
"time scale factor must have one, and only one, "
"corresponding suffix.")
# OGGM preprocessing
# ------------------
# compute RGI region and version from RGI IDs
# assuming all they are all the same
rgi_region = (rgi_ids[0].split('-')[-1]).split('.')[0]
rgi_version = (rgi_ids[0].split('-')[0])[-2:-1]
# load default parameter file
cfg.initialize()
# get environmental variables for working and output directories
WORKING_DIR = os.environ["WORKDIR"]
OUTPUT_DIR = os.environ["OUTDIR"]
# create working directory
utils.mkdir(WORKING_DIR)
utils.mkdir(OUTPUT_DIR)
# set path to working directory
cfg.PATHS['working_dir'] = WORKING_DIR
# set RGI version and region
cfg.PARAMS['rgi_version'] = rgi_version
# define how many grid points to use around the glacier,
# if you expect the glacier to grow large use a larger border
cfg.PARAMS['border'] = 120
# we use HistAlp climate data
cfg.PARAMS['baseline_climate'] = 'HISTALP'
# set the mb hyper parameters accordingly
cfg.PARAMS['prcp_scaling_factor'] = 2.5
cfg.PARAMS['temp_melt'] = -0.5
# the bias is defined to be zero during the calibration process,
# which is why we don't use it here to reproduce the results
cfg.PARAMS['use_bias_for_run'] = use_bias_for_run
# read RGI entry for the glaciers as DataFrame
# containing the outline area as shapefile
rgidf = utils.get_rgi_glacier_entities(rgi_ids)
# get and set path to intersect shapefile
intersects_db = utils.get_rgi_intersects_region_file(region=rgi_region)
cfg.set_intersects_db(intersects_db)
# sort by area for more efficient parallel computing
rgidf = rgidf.sort_values('Area', ascending=False)
cfg.PARAMS['use_multiprocessing'] = True
# operational run, all glaciers should run
cfg.PARAMS['continue_on_error'] = True
# initialize the GlacierDirectory
gdirs = workflow.init_glacier_directories(rgidf, reset=False, force=True)
# define the local grid and glacier mask
workflow.execute_entity_task(gis.define_glacier_region, gdirs)
workflow.execute_entity_task(gis.glacier_masks, gdirs)
# process the given climate file
workflow.execute_entity_task(climate.process_climate_data, gdirs)
# compute local t* and the corresponding mu*
if tstar or use_default_tstar:
# compute mustar from given tstar or reference table
workflow.execute_entity_task(vascaling.local_t_star,
gdirs,
tstar=tstar,
bias=0)
else:
# compute mustar from the reference table for the flowline model
# RGI v6 and HISTALP baseline climate
ref_df = pd.read_csv(
utils.get_demo_file('oggm_ref_tstars_rgi6_histalp.csv'))
workflow.execute_entity_task(vascaling.local_t_star,
gdirs,
ref_df=ref_df)
# Run model with constant/random mass balance model
# -------------------------------------------------
# use t* as center year, even if specified differently
kwargs['y0'] = tstar
# run for 3000 years if not specified otherwise
kwargs.setdefault('nyears', 1000)
# limit parameters to 3 decimal points
scaling_params = recursive_round(scaling_params, 3)
time_scale_factors = recursive_round(time_scale_factors, 3)
# assure that scaling params are handled as tuples (pairs)
scaling_params_list = np.zeros(len(scaling_params), dtype=object)
scaling_params_list[:] = scaling_params
# combine scaling constants, scaling exponents and time scale factor
# into one iterable array
sensitivity_params = np.array(
np.meshgrid(scaling_params_list, time_scale_factors)).T
sensitivity_params = (sensitivity_params.reshape(
-1, sensitivity_params.shape[-1]))
if use_random_mb:
# set random seed to get reproducible results
kwargs.setdefault('seed', 12)
# run RandomMassBalance model centered around t* for each given
# parameter set
for suffix, params in zip(suffixes, sensitivity_params):
cfg.PARAMS['vas_c_length_m'] = params[0][0]
cfg.PARAMS['vas_c_area_m2'] = params[0][1]
cfg.PARAMS['vas_q_length'] = params[0][2]
cfg.PARAMS['vas_gamma_area'] = params[0][3]
kwargs['time_scale_factor'] = params[1]
workflow.execute_entity_task(vascaling.run_random_climate,
gdirs,
temperature_bias=temp_bias,
output_filesuffix=suffix,
**kwargs)
else:
# run ConstantMassBalance model centered around t* for each given
# parameter set
for suffix, params in zip(suffixes, sensitivity_params):
cfg.PARAMS['vas_c_length_m'] = params[0][0]
cfg.PARAMS['vas_c_area_m2'] = params[0][1]
cfg.PARAMS['vas_q_length'] = params[0][2]
cfg.PARAMS['vas_gamma_area'] = params[0][3]
kwargs['time_scale_factor'] = params[1]
workflow.execute_entity_task(vascaling.run_constant_climate,
gdirs,
temperature_bias=temp_bias,
output_filesuffix=suffix,
**kwargs)
# Process output dataset(s)
# -------------------------
# create empty container
ds = list()
# iterate over all scaling constants
for i, params in enumerate(scaling_params):
# create empty container
ds_ = list()
# iterate over all time scale factor
for j, factor in enumerate(time_scale_factors):
# compile the output for each run
pos = j + len(time_scale_factors) * i
ds__ = utils.compile_run_output(np.atleast_1d(gdirs),
filesuffix=suffixes[pos],
path=False)
# add time scale factor as coordinate
ds__.coords['time_scale_factor'] = factor
# add to container
ds_.append(ds__)
# concatenate using time scale factor as concat dimension
ds_ = xr.concat(ds_, dim='time_scale_factor')
# add scaling constants as coordinate
params_list = np.zeros(len([params]), dtype=object)
params_list[:] = [params]
ds_ = ds_.expand_dims(dim={'scaling_params': params_list})
# add to container
ds.append(ds_)
# concatenate using scaling constants as concat dimension
ds = xr.concat(ds, dim='scaling_params')
# add model type as coordinate
ds.coords['model'] = 'vas'
# add mb model type as coordinate
ds.coords['mb_model'] = 'random' if use_random_mb else 'constant'
# normalize glacier geometries (length/area/volume) with start value
if use_mean:
# compute average over all glaciers
ds_normal = normalize_ds_with_start(ds).mean(dim='rgi_id')
ds = ds.mean(dim='rgi_id')
else:
# compute sum over all glaciers
ds_normal = normalize_ds_with_start(ds.sum(dim='rgi_id'))
ds = ds.sum(dim='rgi_id')
# add coordinate to distinguish between normalized and absolute values
ds.coords['normalized'] = False
ds_normal.coords['normalized'] = True
# combine datasets
ds = xr.concat([ds, ds_normal], 'normalized')
# store datasets
if path:
if not isinstance(path, str):
# set default path and filename
mb = 'random' if use_random_mb else 'constant'
path = os.path.join(OUTPUT_DIR, f'run_output_{mb}_vas.nc')
# write to file
log.info(f'Writing run output to {path}')
pickle.dump(ds, open(path, mode='wb'))
# return ds, ds_normal
return ds
def compute_scaling_params(rgi_ids, path=None):
""" The routine computes scaling parameters by fitting a linear regression
to the volume/area and volume/length scatter in log-log space, using the
inversion volume, the RGI area and the longest centerline as "observations"
Thereby, the following two cases apply:
- compute only scaling constants, since scaling exponents have a physical
basis and should not be changed
- compute only scaling constants and scaling exponents
Returns parameters in a 2-level dictionary. The upper level differentiates
between the two cases, the lower level indicates the parameters.
Parameters
----------
rgi_ids: array-like
List of RGI IDs for which the equilibrium experiments are performed.
Returns
-------
Dictionary containing the computed parameters.
"""
log.info('Starting scaling parameter computation')
# compute RGI region and version from RGI IDs
# assuming all they are all the same
rgi_region = (rgi_ids[0].split('-')[-1]).split('.')[0]
rgi_version = (rgi_ids[0].split('-')[0])[-2:-1]
# load default parameter file
cfg.initialize()
# get environmental variables for working and output directories
WORKING_DIR = os.environ["WORKDIR"]
OUTPUT_DIR = os.environ["OUTDIR"]
# create working directory
utils.mkdir(WORKING_DIR)
utils.mkdir(OUTPUT_DIR)
# set path to working directory
cfg.PATHS['working_dir'] = WORKING_DIR
# set RGI version and region
cfg.PARAMS['rgi_version'] = rgi_version
# define how many grid points to use around the glacier,
# if you expect the glacier to grow large use a larger border
cfg.PARAMS['border'] = 120
# we use HistAlp climate data
cfg.PARAMS['baseline_climate'] = 'HISTALP'
# set the mb hyper parameters accordingly
cfg.PARAMS['prcp_scaling_factor'] = 1.75
cfg.PARAMS['temp_melt'] = -1.75
# read RGI entry for the glaciers as DataFrame
# containing the outline area as shapefile
rgidf = utils.get_rgi_glacier_entities(rgi_ids)
# get and set path to intersect shapefile
intersects_db = utils.get_rgi_intersects_region_file(region=rgi_region)
cfg.set_intersects_db(intersects_db)
# the bias is defined to be zero during the calibration process,
# which is why we don't use it here to reproduce the results
cfg.PARAMS['use_bias_for_run'] = True
# sort by area for more efficient parallel computing
rgidf = rgidf.sort_values('Area', ascending=False)
cfg.PARAMS['use_multiprocessing'] = True
# operational run, all glaciers should run
cfg.PARAMS['continue_on_error'] = False
# initialize the GlacierDirectory
gdirs = workflow.init_glacier_directories(rgidf, reset=False, force=True)
# run gis tasks
workflow.gis_prepro_tasks(gdirs)
# run climate tasks
workflow.execute_entity_task(climate.process_climate_data, gdirs)
# compute local t* and the corresponding mu*
workflow.execute_entity_task(climate.local_t_star, gdirs)
workflow.execute_entity_task(climate.mu_star_calibration, gdirs)
# run inversion tasks
workflow.inversion_tasks(gdirs)
# finalize preprocessing
workflow.execute_entity_task(flowline.init_present_time_glacier, gdirs)
# create empty dictionary
params = dict()
# compute scaling constants for given (fixed) slope
params['const_only'] = vascaling.get_scaling_constant(gdirs)
# compute scaling constants and scaling exponent via linear regression
params['const_expo'] = vascaling.get_scaling_constant_exponent(gdirs)
# store to file
if path:
if not isinstance(path, str):
# set default path and filename
path = os.path.join(OUTPUT_DIR, 'scaling_params.json')
json.dump(params, open(path, 'w'))
return params
import pandas as pd import shapely.geometry as shpg # Locals import oggm.cfg as cfg from oggm import utils, workflow, tasks # For timing the run import time start = time.time() # Module logger log = logging.getLogger(__name__) #Initialize OGGM and set up the default run parameters cfg.initialize(logging_level='WORKFLOW') rgi_version = '61' rgi_region = '15' # No need for a big map here cfg.PARAMS['border'] = 80 # Local working directory (where OGGM will write its output) WORKING_DIR = '/Users/louis/Prepro_test/' utils.mkdir(WORKING_DIR, reset=True) cfg.PATHS['working_dir'] = WORKING_DIR # RGI file path = utils.get_rgi_region_file(rgi_region, version=rgi_version) rgidf = gpd.read_file(path)
import geopandas as gpd
import shapely.geometry as shpg
# Locals
import oggm.cfg as cfg
from oggm import utils, workflow, tasks
# For timing the run
import time
start = time.time()
# Module logger
log = logging.getLogger(__name__)
# Initialize OGGM and set up the default run parameters
cfg.initialize(logging_level='ERROR')
rgi_version = '61'
rgi_region = '11' # Region Central Europe
# Here we override some of the default parameters
# How many grid points around the glacier?
# Make it large if you expect your glaciers to grow large:
# here, 80 is more than enough
cfg.PARAMS['border'] = 80
# Local working directory (where OGGM will write its output)
WORKING_DIR = utils.gettempdir('OGGM_Rofental')
utils.mkdir(WORKING_DIR, reset=True)
cfg.PATHS['working_dir'] = WORKING_DIR
# RGI file
def init_hef(reset=False, border=40, invert_with_sliding=True):
from oggm.core.preprocessing import gis, centerlines, geometry
from oggm.core.preprocessing import climate, inversion
import oggm
import oggm.cfg as cfg
from oggm.utils import get_demo_file
# test directory
testdir = TESTDIR_BASE + "_border{}".format(border)
if not invert_with_sliding:
testdir += "_withoutslide"
if not os.path.exists(testdir):
os.makedirs(testdir)
reset = True
if not os.path.exists(os.path.join(testdir, "RGI40-11.00897")):
reset = True
if not os.path.exists(os.path.join(testdir, "RGI40-11.00897", "inversion_params.pkl")):
reset = True
# Init
cfg.initialize()
cfg.PATHS["dem_file"] = get_demo_file("hef_srtm.tif")
cfg.PATHS["climate_file"] = get_demo_file("histalp_merged_hef.nc")
cfg.PARAMS["border"] = border
hef_file = get_demo_file("Hintereisferner.shp")
entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0]
gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=reset)
if not reset:
return gdir
gis.define_glacier_region(gdir, entity=entity)
gis.glacier_masks(gdir)
centerlines.compute_centerlines(gdir)
centerlines.compute_downstream_lines(gdir)
geometry.initialize_flowlines(gdir)
geometry.catchment_area(gdir)
geometry.catchment_width_geom(gdir)
geometry.catchment_width_correction(gdir)
climate.process_histalp_nonparallel([gdir])
climate.mu_candidates(gdir, div_id=0)
hef_file = get_demo_file("mbdata_RGI40-11.00897.csv")
mbdf = pd.read_csv(hef_file).set_index("YEAR")
t_star, bias, prcp_fac = climate.t_star_from_refmb(gdir, mbdf["ANNUAL_BALANCE"])
climate.local_mustar_apparent_mb(gdir, tstar=t_star[-1], bias=bias[-1], prcp_fac=prcp_fac)
inversion.prepare_for_inversion(gdir)
ref_v = 0.573 * 1e9
if invert_with_sliding:
def to_optimize(x):
# For backwards compat
_fd = 1.9e-24 * x[0]
glen_a = (cfg.N + 2) * _fd / 2.0
fs = 5.7e-20 * x[1]
v, _ = inversion.invert_parabolic_bed(gdir, fs=fs, glen_a=glen_a)
return (v - ref_v) ** 2
out = optimization.minimize(to_optimize, [1, 1], bounds=((0.01, 10), (0.01, 10)), tol=1e-4)["x"]
_fd = 1.9e-24 * out[0]
glen_a = (cfg.N + 2) * _fd / 2.0
fs = 5.7e-20 * out[1]
v, _ = inversion.invert_parabolic_bed(gdir, fs=fs, glen_a=glen_a, write=True)
else:
def to_optimize(x):
glen_a = cfg.A * x[0]
v, _ = inversion.invert_parabolic_bed(gdir, fs=0.0, glen_a=glen_a)
return (v - ref_v) ** 2
out = optimization.minimize(to_optimize, [1], bounds=((0.01, 10),), tol=1e-4)["x"]
glen_a = cfg.A * out[0]
fs = 0.0
v, _ = inversion.invert_parabolic_bed(gdir, fs=fs, glen_a=glen_a, write=True)
d = dict(fs=fs, glen_a=glen_a)
d["factor_glen_a"] = out[0]
try:
d["factor_fs"] = out[1]
except IndexError:
d["factor_fs"] = 0.0
gdir.write_pickle(d, "inversion_params")
inversion.distribute_thickness(gdir, how="per_altitude", add_nc_name=True)
inversion.distribute_thickness(gdir, how="per_interpolation", add_slope=False, smooth=False, add_nc_name=True)
return gdir
def setup_module(module):
"""Used by pytest"""
cfg.initialize()
def up_to_inversion(reset=False):
"""Run the tasks you want."""
# test directory
if not os.path.exists(TEST_DIR):
os.makedirs(TEST_DIR)
if reset:
clean_dir(TEST_DIR)
# Init
cfg.initialize()
# Use multiprocessing
cfg.PARAMS['use_multiprocessing'] = not ON_TRAVIS
# Working dir
cfg.PATHS['working_dir'] = TEST_DIR
cfg.PATHS['dem_file'] = get_demo_file('srtm_oetztal.tif')
# Set up the paths and other stuffs
cfg.set_divides_db(get_demo_file('divides_workflow.shp'))
cfg.PATHS['wgms_rgi_links'] = get_demo_file('RGI_WGMS_oetztal.csv')
cfg.PATHS['glathida_rgi_links'] = get_demo_file('RGI_GLATHIDA_oetztal.csv')
# Read in the RGI file
rgi_file = get_demo_file('rgi_oetztal.shp')
rgidf = gpd.GeoDataFrame.from_file(rgi_file)
# Be sure data is downloaded because lock doesn't work
cl = utils.get_cru_cl_file()
# Params
cfg.PARAMS['border'] = 70
cfg.PARAMS['use_optimized_inversion_params'] = True
# Go
gdirs = workflow.init_glacier_regions(rgidf)
try:
flowline.init_present_time_glacier(gdirs[0])
except Exception:
reset = True
if reset:
# First preprocessing tasks
workflow.gis_prepro_tasks(gdirs)
# Climate related tasks
# See if CRU is running
cfg.PARAMS['temp_use_local_gradient'] = False
cfg.PATHS['climate_file'] = '~'
cru_dir = get_demo_file('cru_ts3.23.1901.2014.tmp.dat.nc')
cfg.PATHS['cru_dir'] = os.path.dirname(cru_dir)
with warnings.catch_warnings():
# There is a warning from salem
warnings.simplefilter("ignore")
workflow.execute_entity_task(tasks.distribute_cru_style, gdirs)
tasks.compute_ref_t_stars(gdirs)
tasks.distribute_t_stars(gdirs)
# Use histalp for the actual test
cfg.PARAMS['temp_use_local_gradient'] = True
cfg.PATHS['climate_file'] = get_demo_file('HISTALP_oetztal.nc')
cfg.PATHS['cru_dir'] = '~'
workflow.climate_tasks(gdirs)
# Inversion
workflow.inversion_tasks(gdirs)
return gdirs
def up_to_inversion(reset=False):
"""Run the tasks you want."""
# test directory
if not os.path.exists(TEST_DIR):
os.makedirs(TEST_DIR)
if reset:
clean_dir(TEST_DIR)
# Init
cfg.initialize()
# Use multiprocessing
cfg.PARAMS['use_multiprocessing'] = not ON_TRAVIS
# Working dir
cfg.PATHS['working_dir'] = TEST_DIR
cfg.set_divides_db(get_demo_file('HEF_divided.shp'))
cfg.PATHS['dem_file'] = get_demo_file('srtm_oeztal.tif')
# Set up the paths and other stuffs
cfg.set_divides_db(get_demo_file('HEF_divided.shp'))
cfg.PATHS['wgms_rgi_links'] = get_demo_file('RGI_WGMS_oeztal.csv')
cfg.PATHS['glathida_rgi_links'] = get_demo_file('RGI_GLATHIDA_oeztal.csv')
# Read in the RGI file
rgi_file = get_demo_file('rgi_oeztal.shp')
rgidf = gpd.GeoDataFrame.from_file(rgi_file)
# Params
cfg.PARAMS['border'] = 70
cfg.PARAMS['use_inversion_params'] = True
# Go
gdirs = workflow.init_glacier_regions(rgidf)
try:
flowline.init_present_time_glacier(gdirs[0])
except Exception:
reset = True
if reset:
# First preprocessing tasks
workflow.gis_prepro_tasks(gdirs)
# Climate related tasks
# See if CRU is running
cfg.PARAMS['temp_use_local_gradient'] = False
cfg.PATHS['climate_file'] = '~'
cru_dir = get_demo_file('cru_ts3.23.1901.2014.tmp.dat.nc')
cfg.PATHS['cru_dir'] = os.path.dirname(cru_dir)
workflow.climate_tasks(gdirs)
# Use histalp for the actual test
cfg.PARAMS['temp_use_local_gradient'] = True
cfg.PATHS['climate_file'] = get_demo_file('HISTALP_oeztal.nc')
cfg.PATHS['cru_dir'] = '~'
workflow.climate_tasks(gdirs)
# Inversion
workflow.inversion_tasks(gdirs)
return gdirs
def init_hef(reset=False, border=40, invert_with_sliding=True,
invert_with_rectangular=True):
from oggm.core.preprocessing import gis, centerlines, geometry
from oggm.core.preprocessing import climate, inversion
import oggm
import oggm.cfg as cfg
from oggm.utils import get_demo_file
# test directory
testdir = os.path.join(cfg.PATHS['test_dir'], 'tmp_border{}'.format(border))
if not invert_with_sliding:
testdir += '_withoutslide'
if not invert_with_rectangular:
testdir += '_withoutrectangular'
if not os.path.exists(testdir):
os.makedirs(testdir)
reset = True
# Init
cfg.initialize()
cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif')
cfg.PATHS['climate_file'] = get_demo_file('histalp_merged_hef.nc')
cfg.PARAMS['border'] = border
cfg.PARAMS['use_optimized_inversion_params'] = True
hef_file = get_demo_file('Hintereisferner_RGI5.shp')
entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0]
gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=reset)
if not gdir.has_file('inversion_params'):
reset = True
gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=reset)
if not reset:
return gdir
gis.define_glacier_region(gdir, entity=entity)
gis.glacier_masks(gdir)
centerlines.compute_centerlines(gdir)
centerlines.compute_downstream_lines(gdir)
geometry.initialize_flowlines(gdir)
centerlines.compute_downstream_bedshape(gdir)
geometry.catchment_area(gdir)
geometry.catchment_intersections(gdir)
geometry.catchment_width_geom(gdir)
geometry.catchment_width_correction(gdir)
climate.process_histalp_nonparallel([gdir])
climate.mu_candidates(gdir, div_id=0)
mbdf = gdir.get_ref_mb_data()['ANNUAL_BALANCE']
res = climate.t_star_from_refmb(gdir, mbdf)
climate.local_mustar_apparent_mb(gdir, tstar=res['t_star'][-1],
bias=res['bias'][-1],
prcp_fac=res['prcp_fac'])
inversion.prepare_for_inversion(gdir, add_debug_var=True,
invert_with_rectangular=invert_with_rectangular)
ref_v = 0.573 * 1e9
if invert_with_sliding:
def to_optimize(x):
# For backwards compat
_fd = 1.9e-24 * x[0]
glen_a = (cfg.N+2) * _fd / 2.
fs = 5.7e-20 * x[1]
v, _ = inversion.mass_conservation_inversion(gdir, fs=fs,
glen_a=glen_a)
return (v - ref_v)**2
out = optimization.minimize(to_optimize, [1, 1],
bounds=((0.01, 10), (0.01, 10)),
tol=1e-4)['x']
_fd = 1.9e-24 * out[0]
glen_a = (cfg.N+2) * _fd / 2.
fs = 5.7e-20 * out[1]
v, _ = inversion.mass_conservation_inversion(gdir, fs=fs,
glen_a=glen_a,
write=True)
else:
def to_optimize(x):
glen_a = cfg.A * x[0]
v, _ = inversion.mass_conservation_inversion(gdir, fs=0.,
glen_a=glen_a)
return (v - ref_v)**2
out = optimization.minimize(to_optimize, [1],
bounds=((0.01, 10),),
tol=1e-4)['x']
glen_a = cfg.A * out[0]
fs = 0.
v, _ = inversion.mass_conservation_inversion(gdir, fs=fs,
glen_a=glen_a,
write=True)
d = dict(fs=fs, glen_a=glen_a)
d['factor_glen_a'] = out[0]
try:
d['factor_fs'] = out[1]
except IndexError:
d['factor_fs'] = 0.
gdir.write_pickle(d, 'inversion_params')
# filter
inversion.filter_inversion_output(gdir)
inversion.distribute_thickness(gdir, how='per_altitude',
add_nc_name=True)
inversion.distribute_thickness(gdir, how='per_interpolation',
add_slope=False, smooth=False,
add_nc_name=True)
return gdir
def compare(rgi_id, glacier_name):
"""
:param rgi_id:
:param glacier_name:
:return:
"""
# ---------------------
# PREPROCESSING TASKS
# ---------------------
# create test directory
wdir = os.path.join(os.path.abspath('.'), 'comparison_wdir')
if not os.path.exists(wdir):
os.makedirs(wdir)
shutil.rmtree(wdir)
os.makedirs(wdir)
# load default parameter file
cfg.initialize()
# RGI entity
# get/downlaod the rgi entity including the outline shapefile
rgi_df = utils.get_rgi_glacier_entities([rgi_id])
# set name, since not delivered with RGI
if rgi_df.loc[int(rgi_id[-5:])-1, 'Name'] is None:
rgi_df.loc[int(rgi_id[-5:])-1, 'Name'] = glacier_name
# select single entry
rgi_entity = rgi_df.iloc[0]
# GlacierDirectory
# specify the working directory and define the glacier directory
cfg.PATHS['working_dir'] = wdir
gdir = oggm.GlacierDirectory(rgi_entity)
# DEM and GIS tasks
# get the path to the DEM file (will download if necessary)
dem = utils.get_topo_file(gdir.cenlon, gdir.cenlat)
# set path in config file
cfg.PATHS['dem_file'] = dem[0][0]
cfg.PARAMS['border'] = 10
cfg.PARAMS['use_intersects'] = False
# run GIS tasks
gis.define_glacier_region(gdir, entity=rgi_entity)
gis.glacier_masks(gdir)
# Climate data
# using HistAlp
cfg.PARAMS['baseline_climate'] = 'HISTALP'
# climate records before 1850 are hardly reliable, which is not so drastic for
# qualitative experiments (could be driven with random climate anyway)
# cfg.PARAMS['baseline_y0'] = 1850
# change hyper parameters for HistAlp
cfg.PARAMS['prcp_scaling_factor'] = 1.75
cfg.PARAMS['temp_melt'] = -1.75
# run climate task
climate.process_histalp_data(gdir)
# run center line preprocessing tasks
centerlines.compute_centerlines(gdir)
centerlines.initialize_flowlines(gdir)
centerlines.compute_downstream_line(gdir)
centerlines.compute_downstream_bedshape(gdir)
centerlines.catchment_area(gdir)
centerlines.catchment_intersections(gdir)
centerlines.catchment_width_geom(gdir)
centerlines.catchment_width_correction(gdir)
# --------------------
# SCALING MODEL
# --------------------
# compute local t* and the corresponding mu*
vascaling.local_t_star(gdir)
# instance the mass balance models
vas_mb_mod = vascaling.VAScalingMassBalance(gdir)
# get reference area
a0 = gdir.rgi_area_m2
# get reference year
y0 = gdir.read_pickle('climate_info')['baseline_hydro_yr_0']
y1 = gdir.read_pickle('climate_info')['baseline_hydro_yr_1']
# get min and max glacier surface elevation
h0, h1 = vascaling.get_min_max_elevation(gdir)
# instance VAS model
vas_model = vascaling.VAScalingModel(year_0=y0, area_m2_0=a0,
min_hgt=h0, max_hgt=h1,
mb_model=vas_mb_mod)
# run model over all HistAlp climate period
vas_df = vas_model.run_and_store(y1, reset=True)
# get relevant parameters
years_vas = vas_df.index.values
length_m_vas = vas_df.length_m.values
area_m2_vas = vas_df.area_m2.values
volume_m3_vas = vas_df.volume_m3.values
# ------
# OGGM
# ------
# compute local t* and the corresponding mu*
climate.local_t_star(gdir)
climate.mu_star_calibration(gdir)
# instance the mass balance models
mb_mod = massbalance.PastMassBalance(gdir)
# run inversion tasks
inversion.prepare_for_inversion(gdir)
inversion.mass_conservation_inversion(gdir)
inversion.filter_inversion_output(gdir)
# initialize present time glacier
flowline.init_present_time_glacier(gdir)
# instance flowline model
fls = gdir.read_pickle('model_flowlines')
y0 = gdir.read_pickle('climate_info')['baseline_hydro_yr_0']
y1 = gdir.read_pickle('climate_info')['baseline_hydro_yr_1']
fl_mod = flowline.FluxBasedModel(flowlines=fls, mb_model=mb_mod, y0=y0)
# run model and store output as xarray data set
_, oggm_ds = fl_mod.run_until_and_store(y1)
years_oggm = oggm_ds.hydro_year.values
# annual index must be equal
np.testing.assert_array_equal(years_oggm, years_vas)
length_m_oggm = oggm_ds.length_m.values
area_m2_oggm = oggm_ds.area_m2.values
volume_m3_oggm = oggm_ds.volume_m3.values
# define column names for DataFrame
names = ['length_vas', 'length_oggm',
'area_vas', 'area_oggm',
'volume_vas', 'volume_oggm']
# combine glacier geometries into DataFrame
df = pd.DataFrame(np.array([length_m_vas, length_m_oggm,
area_m2_vas, area_m2_oggm,
volume_m3_vas, volume_m3_oggm]).T,
index=years_vas, columns=names)
# save to file
store = True
if store:
# define path and file names
folder = '/Users/oberrauch/work/master/data/'
df.to_csv(folder+'run_comparison.csv')
def plot_both(vas_df, oggm_df, ref=None, correct_bias=False,
title='', ylabel='', file_path='', exp=0):
""" Plot geometric parameters of both models.
If a `file_path` is given, the figure will be saved.
:param vas_df: (pandas.Series) geometric glacier parameter of the VAS model
:param oggm_df: (pandas.Series) geometric glacier parameter of the OGGM
:param ref: (pandas.Series) measured glacier parameter, optional
:param title: (string) figure title, optional
:param ylabel: (string) label for y-axis, optional
:param file_path: (string) where to store the figure, optional
:param exp: (int) exponent for labels in scientific notation, optional
"""
beamer = True
if beamer:
mpl.rc('axes', titlesize=18)
mpl.rc('axes', labelsize=14)
mpl.rc('xtick', labelsize=14)
mpl.rc('ytick', labelsize=14)
mpl.rc('legend', fontsize=10)
# create figure and first axes
fig = plt.figure(figsize=[6, 4])
ax = fig.add_axes([0.15, 0.1, 0.8, 0.8])
# define colors
c1 = 'C0'
c2 = 'C1'
c3 = 'C3'
# plot vas and OGGM parameters
ax.plot(oggm_df.index, oggm_df.values, c=c2, label='OGGM')
ax.plot(vas_df.index, vas_df.values, c=c1, label='VAS')
if ref:
# plot reference parameter if given
ax.plot(ref.index, ref.values, c=c3, label='measurements')
if correct_bias:
# plot bias corrected vas
df_ = pd.DataFrame([oggm_df, vas_df]).T
bias = vas_df.values - df_.mean().diff().iloc[1]
ax.plot(vas_df.index, bias, c=c1, ls='--',
label='VAS, bias corrected')
# add RMSD as text
ax.text(0.05, 0.05,
'RMSD: {:.1e}'.format(utils.rmsd(oggm_df, bias)),
transform=plt.gca().transAxes)
# add correlation coefficient as text
ax.text(0.05, 0.11, 'Corr. Coef.: {:.2f}'.format(
utils.corrcoef(oggm_df, vas_df)),
transform=plt.gca().transAxes)
# add title, labels, legend
ax.set_title(title)
ax.set_ylabel(ylabel)
ax.legend()
import matplotlib.ticker
class OOMFormatter(matplotlib.ticker.ScalarFormatter):
def __init__(self, order=0, fformat="%1.1f", offset=False, mathText=False):
self.oom = order
self.fformat = fformat
matplotlib.ticker.ScalarFormatter.__init__(self, useOffset=offset, useMathText=mathText)
def _set_orderOfMagnitude(self, nothing):
self.orderOfMagnitude = self.oom
def _set_format(self, vmin, vmax):
self.format = self.fformat
if self._useMathText:
self.format = '$%s$' % matplotlib.ticker._mathdefault(self.format)
# use scientific notation with fixed exponent according
ax.yaxis.set_major_formatter(OOMFormatter(exp, "%1.2f"))
# store to file
if file_path:
plt.savefig(file_path, bbox_inches='tight',
format=file_path.split('.')[-1])
# specify plot directory
folder = '/Users/oberrauch/work/master/plots/'
# plot length
plot_both(df.length_vas, df.length_oggm, correct_bias=True,
title='Glacier length - {}'.format(glacier_name),
ylabel=r'Length [m]',
file_path=os.path.join(folder, '{}_length.pdf'.format(rgi_id)),
exp=3)
# plot area
plot_both(df.area_vas, df.area_oggm, correct_bias=True,
title='Surface area - {}'.format(glacier_name),
ylabel=r'Area [m$^2$]',
file_path=os.path.join(folder, '{}_area.pdf'.format(rgi_id)),
exp=6)
# plot volume
plot_both(df.volume_vas, df.volume_oggm, correct_bias=True,
title='Glacier volume - {}'.format(glacier_name),
ylabel=r'Volume [m$^3$]',
file_path=os.path.join(folder, '{}_volume.pdf'.format(rgi_id)),
exp=9)
def init_hef(reset=False, border=40, invert_with_sliding=True):
# test directory
testdir = TESTDIR_BASE + '_border{}'.format(border)
if not invert_with_sliding:
testdir += '_withoutslide'
if not os.path.exists(testdir):
os.makedirs(testdir)
reset = True
if not os.path.exists(os.path.join(testdir, 'RGI40-11.00897')):
reset = True
if not os.path.exists(os.path.join(testdir, 'RGI40-11.00897',
'inversion_params.pkl')):
reset = True
# Init
cfg.initialize()
cfg.set_divides_db(get_demo_file('HEF_divided.shp'))
cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif')
cfg.PATHS['climate_file'] = get_demo_file('histalp_merged_hef.nc')
cfg.PARAMS['border'] = border
# loop because for some reason indexing wont work
hef_file = get_demo_file('Hintereisferner.shp')
rgidf = gpd.GeoDataFrame.from_file(hef_file)
for index, entity in rgidf.iterrows():
gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=reset)
if not reset:
return gdir
gis.define_glacier_region(gdir, entity=entity)
gis.glacier_masks(gdir)
centerlines.compute_centerlines(gdir)
centerlines.compute_downstream_lines(gdir)
geometry.initialize_flowlines(gdir)
geometry.catchment_area(gdir)
geometry.catchment_width_geom(gdir)
geometry.catchment_width_correction(gdir)
climate.distribute_climate_data([gdir])
climate.mu_candidates(gdir, div_id=0)
hef_file = get_demo_file('mbdata_RGI40-11.00897.csv')
mbdf = pd.read_csv(hef_file).set_index('YEAR')
t_star, bias = climate.t_star_from_refmb(gdir, mbdf['ANNUAL_BALANCE'])
climate.local_mustar_apparent_mb(gdir, tstar=t_star[-1], bias=bias[-1])
inversion.prepare_for_inversion(gdir)
ref_v = 0.573 * 1e9
if invert_with_sliding:
def to_optimize(x):
# For backwards compat
_fd = 1.9e-24 * x[0]
glen_a = (cfg.N+2) * _fd / 2.
fs = 5.7e-20 * x[1]
v, _ = inversion.invert_parabolic_bed(gdir, fs=fs,
glen_a=glen_a)
return (v - ref_v)**2
import scipy.optimize as optimization
out = optimization.minimize(to_optimize, [1, 1],
bounds=((0.01, 10), (0.01, 10)),
tol=1e-4)['x']
_fd = 1.9e-24 * out[0]
glen_a = (cfg.N+2) * _fd / 2.
fs = 5.7e-20 * out[1]
v, _ = inversion.invert_parabolic_bed(gdir, fs=fs,
glen_a=glen_a,
write=True)
else:
def to_optimize(x):
glen_a = cfg.A * x[0]
v, _ = inversion.invert_parabolic_bed(gdir, fs=0.,
glen_a=glen_a)
return (v - ref_v)**2
import scipy.optimize as optimization
out = optimization.minimize(to_optimize, [1],
bounds=((0.01, 10),),
tol=1e-4)['x']
glen_a = cfg.A * out[0]
fs = 0.
v, _ = inversion.invert_parabolic_bed(gdir, fs=fs,
glen_a=glen_a,
write=True)
d = dict(fs=fs, glen_a=glen_a)
d['factor_glen_a'] = out[0]
try:
d['factor_fs'] = out[1]
except IndexError:
d['factor_fs'] = 0.
gdir.write_pickle(d, 'inversion_params')
inversion.distribute_thickness(gdir, how='per_altitude',
add_nc_name=True)
inversion.distribute_thickness(gdir, how='per_interpolation',
add_slope=False, smooth=False,
add_nc_name=True)
return gdir
def init_hef(reset=False, border=40, invert_with_sliding=True):
from oggm.core.preprocessing import gis, centerlines, geometry
from oggm.core.preprocessing import climate, inversion
import oggm
import oggm.cfg as cfg
from oggm.utils import get_demo_file
# test directory
testdir = TESTDIR_BASE + '_border{}'.format(border)
if not invert_with_sliding:
testdir += '_withoutslide'
if not os.path.exists(testdir):
os.makedirs(testdir)
reset = True
if not os.path.exists(os.path.join(testdir, 'RGI40-11.00897')):
reset = True
if not os.path.exists(os.path.join(testdir, 'RGI40-11.00897',
'inversion_params.pkl')):
reset = True
# Init
cfg.initialize()
cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif')
cfg.PATHS['climate_file'] = get_demo_file('histalp_merged_hef.nc')
cfg.PARAMS['border'] = border
hef_file = get_demo_file('Hintereisferner.shp')
entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0]
gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=reset)
if not reset:
return gdir
gis.define_glacier_region(gdir, entity=entity)
gis.glacier_masks(gdir)
centerlines.compute_centerlines(gdir)
centerlines.compute_downstream_lines(gdir)
geometry.initialize_flowlines(gdir)
geometry.catchment_area(gdir)
geometry.catchment_width_geom(gdir)
geometry.catchment_width_correction(gdir)
climate.process_histalp_nonparallel([gdir])
climate.mu_candidates(gdir, div_id=0)
mbdf = gdir.get_ref_mb_data()['ANNUAL_BALANCE']
res = climate.t_star_from_refmb(gdir, mbdf)
climate.local_mustar_apparent_mb(gdir, tstar=res['t_star'][-1],
bias=res['bias'][-1],
prcp_fac=res['prcp_fac'])
inversion.prepare_for_inversion(gdir)
ref_v = 0.573 * 1e9
if invert_with_sliding:
def to_optimize(x):
# For backwards compat
_fd = 1.9e-24 * x[0]
glen_a = (cfg.N+2) * _fd / 2.
fs = 5.7e-20 * x[1]
v, _ = inversion.invert_parabolic_bed(gdir, fs=fs,
glen_a=glen_a)
return (v - ref_v)**2
out = optimization.minimize(to_optimize, [1, 1],
bounds=((0.01, 10), (0.01, 10)),
tol=1e-4)['x']
_fd = 1.9e-24 * out[0]
glen_a = (cfg.N+2) * _fd / 2.
fs = 5.7e-20 * out[1]
v, _ = inversion.invert_parabolic_bed(gdir, fs=fs,
glen_a=glen_a,
write=True)
else:
def to_optimize(x):
glen_a = cfg.A * x[0]
v, _ = inversion.invert_parabolic_bed(gdir, fs=0.,
glen_a=glen_a)
return (v - ref_v)**2
out = optimization.minimize(to_optimize, [1],
bounds=((0.01, 10),),
tol=1e-4)['x']
glen_a = cfg.A * out[0]
fs = 0.
v, _ = inversion.invert_parabolic_bed(gdir, fs=fs,
glen_a=glen_a,
write=True)
d = dict(fs=fs, glen_a=glen_a)
d['factor_glen_a'] = out[0]
try:
d['factor_fs'] = out[1]
except IndexError:
d['factor_fs'] = 0.
gdir.write_pickle(d, 'inversion_params')
inversion.distribute_thickness(gdir, how='per_altitude',
add_nc_name=True)
inversion.distribute_thickness(gdir, how='per_interpolation',
add_slope=False, smooth=False,
add_nc_name=True)
return gdir
def run_benchmark(rgi_version=None, rgi_reg=None, border=None,
output_folder='', working_dir='', is_test=False,
test_rgidf=None, test_intersects_file=None,
test_topofile=None, test_crudir=None):
"""Does the actual job.
Parameters
----------
rgi_version : str
the RGI version to use (defaults to cfg.PARAMS)
rgi_reg : str
the RGI region to process
border : int
the number of pixels at the maps border
output_folder : str
path to the output folder (where to put the preprocessed tar files)
working_dir : str
path to the OGGM working directory
is_test : bool
to test on a couple of glaciers only!
test_rgidf : shapefile
for testing purposes only
test_intersects_file : shapefile
for testing purposes only
test_topofile : str
for testing purposes only
test_crudir : str
for testing purposes only
"""
# TODO: temporarily silence Fiona deprecation warnings
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
# Module logger
log = logging.getLogger(__name__)
# Initialize OGGM and set up the run parameters
cfg.initialize(logging_level='WORKFLOW')
# Local paths
utils.mkdir(working_dir)
cfg.PATHS['working_dir'] = working_dir
# Use multiprocessing?
cfg.PARAMS['use_multiprocessing'] = True
# How many grid points around the glacier?
# Make it large if you expect your glaciers to grow large
cfg.PARAMS['border'] = border
# Set to True for operational runs
cfg.PARAMS['continue_on_error'] = True
# For statistics
odf = pd.DataFrame()
if rgi_version is None:
rgi_version = cfg.PARAMS['rgi_version']
base_dir = os.path.join(output_folder)
# Add a package version file
utils.mkdir(base_dir)
opath = os.path.join(base_dir, 'package_versions.txt')
with open(opath, 'w') as vfile:
vfile.write(utils.show_versions(logger=log))
# Read RGI
start = time.time()
if test_rgidf is None:
# Get the RGI file
rgidf = gpd.read_file(utils.get_rgi_region_file(rgi_reg,
version=rgi_version))
# We use intersects
rgif = utils.get_rgi_intersects_region_file(rgi_reg,
version=rgi_version)
cfg.set_intersects_db(rgif)
else:
rgidf = test_rgidf
cfg.set_intersects_db(test_intersects_file)
if is_test:
# Just for fun
rgidf = rgidf.sample(2)
_add_time_to_df(odf, 'Read RGI', time.time()-start)
# Sort for more efficient parallel computing
rgidf = rgidf.sort_values('Area', ascending=False)
log.workflow('Starting prepro run for RGI reg: {} '
'and border: {}'.format(rgi_reg, border))
log.workflow('Number of glaciers: {}'.format(len(rgidf)))
# Input
if test_topofile:
cfg.PATHS['dem_file'] = test_topofile
# Initialize working directories
start = time.time()
gdirs = workflow.init_glacier_regions(rgidf, reset=True, force=True)
_add_time_to_df(odf, 'init_glacier_regions', time.time()-start)
# Pre-download other files just in case
if test_crudir is None:
_ = utils.get_cru_file(var='tmp')
_ = utils.get_cru_file(var='pre')
else:
cfg.PATHS['cru_dir'] = test_crudir
# Tasks
task_list = [
tasks.process_cru_data,
tasks.glacier_masks,
tasks.compute_centerlines,
tasks.initialize_flowlines,
tasks.compute_downstream_line,
tasks.compute_downstream_bedshape,
tasks.catchment_area,
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
tasks.local_t_star,
tasks.mu_star_calibration,
tasks.prepare_for_inversion,
tasks.mass_conservation_inversion,
tasks.filter_inversion_output,
tasks.init_present_time_glacier,
]
for task in task_list:
start = time.time()
workflow.execute_entity_task(task, gdirs)
_add_time_to_df(odf, task.__name__, time.time()-start)
# Runs
start = time.time()
workflow.execute_entity_task(tasks.run_random_climate, gdirs,
nyears=250, bias=0, seed=0,
output_filesuffix='_tstar')
_add_time_to_df(odf, 'run_random_climate_tstar_250', time.time()-start)
start = time.time()
workflow.execute_entity_task(tasks.run_random_climate, gdirs,
nyears=250, y0=1995, seed=0,
output_filesuffix='_commit')
_add_time_to_df(odf, 'run_random_climate_commit_250', time.time()-start)
# Compile results
start = time.time()
utils.compile_glacier_statistics(gdirs)
_add_time_to_df(odf, 'compile_glacier_statistics', time.time()-start)
start = time.time()
utils.compile_climate_statistics(gdirs,
add_climate_period=[1920, 1960, 2000])
_add_time_to_df(odf, 'compile_climate_statistics', time.time()-start)
start = time.time()
utils.compile_run_output(gdirs, filesuffix='_tstar')
_add_time_to_df(odf, 'compile_run_output_tstar', time.time()-start)
start = time.time()
utils.compile_run_output(gdirs, filesuffix='_commit')
_add_time_to_df(odf, 'compile_run_output_commit', time.time()-start)
# Log
opath = os.path.join(base_dir, 'benchmarks_b{:03d}.csv'.format(border))
odf.index.name = 'Task'
odf.to_csv(opath)
log.workflow('OGGM benchmarks is done!')
def setUp(self):
cfg.initialize()
cfg.PATHS['dl_cache_dir'] = os.path.join(TEST_DIR, 'dl_cache')
cfg.PATHS['working_dir'] = os.path.join(TEST_DIR, 'wd')
cfg.PATHS['tmp_dir'] = os.path.join(TEST_DIR, 'extract')
self.reset_dir()
import geopandas as gpd import matplotlib.pyplot as plt import salem # Locals import oggm.cfg as cfg from oggm import workflow from oggm import tasks from oggm.workflow import execute_entity_task from oggm import graphics, utils from oggm.core.models import flowline # Initialize OGGM and set up the run parameters # --------------------------------------------- cfg.initialize() # Local paths (where to write the OGGM run output) WORKING_DIR = '/home/mowglie/disk/OGGM_Runs/EXAMPLE_GLACIERS' PLOTS_DIR = os.path.join(WORKING_DIR, 'plots') utils.mkdir(WORKING_DIR) cfg.PATHS['working_dir'] = WORKING_DIR # Use multiprocessing? cfg.PARAMS['use_multiprocessing'] = True # How many grid points around the glacier? # Make it large if you expect your glaciers to grow large cfg.PARAMS['border'] = 60 # This is the default in OGGM
def run_prepro_levels(rgi_version=None,
rgi_reg=None,
border=None,
output_folder='',
working_dir='',
dem_source='',
is_test=False,
test_ids=None,
demo=False,
test_rgidf=None,
test_intersects_file=None,
test_topofile=None,
disable_mp=False,
params_file=None,
elev_bands=False,
match_geodetic_mb=False,
centerlines_only=False,
add_consensus=False,
start_level=None,
start_base_url=None,
max_level=5,
logging_level='WORKFLOW',
disable_dl_verify=False):
"""Does the actual job.
Parameters
----------
rgi_version : str
the RGI version to use (defaults to cfg.PARAMS)
rgi_reg : str
the RGI region to process
border : int
the number of pixels at the maps border
output_folder : str
path to the output folder (where to put the preprocessed tar files)
dem_source : str
which DEM source to use: default, SOURCE_NAME or ALL
working_dir : str
path to the OGGM working directory
params_file : str
path to the OGGM parameter file (to override defaults)
is_test : bool
to test on a couple of glaciers only!
test_ids : list
if is_test: list of ids to process
demo : bool
to run the prepro for the list of demo glaciers
test_rgidf : shapefile
for testing purposes only
test_intersects_file : shapefile
for testing purposes only
test_topofile : str
for testing purposes only
test_crudir : str
for testing purposes only
disable_mp : bool
disable multiprocessing
elev_bands : bool
compute all flowlines based on the Huss&Hock 2015 method instead
of the OGGM default, which is a mix of elev_bands and centerlines.
centerlines_only : bool
compute all flowlines based on the OGGM centerline(s) method instead
of the OGGM default, which is a mix of elev_bands and centerlines.
match_geodetic_mb : bool
match the regional mass-balance estimates at the regional level
(currently Hugonnet et al., 2020).
add_consensus : bool
adds (reprojects) the consensus estimates thickness to the glacier
directories. With elev_bands=True, the data will also be binned.
start_level : int
the pre-processed level to start from (default is to start from
scratch). If set, you'll need to indicate start_base_url as well.
start_base_url : str
the pre-processed base-url to fetch the data from.
max_level : int
the maximum pre-processing level before stopping
logging_level : str
the logging level to use (DEBUG, INFO, WARNING, WORKFLOW)
disable_dl_verify : bool
disable the hash verification of OGGM downloads
"""
# TODO: temporarily silence Fiona and other deprecation warnings
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
# Input check
if max_level not in [1, 2, 3, 4, 5]:
raise InvalidParamsError('max_level should be one of [1, 2, 3, 4, 5]')
if start_level is not None:
if start_level not in [0, 1, 2]:
raise InvalidParamsError('start_level should be one of [0, 1, 2]')
if start_level > 0 and start_base_url is None:
raise InvalidParamsError('With start_level, please also indicate '
'start_base_url')
else:
start_level = 0
# Time
start = time.time()
def _time_log():
# Log util
m, s = divmod(time.time() - start, 60)
h, m = divmod(m, 60)
log.workflow('OGGM prepro_levels is done! Time needed: '
'{:02d}:{:02d}:{:02d}'.format(int(h), int(m), int(s)))
# Config Override Params
params = {}
# Local paths
utils.mkdir(working_dir)
params['working_dir'] = working_dir
# Initialize OGGM and set up the run parameters
cfg.initialize(file=params_file,
params=params,
logging_level=logging_level,
future=True)
# Use multiprocessing?
cfg.PARAMS['use_multiprocessing'] = not disable_mp
# How many grid points around the glacier?
# Make it large if you expect your glaciers to grow large
cfg.PARAMS['border'] = border
# Set to True for operational runs
cfg.PARAMS['continue_on_error'] = True
# Check for the integrity of the files OGGM downloads at run time
# For large files (e.g. using a 1 tif DEM like ALASKA) calculating the hash
# takes a long time, so deactivating this can make sense
cfg.PARAMS['dl_verify'] = not disable_dl_verify
# Log the parameters
msg = '# OGGM Run parameters:'
for k, v in cfg.PARAMS.items():
if type(v) in [pd.DataFrame, dict]:
continue
msg += '\n {}: {}'.format(k, v)
log.workflow(msg)
if rgi_version is None:
rgi_version = cfg.PARAMS['rgi_version']
output_base_dir = os.path.join(output_folder, 'RGI{}'.format(rgi_version),
'b_{:03d}'.format(border))
# Add a package version file
utils.mkdir(output_base_dir)
opath = os.path.join(output_base_dir, 'package_versions.txt')
with open(opath, 'w') as vfile:
vfile.write(utils.show_versions(logger=log))
if demo:
rgidf = utils.get_rgi_glacier_entities(cfg.DATA['demo_glaciers'].index)
elif test_rgidf is None:
# Get the RGI file
rgidf = gpd.read_file(
utils.get_rgi_region_file(rgi_reg, version=rgi_version))
# We use intersects
rgif = utils.get_rgi_intersects_region_file(rgi_reg,
version=rgi_version)
cfg.set_intersects_db(rgif)
# Some RGI input quality checks - this is based on visual checks
# of large glaciers in the RGI
ids_to_ice_cap = [
'RGI60-05.10315', # huge Greenland ice cap
'RGI60-03.01466', # strange thing next to Devon
'RGI60-09.00918', # Academy of sciences Ice cap
'RGI60-09.00969',
'RGI60-09.00958',
'RGI60-09.00957',
]
rgidf.loc[rgidf.RGIId.isin(ids_to_ice_cap), 'Form'] = '1'
# In AA almost all large ice bodies are actually ice caps
if rgi_reg == '19':
rgidf.loc[rgidf.Area > 100, 'Form'] = '1'
# For greenland we omit connectivity level 2
if rgi_reg == '05':
rgidf = rgidf.loc[rgidf['Connect'] != 2]
else:
rgidf = test_rgidf
cfg.set_intersects_db(test_intersects_file)
if is_test:
if test_ids is not None:
rgidf = rgidf.loc[rgidf.RGIId.isin(test_ids)]
else:
rgidf = rgidf.sample(4)
log.workflow('Starting prepro run for RGI reg: {} '
'and border: {}'.format(rgi_reg, border))
log.workflow('Number of glaciers: {}'.format(len(rgidf)))
# L0 - go
if start_level == 0:
gdirs = workflow.init_glacier_directories(rgidf,
reset=True,
force=True)
# Glacier stats
sum_dir = os.path.join(output_base_dir, 'L0', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir,
'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# L0 OK - compress all in output directory
log.workflow('L0 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L0')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 0:
_time_log()
return
else:
gdirs = workflow.init_glacier_directories(
rgidf,
reset=True,
force=True,
from_prepro_level=start_level,
prepro_border=border,
prepro_rgi_version=rgi_version,
prepro_base_url=start_base_url)
# L1 - Add dem files
if start_level == 0:
if test_topofile:
cfg.PATHS['dem_file'] = test_topofile
# Which DEM source?
if dem_source.upper() == 'ALL':
# This is the complex one, just do the job and leave
log.workflow('Running prepro on ALL sources')
for i, s in enumerate(utils.DEM_SOURCES):
rs = i == 0
log.workflow('Running prepro on sources: {}'.format(s))
gdirs = workflow.init_glacier_directories(rgidf,
reset=rs,
force=rs)
workflow.execute_entity_task(tasks.define_glacier_region,
gdirs,
source=s)
workflow.execute_entity_task(_rename_dem_folder,
gdirs,
source=s)
# make a GeoTiff mask of the glacier, choose any source
workflow.execute_entity_task(gis.rasterio_glacier_mask,
gdirs,
source='ALL')
# Compress all in output directory
level_base_dir = os.path.join(output_base_dir, 'L1')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
_time_log()
return
# Force a given source
source = dem_source.upper() if dem_source else None
# L1 - go
workflow.execute_entity_task(tasks.define_glacier_region,
gdirs,
source=source)
# Glacier stats
sum_dir = os.path.join(output_base_dir, 'L1', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir,
'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# L1 OK - compress all in output directory
log.workflow('L1 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L1')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 1:
_time_log()
return
# L2 - Tasks
if start_level <= 1:
# Check which glaciers will be processed as what
if elev_bands:
gdirs_band = gdirs
gdirs_cent = []
elif centerlines_only:
gdirs_band = []
gdirs_cent = gdirs
else:
# Default is to centerlines_only, but it used to be a mix
# (e.g. bands for ice caps, etc)
# I still keep this logic here in case we want to mix again
gdirs_band = []
gdirs_cent = gdirs
log.workflow('Start flowline processing with: '
'N centerline type: {}, '
'N elev bands type: {}.'
''.format(len(gdirs_cent), len(gdirs_band)))
# HH2015 method
workflow.execute_entity_task(tasks.simple_glacier_masks, gdirs_band)
# Centerlines OGGM
workflow.execute_entity_task(tasks.glacier_masks, gdirs_cent)
if add_consensus:
from oggm.shop.bedtopo import add_consensus_thickness
workflow.execute_entity_task(add_consensus_thickness, gdirs_band)
workflow.execute_entity_task(add_consensus_thickness, gdirs_cent)
# Elev bands with var data
vn = 'consensus_ice_thickness'
workflow.execute_entity_task(tasks.elevation_band_flowline,
gdirs_band,
bin_variables=vn)
workflow.execute_entity_task(
tasks.fixed_dx_elevation_band_flowline,
gdirs_band,
bin_variables=vn)
else:
# HH2015 method without it
task_list = [
tasks.elevation_band_flowline,
tasks.fixed_dx_elevation_band_flowline,
]
for task in task_list:
workflow.execute_entity_task(task, gdirs_band)
# Centerlines OGGM
task_list = [
tasks.compute_centerlines,
tasks.initialize_flowlines,
tasks.catchment_area,
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
]
for task in task_list:
workflow.execute_entity_task(task, gdirs_cent)
# Same for all glaciers
if border >= 20:
task_list = [
tasks.compute_downstream_line,
tasks.compute_downstream_bedshape,
]
for task in task_list:
workflow.execute_entity_task(task, gdirs)
else:
log.workflow('L2: for map border values < 20, wont compute '
'downstream lines.')
# Glacier stats
sum_dir = os.path.join(output_base_dir, 'L2', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir,
'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# L2 OK - compress all in output directory
log.workflow('L2 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L2')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 2:
_time_log()
return
# L3 - Tasks
# Climate
workflow.execute_entity_task(tasks.process_climate_data, gdirs)
if cfg.PARAMS['climate_qc_months'] > 0:
workflow.execute_entity_task(tasks.historical_climate_qc, gdirs)
workflow.execute_entity_task(tasks.local_t_star, gdirs)
workflow.execute_entity_task(tasks.mu_star_calibration, gdirs)
# Inversion: we match the consensus
workflow.calibrate_inversion_from_consensus(gdirs,
apply_fs_on_mismatch=True,
error_on_mismatch=False)
# Do we want to match geodetic estimates?
# This affects only the bias so we can actually do this *after*
# the inversion, but we really want to take calving into account here
if match_geodetic_mb:
opath = os.path.join(
sum_dir, 'fixed_geometry_mass_balance_'
'before_match_{}.csv'.format(rgi_reg))
utils.compile_fixed_geometry_mass_balance(gdirs, path=opath)
workflow.match_regional_geodetic_mb(gdirs, rgi_reg)
# We get ready for modelling
if border >= 20:
workflow.execute_entity_task(tasks.init_present_time_glacier, gdirs)
else:
log.workflow('L3: for map border values < 20, wont initalize glaciers '
'for the run.')
# Glacier stats
sum_dir = os.path.join(output_base_dir, 'L3', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
opath = os.path.join(sum_dir, 'climate_statistics_{}.csv'.format(rgi_reg))
utils.compile_climate_statistics(gdirs, path=opath)
opath = os.path.join(sum_dir,
'fixed_geometry_mass_balance_{}.csv'.format(rgi_reg))
utils.compile_fixed_geometry_mass_balance(gdirs, path=opath)
# L3 OK - compress all in output directory
log.workflow('L3 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L3')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 3:
_time_log()
return
if border < 20:
log.workflow('L3: for map border values < 20, wont compute L4 and L5.')
_time_log()
return
# L4 - No tasks: add some stats for consistency and make the dirs small
sum_dir_L3 = sum_dir
sum_dir = os.path.join(output_base_dir, 'L4', 'summary')
utils.mkdir(sum_dir)
# Copy L3 files for consistency
for bn in [
'glacier_statistics', 'climate_statistics',
'fixed_geometry_mass_balance'
]:
ipath = os.path.join(sum_dir_L3, bn + '_{}.csv'.format(rgi_reg))
opath = os.path.join(sum_dir, bn + '_{}.csv'.format(rgi_reg))
shutil.copyfile(ipath, opath)
# Copy mini data to new dir
mini_base_dir = os.path.join(working_dir, 'mini_perglacier')
mini_gdirs = workflow.execute_entity_task(tasks.copy_to_basedir,
gdirs,
base_dir=mini_base_dir)
# L4 OK - compress all in output directory
log.workflow('L4 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L4')
workflow.execute_entity_task(utils.gdir_to_tar,
mini_gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
if max_level == 4:
_time_log()
return
# L5 - spinup run in mini gdirs
gdirs = mini_gdirs
# Get end date. The first gdir might have blown up, try some others
i = 0
while True:
if i >= len(gdirs):
raise RuntimeError('Found no valid glaciers!')
try:
y0 = gdirs[i].get_climate_info()['baseline_hydro_yr_0']
# One adds 1 because the run ends at the end of the year
ye = gdirs[i].get_climate_info()['baseline_hydro_yr_1'] + 1
break
except BaseException:
i += 1
# OK - run
workflow.execute_entity_task(tasks.run_from_climate_data,
gdirs,
min_ys=y0,
ye=ye,
output_filesuffix='_historical')
# Now compile the output
sum_dir = os.path.join(output_base_dir, 'L5', 'summary')
utils.mkdir(sum_dir)
opath = os.path.join(sum_dir,
'historical_run_output_{}.nc'.format(rgi_reg))
utils.compile_run_output(gdirs, path=opath, input_filesuffix='_historical')
# Glacier statistics we recompute here for error analysis
opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
utils.compile_glacier_statistics(gdirs, path=opath)
# Other stats for consistency
for bn in ['climate_statistics', 'fixed_geometry_mass_balance']:
ipath = os.path.join(sum_dir_L3, bn + '_{}.csv'.format(rgi_reg))
opath = os.path.join(sum_dir, bn + '_{}.csv'.format(rgi_reg))
shutil.copyfile(ipath, opath)
# Add the extended files
pf = os.path.join(sum_dir, 'historical_run_output_{}.nc'.format(rgi_reg))
mf = os.path.join(sum_dir,
'fixed_geometry_mass_balance_{}.csv'.format(rgi_reg))
# This is crucial - extending calving only possible with L3 data!!!
sf = os.path.join(sum_dir_L3, 'glacier_statistics_{}.csv'.format(rgi_reg))
opath = os.path.join(
sum_dir, 'historical_run_output_extended_{}.nc'.format(rgi_reg))
utils.extend_past_climate_run(past_run_file=pf,
fixed_geometry_mb_file=mf,
glacier_statistics_file=sf,
path=opath)
# L5 OK - compress all in output directory
log.workflow('L5 done. Writing to tar...')
level_base_dir = os.path.join(output_base_dir, 'L5')
workflow.execute_entity_task(utils.gdir_to_tar,
gdirs,
delete=False,
base_dir=level_base_dir)
utils.base_dir_to_tar(level_base_dir)
_time_log()
