def up_to_distrib(reset=False):
# for cross val basically
gdirs = up_to_climate(reset=reset)
with open(CLI_LOGF, 'rb') as f:
clilog = pickle.load(f)
if clilog != 'cru':
reset = True
else:
try:
tasks.compute_ref_t_stars(gdirs)
except Exception:
reset = True
if reset:
# Use CRU
cfg.PARAMS['prcp_scaling_factor'] = 2.5
cfg.PARAMS['temp_use_local_gradient'] = False
cfg.PATHS['climate_file'] = ''
cru_dir = get_demo_file('cru_ts3.23.1901.2014.tmp.dat.nc')
cfg.PATHS['cru_dir'] = os.path.dirname(cru_dir)
with warnings.catch_warnings():
# There is a warning from salem
warnings.simplefilter("ignore")
workflow.execute_entity_task(tasks.process_cru_data, gdirs)
tasks.compute_ref_t_stars(gdirs)
tasks.distribute_t_stars(gdirs)
workflow.execute_entity_task(tasks.apparent_mb, gdirs)
with open(CLI_LOGF, 'wb') as f:
pickle.dump('cru', f)
return gdirs
def climate_tasks(gdirs):
"""Prepare the climate data."""
# Only global tasks
tasks.distribute_climate_data(gdirs)
tasks.compute_ref_t_stars(gdirs)
tasks.distribute_t_stars(gdirs)
def climate_tasks(gdirs):
"""Helper function: run all climate tasks."""
# If not iterable it's ok
try:
len(gdirs)
except TypeError:
gdirs = [gdirs]
# I don't know where this logic is best placed...
if (('climate_file' in cfg.PATHS)
and os.path.exists(cfg.PATHS['climate_file'])):
_process_task = tasks.process_custom_climate_data
else:
# OK, so use the default CRU "high-resolution" method
_process_task = tasks.process_cru_data
execute_entity_task(_process_task, gdirs)
# Then, global tasks
if cfg.PARAMS['run_mb_calibration']:
tasks.compute_ref_t_stars(gdirs)
tasks.distribute_t_stars(gdirs)
# And the apparent mass-balance
execute_entity_task(tasks.apparent_mb, gdirs)
def climate_tasks(gdirs):
"""Prepare the climate data."""
# Only global tasks
tasks.distribute_climate_data(gdirs)
tasks.compute_ref_t_stars(gdirs)
tasks.distribute_t_stars(gdirs)
def test_to_tiff(self):
from oggm import tasks
from oggm.workflow import execute_entity_task
from oggm import GlacierDirectory
hef_file = get_demo_file('Hintereisferner_RGI5.shp')
entity = gpd.read_file(hef_file).iloc[0]
gdir = GlacierDirectory(entity, base_dir=self.testdir)
gdir.rgi_id = gdir.rgi_id.replace('50-', '60-')
gis.define_glacier_region(gdir, entity=entity)
gdirs = [gdir]
# Preprocessing tasks
task_list = [
tasks.glacier_masks,
tasks.compute_centerlines,
tasks.initialize_flowlines,
tasks.catchment_area,
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
]
for task in task_list:
execute_entity_task(task, gdirs)
# Climate tasks -- only data IO and tstar interpolation!
execute_entity_task(tasks.process_cru_data, gdirs)
tasks.distribute_t_stars(gdirs)
execute_entity_task(tasks.apparent_mb, gdirs)
# Inversion tasks
execute_entity_task(tasks.prepare_for_inversion, gdirs)
execute_entity_task(tasks.volume_inversion,
gdirs,
glen_a=cfg.A * 3,
fs=0)
execute_entity_task(tasks.distribute_thickness_per_altitude, gdirs)
g2task.oggm_to_g2ti(gdir)
ft = os.path.join(cfg.PATHS['working_dir'], 'final',
'RGI60-{}'.format(gdir.rgi_region))
ft = os.path.join(ft, 'thickness_{}.tif'.format(gdir.rgi_id))
ds = xr.open_rasterio(ft)
tpl_f = os.path.join(g2ti.geometry_dir, gdir.rgi_id[:8], gdir.rgi_id,
'mask.tif')
da = xr.open_rasterio(tpl_f)
np.testing.assert_allclose(ds.sum(), (da * ds).sum(), rtol=0.05)
if do_plot:
import matplotlib.pyplot as plt
ds.plot()
plt.figure()
da.plot()
plt.show()
def climate_tasks(gdirs):
"""Prepare the climate data."""
# I don't know where this logic is best placed...
if ('climate_file' in cfg.PATHS) and \
os.path.exists(cfg.PATHS['climate_file']):
_process_task = tasks.process_custom_climate_data
else:
# OK, so use the default CRU "high-resolution" method
_process_task = tasks.process_cru_data
execute_entity_task(_process_task, gdirs)
# Then, only global tasks
tasks.compute_ref_t_stars(gdirs)
tasks.distribute_t_stars(gdirs)
def climate_tasks(gdirs):
"""Prepare the climate data."""
# I don't know where this logic is best placed...
if ('climate_file' in cfg.PATHS) and \
os.path.exists(cfg.PATHS['climate_file']):
_process_task = tasks.process_custom_climate_data
else:
# OK, so use the default CRU "high-resolution" method
_process_task = tasks.process_cru_data
execute_entity_task(_process_task, gdirs)
# Then, only global tasks
tasks.compute_ref_t_stars(gdirs)
tasks.distribute_t_stars(gdirs)
def test_crossval(self):
gdirs = up_to_distrib()
# before crossval
refmustars = []
for gdir in gdirs:
tdf = pd.read_csv(gdir.get_filepath('local_mustar'))
refmustars.append(tdf['mu_star'].values[0])
tasks.crossval_t_stars(gdirs)
file = os.path.join(cfg.PATHS['working_dir'], 'crossval_tstars.csv')
df = pd.read_csv(file, index_col=0)
# after crossval we need to rerun
tasks.compute_ref_t_stars(gdirs)
tasks.distribute_t_stars(gdirs)
# see if the process didn't brake anything
mustars = []
for gdir in gdirs:
tdf = pd.read_csv(gdir.get_filepath('local_mustar'))
mustars.append(tdf['mu_star'].values[0])
np.testing.assert_allclose(refmustars, mustars)
# make some mb tests
from oggm.core.models.massbalance import PastMassBalanceModel
for rid in df.index:
gdir = [g for g in gdirs if g.rgi_id == rid][0]
h, w = gdir.get_inversion_flowline_hw()
cfg.PARAMS['use_bias_for_run'] = False
mbmod = PastMassBalanceModel(gdir)
mbdf = gdir.get_ref_mb_data()['ANNUAL_BALANCE'].to_frame(
name='ref')
for yr in mbdf.index:
mbdf.loc[yr, 'mine'] = mbmod.get_specific_mb(h, w, year=yr)
mm = mbdf.mean()
np.testing.assert_allclose(df.loc[rid].bias,
mm['mine'] - mm['ref'],
atol=1e-3)
cfg.PARAMS['use_bias_for_run'] = True
mbmod = PastMassBalanceModel(gdir)
mbdf = gdir.get_ref_mb_data()['ANNUAL_BALANCE'].to_frame(
name='ref')
for yr in mbdf.index:
mbdf.loc[yr, 'mine'] = mbmod.get_specific_mb(h, w, year=yr)
mm = mbdf.mean()
np.testing.assert_allclose(mm['mine'], mm['ref'], atol=1e-3)
def calibration(gdirs, xval, major=0):
# Climate tasks
if mbcfg.PARAMS['histalp']:
cfg.PATHS['climate_file'] = mbcfg.PATHS['histalpfile']
execute_entity_task(tasks.process_custom_climate_data, gdirs)
else:
execute_entity_task(tasks.process_cru_data, gdirs)
with utils.DisableLogger():
tasks.compute_ref_t_stars(gdirs)
tasks.distribute_t_stars(gdirs)
execute_entity_task(tasks.apparent_mb, gdirs)
# do the crossvalidation
xval = quick_crossval(gdirs, xval, major=major)
return xval
def test_workflow(self):
# This is a check that the inversion workflow works fine
# Download the RGI file for the run
# Make a new dataframe of those
rgidf = gpd.read_file(get_demo_file('SouthGlacier.shp'))
# Go - initialize working directories
gdirs = workflow.init_glacier_regions(rgidf)
# Preprocessing tasks
task_list = [
tasks.glacier_masks,
tasks.compute_centerlines,
tasks.initialize_flowlines,
tasks.catchment_area,
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
]
for task in task_list:
execute_entity_task(task, gdirs)
# Climate tasks -- only data IO and tstar interpolation!
execute_entity_task(tasks.process_cru_data, gdirs)
tasks.distribute_t_stars(gdirs)
execute_entity_task(tasks.apparent_mb, gdirs)
# Inversion tasks
execute_entity_task(tasks.prepare_for_inversion, gdirs)
# We use the default parameters for this run
execute_entity_task(tasks.volume_inversion, gdirs, glen_a=cfg.A, fs=0)
execute_entity_task(tasks.filter_inversion_output, gdirs)
df = utils.glacier_characteristics(gdirs)
assert df.inv_thickness_m[0] < 100
if do_plot:
import matplotlib.pyplot as plt
from oggm.graphics import plot_inversion
plot_inversion(gdirs)
plt.show()
def test_inversion(self):
# Download the RGI file for the run
# Make a new dataframe of those
rgidf = gpd.read_file(get_demo_file('SouthGlacier.shp'))
# Go - initialize working directories
gdirs = workflow.init_glacier_regions(rgidf)
# Preprocessing tasks
task_list = [
tasks.glacier_masks,
tasks.compute_centerlines,
tasks.initialize_flowlines,
tasks.catchment_area,
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
]
for task in task_list:
execute_entity_task(task, gdirs)
# Climate tasks -- only data IO and tstar interpolation!
execute_entity_task(tasks.process_cru_data, gdirs)
tasks.distribute_t_stars(gdirs)
execute_entity_task(tasks.apparent_mb, gdirs)
# Inversion tasks
execute_entity_task(tasks.prepare_for_inversion, gdirs)
# We use the default parameters for this run
execute_entity_task(tasks.volume_inversion, gdirs, glen_a=cfg.A, fs=0)
execute_entity_task(tasks.distribute_thickness_per_altitude,
gdirs,
varname_suffix='_alt')
execute_entity_task(tasks.distribute_thickness_interp,
gdirs,
varname_suffix='_int')
# Reference data
gdir = gdirs[0]
df = self.get_ref_data(gdir)
with xr.open_dataset(gdir.get_filepath('gridded_data')) as ds:
df['oggm_alt'] = ds.distributed_thickness_alt.isel_points(
x=df['i'], y=df['j'])
df['oggm_int'] = ds.distributed_thickness_int.isel_points(
x=df['i'], y=df['j'])
ds['ref'] = xr.zeros_like(ds.distributed_thickness_int) * np.NaN
ds['ref'].data[df['j'], df['i']] = df['thick']
rmsd_int = ((df.oggm_int - df.thick)**2).mean()**.5
rmsd_alt = ((df.oggm_int - df.thick)**2).mean()**.5
assert rmsd_int < 80
assert rmsd_alt < 80
dfm = df.mean()
np.testing.assert_allclose(dfm.thick, dfm.oggm_int, 50)
np.testing.assert_allclose(dfm.thick, dfm.oggm_alt, 50)
if do_plot:
import matplotlib.pyplot as plt
df.plot(kind='scatter', x='oggm_int', y='thick')
plt.axis('equal')
df.plot(kind='scatter', x='oggm_alt', y='thick')
plt.axis('equal')
f, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(12, 3))
ds.ref.plot(ax=ax1)
ds.distributed_thickness_int.plot(ax=ax2)
ds.distributed_thickness_alt.plot(ax=ax3)
plt.tight_layout()
plt.show()
def test_mb(self):
# This is a function to produce the MB function needed by Anna
# Download the RGI file for the run
# Make a new dataframe of those
rgidf = gpd.read_file(get_demo_file('SouthGlacier.shp'))
# Go - initialize working directories
gdirs = workflow.init_glacier_regions(rgidf)
# Preprocessing tasks
task_list = [
tasks.glacier_masks,
tasks.compute_centerlines,
tasks.initialize_flowlines,
tasks.catchment_area,
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
]
for task in task_list:
execute_entity_task(task, gdirs)
# Climate tasks -- only data IO and tstar interpolation!
execute_entity_task(tasks.process_cru_data, gdirs)
tasks.distribute_t_stars(gdirs)
execute_entity_task(tasks.apparent_mb, gdirs)
mbref = salem.GeoTiff(get_demo_file('mb_SouthGlacier.tif'))
demref = salem.GeoTiff(get_demo_file('dem_SouthGlacier.tif'))
mbref = mbref.get_vardata()
mbref[mbref == -9999] = np.NaN
demref = demref.get_vardata()[np.isfinite(mbref)]
mbref = mbref[np.isfinite(mbref)] * 1000
# compute the bias to make it 0 SMB on the 2D DEM
mbmod = ConstantMassBalance(gdirs[0], bias=0)
mymb = mbmod.get_annual_mb(demref) * cfg.SEC_IN_YEAR * cfg.RHO
mbmod = ConstantMassBalance(gdirs[0], bias=np.average(mymb))
mymb = mbmod.get_annual_mb(demref) * cfg.SEC_IN_YEAR * cfg.RHO
np.testing.assert_allclose(np.average(mymb), 0., atol=1e-3)
# Same for ref
mbref = mbref - np.average(mbref)
np.testing.assert_allclose(np.average(mbref), 0., atol=1e-3)
# Fit poly
p = np.polyfit(demref, mbref, deg=2)
poly = np.poly1d(p)
myfit = poly(demref)
np.testing.assert_allclose(np.average(myfit), 0., atol=1e-3)
if do_plot:
import matplotlib.pyplot as plt
plt.scatter(mbref, demref, s=5, label='Obs (2007-2012), shifted to '
'Avg(SMB) = 0')
plt.scatter(mymb, demref, s=5, label='OGGM MB at t*')
plt.scatter(myfit, demref, s=5, label='Polyfit', c='C3')
plt.xlabel('MB (mm w.e yr-1)')
plt.ylabel('Altidude (m)')
plt.legend()
plt.show()
# Basic tasks
task_list = [
itmix.glacier_masks_itmix,
tasks.compute_centerlines,
tasks.catchment_area,
tasks.initialize_flowlines,
tasks.catchment_width_geom,
tasks.catchment_width_correction
]
for task in task_list:
execute_entity_task(task, gdirs)
# Climate related tasks
execute_entity_task(tasks.process_cru_data, gdirs)
tasks.compute_ref_t_stars(gdirs)
tasks.distribute_t_stars(gdirs)
# Inversion
execute_entity_task(tasks.prepare_for_inversion, gdirs)
itmix.optimize_thick(gdirs)
execute_entity_task(tasks.volume_inversion, gdirs)
# Write out glacier statistics
df = utils.glacier_characteristics(gdirs)
fpath = os.path.join(cfg.PATHS['working_dir'], 'glacier_char.csv')
df.to_csv(fpath)
if do_itmix:
done = False
for gd in gdirs:
def up_to_inversion(reset=False):
"""Run the tasks you want."""
# test directory
if not os.path.exists(TEST_DIR):
os.makedirs(TEST_DIR)
if reset:
clean_dir(TEST_DIR)
# Init
cfg.initialize()
# Use multiprocessing
cfg.PARAMS['use_multiprocessing'] = not ON_TRAVIS
# Working dir
cfg.PATHS['working_dir'] = TEST_DIR
cfg.PATHS['dem_file'] = get_demo_file('srtm_oetztal.tif')
# Set up the paths and other stuffs
cfg.set_divides_db(get_demo_file('divides_workflow.shp'))
cfg.PATHS['wgms_rgi_links'] = get_demo_file('RGI_WGMS_oetztal.csv')
cfg.PATHS['glathida_rgi_links'] = get_demo_file('RGI_GLATHIDA_oetztal.csv')
# Read in the RGI file
rgi_file = get_demo_file('rgi_oetztal.shp')
rgidf = gpd.GeoDataFrame.from_file(rgi_file)
# Be sure data is downloaded because lock doesn't work
cl = utils.get_cru_cl_file()
# Params
cfg.PARAMS['border'] = 70
cfg.PARAMS['use_optimized_inversion_params'] = True
# Go
gdirs = workflow.init_glacier_regions(rgidf)
try:
flowline.init_present_time_glacier(gdirs[0])
except Exception:
reset = True
if reset:
# First preprocessing tasks
workflow.gis_prepro_tasks(gdirs)
# Climate related tasks
# See if CRU is running
cfg.PARAMS['temp_use_local_gradient'] = False
cfg.PATHS['climate_file'] = '~'
cru_dir = get_demo_file('cru_ts3.23.1901.2014.tmp.dat.nc')
cfg.PATHS['cru_dir'] = os.path.dirname(cru_dir)
with warnings.catch_warnings():
# There is a warning from salem
warnings.simplefilter("ignore")
workflow.execute_entity_task(tasks.distribute_cru_style, gdirs)
tasks.compute_ref_t_stars(gdirs)
tasks.distribute_t_stars(gdirs)
# Use histalp for the actual test
cfg.PARAMS['temp_use_local_gradient'] = True
cfg.PATHS['climate_file'] = get_demo_file('HISTALP_oetztal.nc')
cfg.PATHS['cru_dir'] = '~'
workflow.climate_tasks(gdirs)
# Inversion
workflow.inversion_tasks(gdirs)
return gdirs
task_list = [
tasks.glacier_masks,
tasks.compute_centerlines,
tasks.initialize_flowlines,
tasks.catchment_area,
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
]
for task in task_list:
execute_entity_task(task, gdirs)
# Climate tasks
execute_entity_task(tasks.process_cru_data, gdirs)
tasks.compute_ref_t_stars(gdirs)
tasks.distribute_t_stars(gdirs)
execute_entity_task(tasks.apparent_mb, gdirs)
# Recompute after the first round - this is being picky but this is
# Because geometries may change after apparent_mb's filtering
tasks.compute_ref_t_stars(gdirs)
tasks.distribute_t_stars(gdirs)
execute_entity_task(tasks.apparent_mb, gdirs)
# Model validation
tasks.quick_crossval_t_stars(gdirs) # for later
tasks.distribute_t_stars(gdirs) # To restore after cross-val
# Tests: for all glaciers, the mass-balance around tstar and the
# bias with observation should be approx 0
from oggm.core.massbalance import (ConstantMassBalance, PastMassBalance)
for gd in gdirs:
tasks.compute_downstream_lines,
tasks.catchment_area,
tasks.initialize_flowlines,
tasks.catchment_width_geom,
tasks.catchment_width_correction
]
if RUN_GIS_PREPRO:
for task in task_list:
execute_entity_task(task, gdirs)
if RUN_CLIMATE_PREPRO:
# Climate related tasks
# see if we can distribute
workflow.execute_entity_task(tasks.process_cru_data, gdirs)
tasks.compute_ref_t_stars(gdirs)
tasks.distribute_t_stars(gdirs)
if RUN_INVERSION:
# Inversion
execute_entity_task(tasks.prepare_for_inversion, gdirs)
tasks.optimize_inversion_params(gdirs)
execute_entity_task(tasks.volume_inversion, gdirs)
if RUN_DYNAMICS:
# Random dynamics
execute_entity_task(tasks.init_present_time_glacier, gdirs)
execute_entity_task(tasks.random_glacier_evolution, gdirs)
# Plots (if you want)
PLOTS_DIR = ''
if PLOTS_DIR == '':
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
]
if RUN_GIS_PREPRO:
for task in task_list:
execute_entity_task(task, gdirs)
if RUN_CLIMATE_PREPRO:
for gdir in gdirs:
gdir.inversion_calving_rate = 0
cfg.PARAMS['correct_for_neg_flux'] = False
cfg.PARAMS['filter_for_neg_flux'] = False
execute_entity_task(tasks.process_cru_data, gdirs)
tasks.distribute_t_stars(gdirs)
execute_entity_task(tasks.apparent_mb, gdirs)
if RUN_INVERSION:
# Inversion tasks
execute_entity_task(tasks.prepare_for_inversion, gdirs)
tasks.optimize_inversion_params(gdirs)
execute_entity_task(tasks.volume_inversion, gdirs, )
# execute_entity_task(tasks.filter_inversion_output, gdirs)
# Compile output if no calving
if No_calving:
utils.glacier_characteristics(gdirs, filesuffix='_no_calving')
# Log
m, s = divmod(time.time() - start, 60)
# Basic tasks
task_list = [
itmix.glacier_masks_itmix,
tasks.compute_centerlines,
tasks.catchment_area,
tasks.initialize_flowlines,
tasks.catchment_width_geom,
tasks.catchment_width_correction
]
for task in task_list:
execute_entity_task(task, gdirs)
# Climate related tasks
execute_entity_task(tasks.process_cru_data, gdirs)
tasks.compute_ref_t_stars(gdirs)
tasks.distribute_t_stars(gdirs)
# Inversion
execute_entity_task(tasks.prepare_for_inversion, gdirs)
itmix.optimize_thick(gdirs)
execute_entity_task(tasks.volume_inversion, gdirs)
# Write out glacier statistics
df = utils.glacier_characteristics(gdirs)
fpath = os.path.join(cfg.PATHS['working_dir'], 'glacier_char.csv')
df.to_csv(fpath)
if do_itmix:
done = False
for gd in gdirs:
def test_crossval(self):
gdirs = up_to_distrib()
# in case we ran crossval we need to rerun
tasks.compute_ref_t_stars(gdirs)
tasks.distribute_t_stars(gdirs)
workflow.execute_entity_task(tasks.apparent_mb, gdirs)
# before crossval
refmustars = []
for gdir in gdirs:
tdf = pd.read_csv(gdir.get_filepath('local_mustar'))
refmustars.append(tdf['mu_star'].values[0])
tasks.crossval_t_stars(gdirs)
file = os.path.join(cfg.PATHS['working_dir'], 'crossval_tstars.csv')
df = pd.read_csv(file, index_col=0)
# after crossval we need to rerun
tasks.compute_ref_t_stars(gdirs)
tasks.distribute_t_stars(gdirs)
workflow.execute_entity_task(tasks.apparent_mb, gdirs)
# Test if quicker crossval is also OK
tasks.quick_crossval_t_stars(gdirs)
file = os.path.join(cfg.PATHS['working_dir'], 'crossval_tstars.csv')
dfq = pd.read_csv(file, index_col=0)
# after crossval we need to rerun
tasks.compute_ref_t_stars(gdirs)
tasks.distribute_t_stars(gdirs)
workflow.execute_entity_task(tasks.apparent_mb, gdirs)
assert np.all(np.abs(df.cv_bias) < 50)
assert np.all(np.abs(dfq.cv_bias) < 50)
# The biases aren't entirely equivalent and its ok
np.testing.assert_allclose(df.cv_prcp_fac, dfq.cv_prcp_fac)
# see if the process didn't brake anything
mustars = []
for gdir in gdirs:
tdf = pd.read_csv(gdir.get_filepath('local_mustar'))
mustars.append(tdf['mu_star'].values[0])
np.testing.assert_allclose(refmustars, mustars)
# make some mb tests
from oggm.core.massbalance import PastMassBalance
for rid in df.index:
gdir = [g for g in gdirs if g.rgi_id == rid][0]
h, w = gdir.get_inversion_flowline_hw()
cfg.PARAMS['use_bias_for_run'] = False
mbmod = PastMassBalance(gdir)
mbdf = gdir.get_ref_mb_data()['ANNUAL_BALANCE'].to_frame(name='ref')
for yr in mbdf.index:
mbdf.loc[yr, 'mine'] = mbmod.get_specific_mb(h, w, year=yr)
mm = mbdf.mean()
np.testing.assert_allclose(df.loc[rid].bias,
mm['mine'] - mm['ref'], atol=1e-3)
cfg.PARAMS['use_bias_for_run'] = True
mbmod = PastMassBalance(gdir)
mbdf = gdir.get_ref_mb_data()['ANNUAL_BALANCE'].to_frame(name='ref')
for yr in mbdf.index:
mbdf.loc[yr, 'mine'] = mbmod.get_specific_mb(h, w, year=yr)
mm = mbdf.mean()
np.testing.assert_allclose(mm['mine'], mm['ref'], atol=1e-3)
def quick_crossval(gdirs, xval, major=0):
# following climate.quick_crossval_t_stars
# but minimized for performance
full_ref_df = pd.read_csv(os.path.join(cfg.PATHS['working_dir'],
'ref_tstars.csv'),
index_col=0)
tmpdf = pd.DataFrame(
[], columns=['std_oggm', 'std_ref', 'rmse', 'core', 'bias'])
for i, rid in enumerate(full_ref_df.index):
# the glacier to look at
gdir = [g for g in gdirs if g.rgi_id == rid][0]
# the reference glaciers
tmp_ref_df = full_ref_df.loc[full_ref_df.index != rid]
# select reference glacier directories
# Only necessary if tasks.compute_ref_t_stars is uncommented below
# ref_gdirs = [g for g in gdirs if g.rgi_id != rid]
# before the cross-val store the info about "real" mustar
rdf = pd.read_csv(gdir.get_filepath('local_mustar'))
full_ref_df.loc[rid, 'mustar'] = rdf['mu_star'].values[0]
# redistribute t_star
with utils.DisableLogger():
# compute_ref_t_stars should be done again for
# every crossvalidation step
# This will/might have an influence if one of the 10 surrounding
# glaciers of the current glacier has more than one t_star
# If so, the currently crossvalidated glacier was probably
# used to select one t_star for this surrounding glacier.
#
# But: compute_ref_t_stars is very time consuming. And the
# influence is probably very small. Also only 40 out of the 253
# reference glaciers do have more than one possible t_star.
#
# tasks.compute_ref_t_stars(ref_gdirs)
tasks.distribute_t_stars([gdir], ref_df=tmp_ref_df)
# read crossvalidated values
rdf = pd.read_csv(gdir.get_filepath('local_mustar'))
# ----
# --- MASS-BALANCE MODEL
heights, widths = gdir.get_inversion_flowline_hw()
mb_mod = PastMassBalance(gdir,
mu_star=rdf['mu_star'].values[0],
bias=rdf['bias'].values[0],
prcp_fac=rdf['prcp_fac'].values[0])
# Mass-blaance timeseries, observed and simulated
refmb = gdir.get_ref_mb_data().copy()
refmb['OGGM'] = mb_mod.get_specific_mb(heights,
widths,
year=refmb.index)
# store single glacier results
bias = refmb.OGGM.mean() - refmb.ANNUAL_BALANCE.mean()
rmse = np.sqrt(np.mean(refmb.OGGM - refmb.ANNUAL_BALANCE)**2)
rcor = np.corrcoef(refmb.OGGM, refmb.ANNUAL_BALANCE)[0, 1]
ref_std = refmb.ANNUAL_BALANCE.std()
# unclear how to treat this best
if ref_std == 0:
ref_std = refmb.OGGM.std()
rcor = 1
tmpdf.loc[len(tmpdf.index)] = {
'std_oggm': refmb.OGGM.std(),
'std_ref': ref_std,
'bias': bias,
'rmse': rmse,
'core': rcor
}
if not major:
# store cross validated values
full_ref_df.loc[rid, 'cv_tstar'] = int(rdf['t_star'].values[0])
full_ref_df.loc[rid, 'cv_mustar'] = rdf['mu_star'].values[0]
full_ref_df.loc[rid, 'cv_bias'] = rdf['bias'].values[0]
full_ref_df.loc[rid, 'cv_prcp_fac'] = rdf['prcp_fac'].values[0]
# and store mean values
std_quot = np.mean(tmpdf.std_oggm / tmpdf.std_ref)
xval.loc[len(xval.index)] = {
'prcpsf': cfg.PARAMS['prcp_scaling_factor'],
'tliq': cfg.PARAMS['temp_all_liq'],
'tmelt': cfg.PARAMS['temp_melt'],
'tgrad': cfg.PARAMS['temp_default_gradient'],
'std_quot': std_quot,
'bias': tmpdf['bias'].mean(),
'rmse': tmpdf['rmse'].mean(),
'core': tmpdf['core'].mean()
}
if major:
return xval
else:
for i, rid in enumerate(full_ref_df.index):
# the glacier to look at
gdir = full_ref_df.loc[full_ref_df.index == rid]
# the reference glaciers
tmp_ref_df = full_ref_df.loc[full_ref_df.index != rid]
# Compute the distance
distances = utils.haversine(gdir.lon.values[0], gdir.lat.values[0],
tmp_ref_df.lon, tmp_ref_df.lat)
# Take the 10 closests
aso = np.argsort(distances)[0:9]
amin = tmp_ref_df.iloc[aso]
distances = distances[aso]**2
interp = np.average(amin.mustar, weights=1. / distances)
full_ref_df.loc[rid, 'interp_mustar'] = interp
# write
file = os.path.join(cfg.PATHS['working_dir'], 'crossval_tstars.csv')
full_ref_df.to_csv(file)
# alternative: do not write csv file, but store the needed values
# within xval_minor_statistics
return xval
for task in task_list:
execute_entity_task(task, gdirs)
# For some glaciers we use a width we know
for gdir in gdirs:
if gdir.rgi_id in dfmac.index:
width = dfmac.loc[gdir.rgi_id]['width']
tasks.terminus_width_correction(gdir, new_width=width)
if RUN_CLIMATE_PREPRO:
for gdir in gdirs:
gdir.inversion_calving_rate = 0
cfg.PARAMS['correct_for_neg_flux'] = False
cfg.PARAMS['filter_for_neg_flux'] = False
execute_entity_task(tasks.process_cru_data, gdirs)
tasks.distribute_t_stars(gdirs)
execute_entity_task(tasks.apparent_mb, gdirs)
suf = '_cfgA_cfgFS' + str(k)
if RUN_INVERSION:
# Inversion tasks
execute_entity_task(tasks.prepare_for_inversion, gdirs, add_debug_var=True)
execute_entity_task(tasks.volume_inversion, gdirs, glen_a=cfg.A, fs=cfg.FS)
execute_entity_task(tasks.filter_inversion_output, gdirs)
# Log
m, s = divmod(time.time() - start, 60)
h, m = divmod(m, 60)
log.info("OGGM no_calving is done! Time needed: %02d:%02d:%02d" %
(h, m, s))
def test_optimize_inversion(self):
# Download the RGI file for the run
# Make a new dataframe of those
rgidf = gpd.read_file(get_demo_file('SouthGlacier.shp'))
# Go - initialize working directories
gdirs = workflow.init_glacier_regions(rgidf)
# Preprocessing tasks
task_list = [
tasks.glacier_masks,
tasks.compute_centerlines,
tasks.initialize_flowlines,
tasks.catchment_area,
tasks.catchment_intersections,
tasks.catchment_width_geom,
tasks.catchment_width_correction,
]
for task in task_list:
execute_entity_task(task, gdirs)
# Climate tasks -- only data IO and tstar interpolation!
execute_entity_task(tasks.process_cru_data, gdirs)
tasks.distribute_t_stars(gdirs)
execute_entity_task(tasks.apparent_mb, gdirs)
# Reference data
gdir = gdirs[0]
df = self.get_ref_data(gdir)
# Inversion tasks
execute_entity_task(tasks.prepare_for_inversion, gdirs)
def to_optimize(x):
execute_entity_task(tasks.volume_inversion,
gdirs,
glen_a=cfg.A * x[0],
fs=cfg.FS * x[1])
execute_entity_task(tasks.distribute_thickness_per_altitude, gdirs)
with xr.open_dataset(gdir.get_filepath('gridded_data')) as ds:
thick = ds.distributed_thickness.isel_points(x=df['i'],
y=df['j'])
return (np.abs(thick - df.thick)).mean()
opti = optimization.minimize(to_optimize, [1., 1.],
bounds=((0.01, 10), (0.01, 10)),
tol=0.1)
# Check results and save.
execute_entity_task(tasks.volume_inversion,
gdirs,
glen_a=cfg.A * opti['x'][0],
fs=0)
execute_entity_task(tasks.distribute_thickness_per_altitude, gdirs)
with xr.open_dataset(gdir.get_filepath('gridded_data')) as ds:
df['oggm'] = ds.distributed_thickness.isel_points(x=df['i'],
y=df['j'])
ds['ref'] = xr.zeros_like(ds.distributed_thickness) * np.NaN
ds['ref'].data[df['j'], df['i']] = df['thick']
rmsd = ((df.oggm - df.thick)**2).mean()**.5
assert rmsd < 60
dfm = df.mean()
np.testing.assert_allclose(dfm.thick, dfm.oggm, 10)
if do_plot:
import matplotlib.pyplot as plt
df.plot(kind='scatter', x='oggm', y='thick')
plt.axis('equal')
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 3))
ds.ref.plot(ax=ax1)
ds.distributed_thickness.plot(ax=ax2)
plt.tight_layout()
plt.show()
