def test_ice_cap(): testdir = os.path.join(cfg.PATHS['test_dir'], 'tmp_icecap') utils.mkdir(testdir) cfg.initialize() cfg.PATHS['dem_file'] = get_demo_file('dem_RGI50-05.08389.tif') cfg.PARAMS['border'] = 20 cfg.set_divides_db(get_demo_file('divides_RGI50-05.08389.shp')) hef_file = get_demo_file('RGI50-05.08389.shp') entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0] gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=True) gis.define_glacier_region(gdir, entity=entity) gis.glacier_masks(gdir) centerlines.compute_centerlines(gdir) centerlines.compute_downstream_lines(gdir) geometry.initialize_flowlines(gdir) # We should have five groups lines = gdir.read_pickle('downstream_lines', div_id=0) assert len(np.unique(lines.group))==5 # This just checks that it works geometry.catchment_area(gdir) geometry.catchment_intersections(gdir) geometry.catchment_width_geom(gdir) geometry.catchment_width_correction(gdir) fig, ax = plt.subplots() graphics.plot_catchment_width(gdir, ax=ax, add_intersects=True, add_touches=True) fig.tight_layout() return fig
def test_nodivide_corrected(): # test directory testdir = os.path.join(cfg.PATHS['test_dir'], 'tmp_nodiv') if not os.path.exists(testdir): os.makedirs(testdir) # Init cfg.initialize() cfg.set_divides_db() cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif') cfg.PATHS['climate_file'] = get_demo_file('histalp_merged_hef.nc') cfg.PARAMS['border'] = 40 hef_file = get_demo_file('Hintereisferner_RGI5.shp') entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0] gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=True) gis.define_glacier_region(gdir, entity=entity) gis.glacier_masks(gdir) centerlines.compute_centerlines(gdir) geometry.initialize_flowlines(gdir) geometry.catchment_area(gdir) geometry.catchment_intersections(gdir) geometry.catchment_width_geom(gdir) geometry.catchment_width_correction(gdir) fig, ax = plt.subplots() graphics.plot_catchment_width(gdir, ax=ax, corrected=True, add_intersects=True, add_touches=True) fig.tight_layout() shutil.rmtree(testdir) return fig
def test_nodivide(): # test directory testdir = TESTDIR_BASE + '_nodiv' if not os.path.exists(testdir): os.makedirs(testdir) # Init cfg.initialize() cfg.set_divides_db() cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif') cfg.PATHS['climate_file'] = get_demo_file('histalp_merged_hef.nc') cfg.PARAMS['border'] = 40 # loop because for some reason indexing wont work hef_file = get_demo_file('Hintereisferner.shp') rgidf = gpd.GeoDataFrame.from_file(hef_file) for index, entity in rgidf.iterrows(): gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=True) gis.define_glacier_region(gdir, entity=entity) gis.glacier_masks(gdir) centerlines.compute_centerlines(gdir) graphics.plot_centerlines(gdir)
def setUp(self): N = 3 cfg.initialize() self.glen_a = 2.4e-24 # Modern style Glen parameter A self.aglen_old = (N + 2) * 1.9e-24 / 2. # outdated value self.fd = 2. * self.glen_a / (N + 2.) # equivalent to glen_a self.fs = 0 # set slidin self.fs_old = 5.7e-20 # outdated value
def test_download_cru(self): cfg.initialize() tmp = cfg.PATHS['cru_dir'] cfg.PATHS['cru_dir'] = TEST_DIR of = utils.get_cru_file('tmp') self.assertTrue(os.path.exists(of)) cfg.PATHS['cru_dir'] = tmp
def setUp(self): # test directory self.testdir = os.path.join(current_dir, "tmp") if not os.path.exists(self.testdir): os.makedirs(self.testdir) self.clean_dir() # Init cfg.initialize() cfg.PATHS["dem_file"] = get_demo_file("hef_srtm.tif")
def setUp(self): # test directory self.testdir = os.path.join(current_dir, 'tmp') if not os.path.exists(self.testdir): os.makedirs(self.testdir) self.clean_dir() # Init cfg.initialize() cfg.set_divides_db(get_demo_file('HEF_divided.shp')) cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif')
def test_download_rgi(self): cfg.initialize() tmp = cfg.PATHS['rgi_dir'] cfg.PATHS['rgi_dir'] = TEST_DIR of = utils.get_rgi_dir() of = os.path.join(of, '01_rgi50_Alaska', '01_rgi50_Alaska.shp') self.assertTrue(os.path.exists(of)) cfg.PATHS['rgi_dir'] = tmp
def test_coxe(): testdir = os.path.join(cfg.PATHS['test_dir'], 'tmp_coxe') utils.mkdir(testdir) # Init cfg.initialize() cfg.PATHS['dem_file'] = get_demo_file('dem_RGI50-01.10299.tif') cfg.PARAMS['border'] = 40 cfg.PARAMS['use_multiple_flowlines'] = False hef_file = get_demo_file('rgi_RGI50-01.10299.shp') entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0] gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=True) gis.define_glacier_region(gdir, entity=entity) gis.glacier_masks(gdir) centerlines.compute_centerlines(gdir) centerlines.compute_downstream_lines(gdir) geometry.initialize_flowlines(gdir) # Just check if the rest runs centerlines.compute_downstream_bedshape(gdir) geometry.catchment_area(gdir) geometry.catchment_intersections(gdir) geometry.catchment_width_geom(gdir) geometry.catchment_width_correction(gdir) climate.apparent_mb_from_linear_mb(gdir) inversion.prepare_for_inversion(gdir) inversion.volume_inversion(gdir, use_cfg_params={'glen_a': cfg.A, 'fs': 0}) inversion.filter_inversion_output(gdir) flowline.init_present_time_glacier(gdir) fls = gdir.read_pickle('model_flowlines') p = gdir.read_pickle('linear_mb_params') mb_mod = massbalance.LinearMassBalanceModel(ela_h=p['ela_h'], grad=p['grad']) mb_mod.temp_bias = -0.3 model = flowline.FluxBasedModel(fls, mb_model=mb_mod, y0=0, is_tidewater=True) # run model.run_until(200) assert model.calving_m3_since_y0 > 0 fig, ax = plt.subplots() graphics.plot_modeloutput_map(gdir, ax=ax, model=model) fig.tight_layout() return fig
def setUp(self): # test directory self.testdir = os.path.join(current_dir, "tmp") if not os.path.exists(self.testdir): os.makedirs(self.testdir) self.clean_dir() # Init cfg.initialize() cfg.PATHS["dem_file"] = get_demo_file("hef_srtm.tif") cfg.PATHS["climate_file"] = get_demo_file("histalp_merged_hef.nc") cfg.PARAMS["border"] = 10
def setUp(self): # test directory self.testdir = os.path.join(current_dir, "tmp_grindel") self.clean_dir() # Init cfg.initialize() cfg.PARAMS["use_multiple_flowlines"] = False # not crop cfg.PARAMS["max_thick_to_width_ratio"] = 10 cfg.PARAMS["max_shape_param"] = 10 cfg.PARAMS["section_smoothing"] = 0.0
def setUp(self): # test directory self.testdir = os.path.join(current_dir, 'tmp') if not os.path.exists(self.testdir): os.makedirs(self.testdir) self.clean_dir() # Init cfg.initialize() cfg.set_divides_db(get_demo_file('divides_workflow.shp')) cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif') cfg.PATHS['climate_file'] = get_demo_file('histalp_merged_hef.nc') cfg.PARAMS['border'] = 10
def up_to_climate(reset=False): """Run the tasks you want.""" # test directory if not os.path.exists(TEST_DIR): os.makedirs(TEST_DIR) if reset: clean_dir(TEST_DIR) if not os.path.exists(CLI_LOGF): with open(CLI_LOGF, 'wb') as f: pickle.dump('none', f) # Init cfg.initialize() # Use multiprocessing cfg.PARAMS['use_multiprocessing'] = use_multiprocessing() # Working dir cfg.PATHS['working_dir'] = TEST_DIR cfg.PATHS['dem_file'] = get_demo_file('srtm_oetztal.tif') cfg.set_intersects_db(get_demo_file('rgi_intersect_oetztal.shp')) # Read in the RGI file rgi_file = get_demo_file('rgi_oetztal.shp') rgidf = gpd.read_file(rgi_file) # Be sure data is downloaded utils.get_cru_cl_file() # Params cfg.PARAMS['border'] = 70 cfg.PARAMS['tstar_search_window'] = [1902, 0] cfg.PARAMS['run_mb_calibration'] = True # Go gdirs = workflow.init_glacier_regions(rgidf) try: tasks.catchment_width_correction(gdirs[0]) except Exception: reset = True if reset: # First preprocessing tasks workflow.gis_prepro_tasks(gdirs) return gdirs
def setUp(self): # test directory self.testdir = os.path.join(get_test_dir(), 'tmp') if not os.path.exists(self.testdir): os.makedirs(self.testdir) self.clean_dir() self.rgi_file = get_demo_file('rgi_RGI50-01.10299.shp') # Init cfg.initialize() cfg.PARAMS['use_intersects'] = False cfg.PATHS['dem_file'] = get_demo_file('dem_RGI50-01.10299.tif') cfg.PARAMS['border'] = 40
def setUp(self): # test directory self.testdir = os.path.join(get_test_dir(), 'tmp') if not os.path.exists(self.testdir): os.makedirs(self.testdir) self.clean_dir() # Init cfg.initialize() cfg.PARAMS['use_intersects'] = False cfg.PATHS['working_dir'] = self.testdir cfg.PATHS['dem_file'] = get_demo_file('dem_SouthGlacier.tif') cfg.PATHS['cru_dir'] = os.path.dirname(cfg.PATHS['dem_file']) cfg.PARAMS['border'] = 10
def test_chhota_shigri(): testdir = os.path.join(cfg.PATHS['test_dir'], 'tmp_chhota') utils.mkdir(testdir) # Init cfg.initialize() cfg.PATHS['dem_file'] = get_demo_file('dem_chhota_shigri.tif') cfg.PARAMS['border'] = 60 cfg.set_divides_db(get_demo_file('divides_RGI50-14.15990.shp')) hef_file = get_demo_file('RGI50-14.15990.shp') entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0] gdir = oggm.GlacierDirectory(entity, base_dir=testdir) gis.define_glacier_region(gdir, entity=entity) gis.glacier_masks(gdir) centerlines.compute_centerlines(gdir) centerlines.compute_downstream_lines(gdir) geometry.initialize_flowlines(gdir) # We should have two groups lines = gdir.read_pickle('downstream_lines', div_id=0) assert len(np.unique(lines.group)) == 2 # Just check if the rest runs centerlines.compute_downstream_bedshape(gdir) geometry.catchment_area(gdir) geometry.catchment_intersections(gdir) geometry.catchment_width_geom(gdir) geometry.catchment_width_correction(gdir) climate.apparent_mb_from_linear_mb(gdir) inversion.prepare_for_inversion(gdir) inversion.volume_inversion(gdir, use_cfg_params={'glen_a': cfg.A, 'fs': 0}) inversion.filter_inversion_output(gdir) flowline.init_present_time_glacier(gdir) fls = gdir.read_pickle('model_flowlines') for fl in fls: fl.thick = np.clip(fl.thick, 100, 1000) model = flowline.FlowlineModel(fls) fig, ax = plt.subplots() graphics.plot_modeloutput_map(gdir, ax=ax, model=model) fig.tight_layout() return fig
def test_download_demo_files(self): f = utils.get_demo_file('Hintereisferner.shp') self.assertTrue(os.path.exists(f)) sh = salem.read_shapefile(f) self.assertTrue(hasattr(sh, 'geometry')) # Data files cfg.initialize() lf, df = utils.get_wgms_files() self.assertTrue(os.path.exists(lf)) lf = utils.get_glathida_file() self.assertTrue(os.path.exists(lf))
def test_nodivide(): # test directory testdir = TESTDIR_BASE + "_nodiv" if not os.path.exists(testdir): os.makedirs(testdir) # Init cfg.initialize() cfg.set_divides_db() cfg.PATHS["dem_file"] = get_demo_file("hef_srtm.tif") cfg.PATHS["climate_file"] = get_demo_file("histalp_merged_hef.nc") cfg.PARAMS["border"] = 40 hef_file = get_demo_file("Hintereisferner.shp") entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0] gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=True) gis.define_glacier_region(gdir, entity=entity) gis.glacier_masks(gdir) centerlines.compute_centerlines(gdir) graphics.plot_centerlines(gdir)
def init_hef(reset=False, border=40): # test directory testdir = os.path.join(get_test_dir(), 'tmp_border{}'.format(border)) if not os.path.exists(testdir): os.makedirs(testdir) reset = True # Init cfg.initialize() cfg.set_intersects_db(get_demo_file('rgi_intersect_oetztal.shp')) cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif') cfg.PATHS['climate_file'] = get_demo_file('histalp_merged_hef.nc') cfg.PARAMS['baseline_climate'] = '' cfg.PATHS['working_dir'] = testdir cfg.PARAMS['border'] = border hef_file = get_demo_file('Hintereisferner_RGI5.shp') entity = gpd.read_file(hef_file).iloc[0] gdir = oggm.GlacierDirectory(entity, reset=reset) if not gdir.has_file('inversion_params'): reset = True gdir = oggm.GlacierDirectory(entity, reset=reset) if not reset: return gdir gis.define_glacier_region(gdir, entity=entity) execute_entity_task(gis.glacier_masks, [gdir]) execute_entity_task(centerlines.compute_centerlines, [gdir]) centerlines.initialize_flowlines(gdir) centerlines.compute_downstream_line(gdir) centerlines.compute_downstream_bedshape(gdir) centerlines.catchment_area(gdir) centerlines.catchment_intersections(gdir) centerlines.catchment_width_geom(gdir) centerlines.catchment_width_correction(gdir) climate.process_custom_climate_data(gdir) mbdf = gdir.get_ref_mb_data()['ANNUAL_BALANCE'] res = climate.t_star_from_refmb(gdir, mbdf=mbdf) climate.local_t_star(gdir, tstar=res['t_star'], bias=res['bias']) climate.mu_star_calibration(gdir) inversion.prepare_for_inversion(gdir, add_debug_var=True) ref_v = 0.573 * 1e9 glen_n = cfg.PARAMS['glen_n'] def to_optimize(x): # For backwards compat _fd = 1.9e-24 * x[0] glen_a = (glen_n+2) * _fd / 2. fs = 5.7e-20 * x[1] v, _ = inversion.mass_conservation_inversion(gdir, fs=fs, glen_a=glen_a) return (v - ref_v)**2 out = optimization.minimize(to_optimize, [1, 1], bounds=((0.01, 10), (0.01, 10)), tol=1e-4)['x'] _fd = 1.9e-24 * out[0] glen_a = (glen_n+2) * _fd / 2. fs = 5.7e-20 * out[1] v, _ = inversion.mass_conservation_inversion(gdir, fs=fs, glen_a=glen_a, write=True) d = dict(fs=fs, glen_a=glen_a) d['factor_glen_a'] = out[0] d['factor_fs'] = out[1] gdir.write_pickle(d, 'inversion_params') # filter inversion.filter_inversion_output(gdir) inversion.distribute_thickness_interp(gdir, varname_suffix='_interp') inversion.distribute_thickness_per_altitude(gdir, varname_suffix='_alt') flowline.init_present_time_glacier(gdir) return gdir
def run_prepro_levels(rgi_version=None, rgi_reg=None, border=None, output_folder='', working_dir='', is_test=False, demo=False, test_rgidf=None, test_intersects_file=None, test_topofile=None, test_crudir=None): """Does the actual job. Parameters ---------- rgi_version : str the RGI version to use (defaults to cfg.PARAMS) rgi_reg : str the RGI region to process border : int the number of pixels at the maps border output_folder : str path to the output folder (where to put the preprocessed tar files) working_dir : str path to the OGGM working directory is_test : bool to test on a couple of glaciers only! demo : bool to run the prepro for the list of demo glaciers test_rgidf : shapefile for testing purposes only test_intersects_file : shapefile for testing purposes only test_topofile : str for testing purposes only test_crudir : str for testing purposes only """ # TODO: temporarily silence Fiona deprecation warnings import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) # Module logger log = logging.getLogger(__name__) # Time start = time.time() # Initialize OGGM and set up the run parameters cfg.initialize(logging_level='WORKFLOW') # Local paths utils.mkdir(working_dir) cfg.PATHS['working_dir'] = working_dir # Use multiprocessing? cfg.PARAMS['use_multiprocessing'] = True # How many grid points around the glacier? # Make it large if you expect your glaciers to grow large cfg.PARAMS['border'] = border # Set to True for operational runs cfg.PARAMS['continue_on_error'] = True # For statistics climate_periods = [1920, 1960, 2000] if rgi_version is None: rgi_version = cfg.PARAMS['rgi_version'] rgi_dir_name = 'RGI{}'.format(rgi_version) border_dir_name = 'b_{:03d}'.format(border) base_dir = os.path.join(output_folder, rgi_dir_name, border_dir_name) # Add a package version file utils.mkdir(base_dir) opath = os.path.join(base_dir, 'package_versions.txt') with open(opath, 'w') as vfile: vfile.write(utils.show_versions(logger=log)) if demo: rgidf = utils.get_rgi_glacier_entities(cfg.DEMO_GLACIERS.index) elif test_rgidf is None: # Get the RGI file rgidf = gpd.read_file(utils.get_rgi_region_file(rgi_reg, version=rgi_version)) # We use intersects rgif = utils.get_rgi_intersects_region_file(rgi_reg, version=rgi_version) cfg.set_intersects_db(rgif) else: rgidf = test_rgidf cfg.set_intersects_db(test_intersects_file) if is_test: # Just for fun rgidf = rgidf.sample(4) # Sort for more efficient parallel computing rgidf = rgidf.sort_values('Area', ascending=False) log.workflow('Starting prepro run for RGI reg: {} ' 'and border: {}'.format(rgi_reg, border)) log.workflow('Number of glaciers: {}'.format(len(rgidf))) # Input if test_topofile: cfg.PATHS['dem_file'] = test_topofile # L1 - initialize working directories gdirs = workflow.init_glacier_regions(rgidf, reset=True, force=True) # Glacier stats sum_dir = os.path.join(base_dir, 'L1', 'summary') utils.mkdir(sum_dir) opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg)) utils.compile_glacier_statistics(gdirs, path=opath) # L1 OK - compress all in output directory l_base_dir = os.path.join(base_dir, 'L1') workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False, base_dir=l_base_dir) utils.base_dir_to_tar(l_base_dir) # L2 - Tasks # Pre-download other files just in case if test_crudir is None: _ = utils.get_cru_file(var='tmp') _ = utils.get_cru_file(var='pre') else: cfg.PATHS['cru_dir'] = test_crudir workflow.execute_entity_task(tasks.process_cru_data, gdirs) # Glacier stats sum_dir = os.path.join(base_dir, 'L2', 'summary') utils.mkdir(sum_dir) opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg)) utils.compile_glacier_statistics(gdirs, path=opath) # L2 OK - compress all in output directory l_base_dir = os.path.join(base_dir, 'L2') workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False, base_dir=l_base_dir) utils.base_dir_to_tar(l_base_dir) # L3 - Tasks task_list = [ tasks.glacier_masks, tasks.compute_centerlines, tasks.initialize_flowlines, tasks.compute_downstream_line, tasks.compute_downstream_bedshape, tasks.catchment_area, tasks.catchment_intersections, tasks.catchment_width_geom, tasks.catchment_width_correction, tasks.local_t_star, tasks.mu_star_calibration, tasks.prepare_for_inversion, tasks.mass_conservation_inversion, tasks.filter_inversion_output, ] for task in task_list: workflow.execute_entity_task(task, gdirs) # Glacier stats sum_dir = os.path.join(base_dir, 'L3', 'summary') utils.mkdir(sum_dir) opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg)) utils.compile_glacier_statistics(gdirs, path=opath) opath = os.path.join(sum_dir, 'climate_statistics_{}.csv'.format(rgi_reg)) utils.compile_climate_statistics(gdirs, add_climate_period=climate_periods, path=opath) # L3 OK - compress all in output directory l_base_dir = os.path.join(base_dir, 'L3') workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False, base_dir=l_base_dir) utils.base_dir_to_tar(l_base_dir) # L4 - Tasks workflow.execute_entity_task(tasks.init_present_time_glacier, gdirs) # Glacier stats sum_dir = os.path.join(base_dir, 'L4', 'summary') utils.mkdir(sum_dir) opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg)) utils.compile_glacier_statistics(gdirs, path=opath) # Copy mini data to new dir base_dir = os.path.join(base_dir, 'L4') mini_gdirs = workflow.execute_entity_task(tasks.copy_to_basedir, gdirs, base_dir=base_dir) # L4 OK - compress all in output directory workflow.execute_entity_task(utils.gdir_to_tar, mini_gdirs, delete=True) utils.base_dir_to_tar(base_dir) # Log m, s = divmod(time.time() - start, 60) h, m = divmod(m, 60) log.workflow('OGGM prepro_levels is done! Time needed: ' '{:02d}:{:02d}:{:02d}'.format(int(h), int(m), int(s)))
def setUp(self): cfg.initialize()
def setUp(self): cfg.initialize() cfg.PATHS['dl_cache_dir'] = os.path.join(TEST_DIR, 'dl_cache') cfg.PATHS['working_dir'] = os.path.join(TEST_DIR, 'wd') cfg.PATHS['tmp_dir'] = os.path.join(TEST_DIR, 'extract') self.reset_dir()
import geopandas as gpd import shapely.geometry as shpg # Locals import oggm.cfg as cfg from oggm import utils, workflow, tasks # For timing the run import time start = time.time() # Module logger log = logging.getLogger(__name__) # Initialize OGGM and set up the default run parameters cfg.initialize(logging_level='WORKFLOW') rgi_version = '61' rgi_region = '11' # Region Central Europe # Here we override some of the default parameters # How many grid points around the glacier? # Make it large if you expect your glaciers to grow large: # here, 80 is more than enough cfg.PARAMS['border'] = 80 # Local working directory (where OGGM will write its output) WORKING_DIR = utils.gettempdir('OGGM_Rofental') utils.mkdir(WORKING_DIR, reset=True) cfg.PATHS['working_dir'] = WORKING_DIR # RGI file
from oggm.workflow import execute_entity_task from oggm import utils from oggm.core import inversion import pandas as pd # Time import time start = time.time() # Regions: Alaska rgi_region = '01' # Initialize OGGM and set up the run parameters # --------------------------------------------- cfg.initialize() rgi_version = '6' SLURM_WORKDIR = os.environ["WORKDIR"] # Local paths (where to write output and where to download input) WORKING_DIR = SLURM_WORKDIR MAIN_PATH = os.path.expanduser('~/cryo_calving_2019/') RGI_FILE = os.path.join( MAIN_PATH, 'input_data/01_rgi60_Alaska_modify/01_rgi60_Alaska.shp') Columbia_prepro = os.path.join( MAIN_PATH, 'output_data/1_Columbia_Glacier_runs/1_preprocessing/per_glacier/')
def setUp(self): cfg.initialize() self.homedir = os.path.expanduser('~')
def sensitivity_run_vas(rgi_ids, use_random_mb=False, use_mean=False, path=True, temp_bias=0, tstar=None, use_default_tstar=True, use_bias_for_run=True, scaling_params=[(4.5507, 0.191, 2.2, 1.375)], time_scale_factors=[1], suffixes=['_default'], **kwargs): """ The routine runs all steps for the equilibrium experiments using the volume/area scaling model (cf. `equilibrium_run_vas`) but for only one given temperature bias. However, it is possible to supply a list of sensitivity parameters (the scaling constants, and time scale factor) to alter the model behavior. - OGGM preprocessing, including initialization, GIS tasks, climate tasks and massbalance tasks. - Run model for all glaciers with constant (or random) massbalance model over 3000 years (default value). - Process the model output dataset(s), i.e. normalization, average/sum, ... The final dataset containing all results is returned. Given a path is is also stored to file. Parameters ---------- rgi_ids: array-like List of RGI IDs for which the equilibrium experiments are performed. use_random_mb: bool, optional, default=True Choose between random massbalance model and constant massbalance model. use_mean: bool, optional, default=False Choose between the mean or summation over all glaciers path: bool or str, optional, default=True If a path is given (or True), the resulting dataset is stored to file. temp_bias: float, optional, default=0 Temperature bias (degC) for the mass balance model. sensitivity_params: multi-dimensional array-like, optional, default=[(4.5507, 0.191, 2.2, 1.375)] List containing the scaling constants and scaling exponents for length and area scaling as tuples, e.g., (c_l, c_a, q, gamma) suffixes: array-like, optional, default=['_default'] Descriptive suffixes corresponding to the given sensitivity params tstar: float, optional, default=None 'Equilibrium year' used for the mass balance calibration. use_default_tstar: bool, optional, default=True Flag deciding whether or not to compute mustar from given from reference table. Overridden by a given tstar. use_bias_for_run: bool, optional, default=True Flag deciding whether or not to use the mass balance residual. kwargs: Additional key word arguments for the `run_random_climate` or `run_constant_climate` routines of the vascaling module. Returns ------- Dataset containing yearly values of all glacier geometries. """ # assert correct output file suffixes for temp biases if len(scaling_params) * len(time_scale_factors) != len(suffixes): raise RuntimeError("Each given combination of scaling parameters and " "time scale factor must have one, and only one, " "corresponding suffix.") # OGGM preprocessing # ------------------ # compute RGI region and version from RGI IDs # assuming all they are all the same rgi_region = (rgi_ids[0].split('-')[-1]).split('.')[0] rgi_version = (rgi_ids[0].split('-')[0])[-2:-1] # load default parameter file cfg.initialize() # get environmental variables for working and output directories WORKING_DIR = os.environ["WORKDIR"] OUTPUT_DIR = os.environ["OUTDIR"] # create working directory utils.mkdir(WORKING_DIR) utils.mkdir(OUTPUT_DIR) # set path to working directory cfg.PATHS['working_dir'] = WORKING_DIR # set RGI version and region cfg.PARAMS['rgi_version'] = rgi_version # define how many grid points to use around the glacier, # if you expect the glacier to grow large use a larger border cfg.PARAMS['border'] = 120 # we use HistAlp climate data cfg.PARAMS['baseline_climate'] = 'HISTALP' # set the mb hyper parameters accordingly cfg.PARAMS['prcp_scaling_factor'] = 2.5 cfg.PARAMS['temp_melt'] = -0.5 # the bias is defined to be zero during the calibration process, # which is why we don't use it here to reproduce the results cfg.PARAMS['use_bias_for_run'] = use_bias_for_run # read RGI entry for the glaciers as DataFrame # containing the outline area as shapefile rgidf = utils.get_rgi_glacier_entities(rgi_ids) # get and set path to intersect shapefile intersects_db = utils.get_rgi_intersects_region_file(region=rgi_region) cfg.set_intersects_db(intersects_db) # sort by area for more efficient parallel computing rgidf = rgidf.sort_values('Area', ascending=False) cfg.PARAMS['use_multiprocessing'] = True # operational run, all glaciers should run cfg.PARAMS['continue_on_error'] = True # initialize the GlacierDirectory gdirs = workflow.init_glacier_directories(rgidf, reset=False, force=True) # define the local grid and glacier mask workflow.execute_entity_task(gis.define_glacier_region, gdirs) workflow.execute_entity_task(gis.glacier_masks, gdirs) # process the given climate file workflow.execute_entity_task(climate.process_climate_data, gdirs) # compute local t* and the corresponding mu* if tstar or use_default_tstar: # compute mustar from given tstar or reference table workflow.execute_entity_task(vascaling.local_t_star, gdirs, tstar=tstar, bias=0) else: # compute mustar from the reference table for the flowline model # RGI v6 and HISTALP baseline climate ref_df = pd.read_csv( utils.get_demo_file('oggm_ref_tstars_rgi6_histalp.csv')) workflow.execute_entity_task(vascaling.local_t_star, gdirs, ref_df=ref_df) # Run model with constant/random mass balance model # ------------------------------------------------- # use t* as center year, even if specified differently kwargs['y0'] = tstar # run for 3000 years if not specified otherwise kwargs.setdefault('nyears', 1000) # limit parameters to 3 decimal points scaling_params = recursive_round(scaling_params, 3) time_scale_factors = recursive_round(time_scale_factors, 3) # assure that scaling params are handled as tuples (pairs) scaling_params_list = np.zeros(len(scaling_params), dtype=object) scaling_params_list[:] = scaling_params # combine scaling constants, scaling exponents and time scale factor # into one iterable array sensitivity_params = np.array( np.meshgrid(scaling_params_list, time_scale_factors)).T sensitivity_params = (sensitivity_params.reshape( -1, sensitivity_params.shape[-1])) if use_random_mb: # set random seed to get reproducible results kwargs.setdefault('seed', 12) # run RandomMassBalance model centered around t* for each given # parameter set for suffix, params in zip(suffixes, sensitivity_params): cfg.PARAMS['vas_c_length_m'] = params[0][0] cfg.PARAMS['vas_c_area_m2'] = params[0][1] cfg.PARAMS['vas_q_length'] = params[0][2] cfg.PARAMS['vas_gamma_area'] = params[0][3] kwargs['time_scale_factor'] = params[1] workflow.execute_entity_task(vascaling.run_random_climate, gdirs, temperature_bias=temp_bias, output_filesuffix=suffix, **kwargs) else: # run ConstantMassBalance model centered around t* for each given # parameter set for suffix, params in zip(suffixes, sensitivity_params): cfg.PARAMS['vas_c_length_m'] = params[0][0] cfg.PARAMS['vas_c_area_m2'] = params[0][1] cfg.PARAMS['vas_q_length'] = params[0][2] cfg.PARAMS['vas_gamma_area'] = params[0][3] kwargs['time_scale_factor'] = params[1] workflow.execute_entity_task(vascaling.run_constant_climate, gdirs, temperature_bias=temp_bias, output_filesuffix=suffix, **kwargs) # Process output dataset(s) # ------------------------- # create empty container ds = list() # iterate over all scaling constants for i, params in enumerate(scaling_params): # create empty container ds_ = list() # iterate over all time scale factor for j, factor in enumerate(time_scale_factors): # compile the output for each run pos = j + len(time_scale_factors) * i ds__ = utils.compile_run_output(np.atleast_1d(gdirs), filesuffix=suffixes[pos], path=False) # add time scale factor as coordinate ds__.coords['time_scale_factor'] = factor # add to container ds_.append(ds__) # concatenate using time scale factor as concat dimension ds_ = xr.concat(ds_, dim='time_scale_factor') # add scaling constants as coordinate params_list = np.zeros(len([params]), dtype=object) params_list[:] = [params] ds_ = ds_.expand_dims(dim={'scaling_params': params_list}) # add to container ds.append(ds_) # concatenate using scaling constants as concat dimension ds = xr.concat(ds, dim='scaling_params') # add model type as coordinate ds.coords['model'] = 'vas' # add mb model type as coordinate ds.coords['mb_model'] = 'random' if use_random_mb else 'constant' # normalize glacier geometries (length/area/volume) with start value if use_mean: # compute average over all glaciers ds_normal = normalize_ds_with_start(ds).mean(dim='rgi_id') ds = ds.mean(dim='rgi_id') else: # compute sum over all glaciers ds_normal = normalize_ds_with_start(ds.sum(dim='rgi_id')) ds = ds.sum(dim='rgi_id') # add coordinate to distinguish between normalized and absolute values ds.coords['normalized'] = False ds_normal.coords['normalized'] = True # combine datasets ds = xr.concat([ds, ds_normal], 'normalized') # store datasets if path: if not isinstance(path, str): # set default path and filename mb = 'random' if use_random_mb else 'constant' path = os.path.join(OUTPUT_DIR, f'run_output_{mb}_vas.nc') # write to file log.info(f'Writing run output to {path}') pickle.dump(ds, open(path, mode='wb')) # return ds, ds_normal return ds
def compute_scaling_params(rgi_ids, path=None): """ The routine computes scaling parameters by fitting a linear regression to the volume/area and volume/length scatter in log-log space, using the inversion volume, the RGI area and the longest centerline as "observations" Thereby, the following two cases apply: - compute only scaling constants, since scaling exponents have a physical basis and should not be changed - compute only scaling constants and scaling exponents Returns parameters in a 2-level dictionary. The upper level differentiates between the two cases, the lower level indicates the parameters. Parameters ---------- rgi_ids: array-like List of RGI IDs for which the equilibrium experiments are performed. Returns ------- Dictionary containing the computed parameters. """ log.info('Starting scaling parameter computation') # compute RGI region and version from RGI IDs # assuming all they are all the same rgi_region = (rgi_ids[0].split('-')[-1]).split('.')[0] rgi_version = (rgi_ids[0].split('-')[0])[-2:-1] # load default parameter file cfg.initialize() # get environmental variables for working and output directories WORKING_DIR = os.environ["WORKDIR"] OUTPUT_DIR = os.environ["OUTDIR"] # create working directory utils.mkdir(WORKING_DIR) utils.mkdir(OUTPUT_DIR) # set path to working directory cfg.PATHS['working_dir'] = WORKING_DIR # set RGI version and region cfg.PARAMS['rgi_version'] = rgi_version # define how many grid points to use around the glacier, # if you expect the glacier to grow large use a larger border cfg.PARAMS['border'] = 120 # we use HistAlp climate data cfg.PARAMS['baseline_climate'] = 'HISTALP' # set the mb hyper parameters accordingly cfg.PARAMS['prcp_scaling_factor'] = 1.75 cfg.PARAMS['temp_melt'] = -1.75 # read RGI entry for the glaciers as DataFrame # containing the outline area as shapefile rgidf = utils.get_rgi_glacier_entities(rgi_ids) # get and set path to intersect shapefile intersects_db = utils.get_rgi_intersects_region_file(region=rgi_region) cfg.set_intersects_db(intersects_db) # the bias is defined to be zero during the calibration process, # which is why we don't use it here to reproduce the results cfg.PARAMS['use_bias_for_run'] = True # sort by area for more efficient parallel computing rgidf = rgidf.sort_values('Area', ascending=False) cfg.PARAMS['use_multiprocessing'] = True # operational run, all glaciers should run cfg.PARAMS['continue_on_error'] = False # initialize the GlacierDirectory gdirs = workflow.init_glacier_directories(rgidf, reset=False, force=True) # run gis tasks workflow.gis_prepro_tasks(gdirs) # run climate tasks workflow.execute_entity_task(climate.process_climate_data, gdirs) # compute local t* and the corresponding mu* workflow.execute_entity_task(climate.local_t_star, gdirs) workflow.execute_entity_task(climate.mu_star_calibration, gdirs) # run inversion tasks workflow.inversion_tasks(gdirs) # finalize preprocessing workflow.execute_entity_task(flowline.init_present_time_glacier, gdirs) # create empty dictionary params = dict() # compute scaling constants for given (fixed) slope params['const_only'] = vascaling.get_scaling_constant(gdirs) # compute scaling constants and scaling exponent via linear regression params['const_expo'] = vascaling.get_scaling_constant_exponent(gdirs) # store to file if path: if not isinstance(path, str): # set default path and filename path = os.path.join(OUTPUT_DIR, 'scaling_params.json') json.dump(params, open(path, 'w')) return params
import pandas as pd import shapely.geometry as shpg # Locals import oggm.cfg as cfg from oggm import utils, workflow, tasks # For timing the run import time start = time.time() # Module logger log = logging.getLogger(__name__) #Initialize OGGM and set up the default run parameters cfg.initialize(logging_level='WORKFLOW') rgi_version = '61' rgi_region = '15' # No need for a big map here cfg.PARAMS['border'] = 80 # Local working directory (where OGGM will write its output) WORKING_DIR = '/Users/louis/Prepro_test/' utils.mkdir(WORKING_DIR, reset=True) cfg.PATHS['working_dir'] = WORKING_DIR # RGI file path = utils.get_rgi_region_file(rgi_region, version=rgi_version) rgidf = gpd.read_file(path)
import geopandas as gpd import shapely.geometry as shpg # Locals import oggm.cfg as cfg from oggm import utils, workflow, tasks # For timing the run import time start = time.time() # Module logger log = logging.getLogger(__name__) # Initialize OGGM and set up the default run parameters cfg.initialize(logging_level='ERROR') rgi_version = '61' rgi_region = '11' # Region Central Europe # Here we override some of the default parameters # How many grid points around the glacier? # Make it large if you expect your glaciers to grow large: # here, 80 is more than enough cfg.PARAMS['border'] = 80 # Local working directory (where OGGM will write its output) WORKING_DIR = utils.gettempdir('OGGM_Rofental') utils.mkdir(WORKING_DIR, reset=True) cfg.PATHS['working_dir'] = WORKING_DIR # RGI file
def init_hef(reset=False, border=40, invert_with_sliding=True): from oggm.core.preprocessing import gis, centerlines, geometry from oggm.core.preprocessing import climate, inversion import oggm import oggm.cfg as cfg from oggm.utils import get_demo_file # test directory testdir = TESTDIR_BASE + "_border{}".format(border) if not invert_with_sliding: testdir += "_withoutslide" if not os.path.exists(testdir): os.makedirs(testdir) reset = True if not os.path.exists(os.path.join(testdir, "RGI40-11.00897")): reset = True if not os.path.exists(os.path.join(testdir, "RGI40-11.00897", "inversion_params.pkl")): reset = True # Init cfg.initialize() cfg.PATHS["dem_file"] = get_demo_file("hef_srtm.tif") cfg.PATHS["climate_file"] = get_demo_file("histalp_merged_hef.nc") cfg.PARAMS["border"] = border hef_file = get_demo_file("Hintereisferner.shp") entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0] gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=reset) if not reset: return gdir gis.define_glacier_region(gdir, entity=entity) gis.glacier_masks(gdir) centerlines.compute_centerlines(gdir) centerlines.compute_downstream_lines(gdir) geometry.initialize_flowlines(gdir) geometry.catchment_area(gdir) geometry.catchment_width_geom(gdir) geometry.catchment_width_correction(gdir) climate.process_histalp_nonparallel([gdir]) climate.mu_candidates(gdir, div_id=0) hef_file = get_demo_file("mbdata_RGI40-11.00897.csv") mbdf = pd.read_csv(hef_file).set_index("YEAR") t_star, bias, prcp_fac = climate.t_star_from_refmb(gdir, mbdf["ANNUAL_BALANCE"]) climate.local_mustar_apparent_mb(gdir, tstar=t_star[-1], bias=bias[-1], prcp_fac=prcp_fac) inversion.prepare_for_inversion(gdir) ref_v = 0.573 * 1e9 if invert_with_sliding: def to_optimize(x): # For backwards compat _fd = 1.9e-24 * x[0] glen_a = (cfg.N + 2) * _fd / 2.0 fs = 5.7e-20 * x[1] v, _ = inversion.invert_parabolic_bed(gdir, fs=fs, glen_a=glen_a) return (v - ref_v) ** 2 out = optimization.minimize(to_optimize, [1, 1], bounds=((0.01, 10), (0.01, 10)), tol=1e-4)["x"] _fd = 1.9e-24 * out[0] glen_a = (cfg.N + 2) * _fd / 2.0 fs = 5.7e-20 * out[1] v, _ = inversion.invert_parabolic_bed(gdir, fs=fs, glen_a=glen_a, write=True) else: def to_optimize(x): glen_a = cfg.A * x[0] v, _ = inversion.invert_parabolic_bed(gdir, fs=0.0, glen_a=glen_a) return (v - ref_v) ** 2 out = optimization.minimize(to_optimize, [1], bounds=((0.01, 10),), tol=1e-4)["x"] glen_a = cfg.A * out[0] fs = 0.0 v, _ = inversion.invert_parabolic_bed(gdir, fs=fs, glen_a=glen_a, write=True) d = dict(fs=fs, glen_a=glen_a) d["factor_glen_a"] = out[0] try: d["factor_fs"] = out[1] except IndexError: d["factor_fs"] = 0.0 gdir.write_pickle(d, "inversion_params") inversion.distribute_thickness(gdir, how="per_altitude", add_nc_name=True) inversion.distribute_thickness(gdir, how="per_interpolation", add_slope=False, smooth=False, add_nc_name=True) return gdir
def setup_module(module): """Used by pytest""" cfg.initialize()
def up_to_inversion(reset=False): """Run the tasks you want.""" # test directory if not os.path.exists(TEST_DIR): os.makedirs(TEST_DIR) if reset: clean_dir(TEST_DIR) # Init cfg.initialize() # Use multiprocessing cfg.PARAMS['use_multiprocessing'] = not ON_TRAVIS # Working dir cfg.PATHS['working_dir'] = TEST_DIR cfg.PATHS['dem_file'] = get_demo_file('srtm_oetztal.tif') # Set up the paths and other stuffs cfg.set_divides_db(get_demo_file('divides_workflow.shp')) cfg.PATHS['wgms_rgi_links'] = get_demo_file('RGI_WGMS_oetztal.csv') cfg.PATHS['glathida_rgi_links'] = get_demo_file('RGI_GLATHIDA_oetztal.csv') # Read in the RGI file rgi_file = get_demo_file('rgi_oetztal.shp') rgidf = gpd.GeoDataFrame.from_file(rgi_file) # Be sure data is downloaded because lock doesn't work cl = utils.get_cru_cl_file() # Params cfg.PARAMS['border'] = 70 cfg.PARAMS['use_optimized_inversion_params'] = True # Go gdirs = workflow.init_glacier_regions(rgidf) try: flowline.init_present_time_glacier(gdirs[0]) except Exception: reset = True if reset: # First preprocessing tasks workflow.gis_prepro_tasks(gdirs) # Climate related tasks # See if CRU is running cfg.PARAMS['temp_use_local_gradient'] = False cfg.PATHS['climate_file'] = '~' cru_dir = get_demo_file('cru_ts3.23.1901.2014.tmp.dat.nc') cfg.PATHS['cru_dir'] = os.path.dirname(cru_dir) with warnings.catch_warnings(): # There is a warning from salem warnings.simplefilter("ignore") workflow.execute_entity_task(tasks.distribute_cru_style, gdirs) tasks.compute_ref_t_stars(gdirs) tasks.distribute_t_stars(gdirs) # Use histalp for the actual test cfg.PARAMS['temp_use_local_gradient'] = True cfg.PATHS['climate_file'] = get_demo_file('HISTALP_oetztal.nc') cfg.PATHS['cru_dir'] = '~' workflow.climate_tasks(gdirs) # Inversion workflow.inversion_tasks(gdirs) return gdirs
def up_to_inversion(reset=False): """Run the tasks you want.""" # test directory if not os.path.exists(TEST_DIR): os.makedirs(TEST_DIR) if reset: clean_dir(TEST_DIR) # Init cfg.initialize() # Use multiprocessing cfg.PARAMS['use_multiprocessing'] = not ON_TRAVIS # Working dir cfg.PATHS['working_dir'] = TEST_DIR cfg.set_divides_db(get_demo_file('HEF_divided.shp')) cfg.PATHS['dem_file'] = get_demo_file('srtm_oeztal.tif') # Set up the paths and other stuffs cfg.set_divides_db(get_demo_file('HEF_divided.shp')) cfg.PATHS['wgms_rgi_links'] = get_demo_file('RGI_WGMS_oeztal.csv') cfg.PATHS['glathida_rgi_links'] = get_demo_file('RGI_GLATHIDA_oeztal.csv') # Read in the RGI file rgi_file = get_demo_file('rgi_oeztal.shp') rgidf = gpd.GeoDataFrame.from_file(rgi_file) # Params cfg.PARAMS['border'] = 70 cfg.PARAMS['use_inversion_params'] = True # Go gdirs = workflow.init_glacier_regions(rgidf) try: flowline.init_present_time_glacier(gdirs[0]) except Exception: reset = True if reset: # First preprocessing tasks workflow.gis_prepro_tasks(gdirs) # Climate related tasks # See if CRU is running cfg.PARAMS['temp_use_local_gradient'] = False cfg.PATHS['climate_file'] = '~' cru_dir = get_demo_file('cru_ts3.23.1901.2014.tmp.dat.nc') cfg.PATHS['cru_dir'] = os.path.dirname(cru_dir) workflow.climate_tasks(gdirs) # Use histalp for the actual test cfg.PARAMS['temp_use_local_gradient'] = True cfg.PATHS['climate_file'] = get_demo_file('HISTALP_oeztal.nc') cfg.PATHS['cru_dir'] = '~' workflow.climate_tasks(gdirs) # Inversion workflow.inversion_tasks(gdirs) return gdirs
def init_hef(reset=False, border=40, invert_with_sliding=True, invert_with_rectangular=True): from oggm.core.preprocessing import gis, centerlines, geometry from oggm.core.preprocessing import climate, inversion import oggm import oggm.cfg as cfg from oggm.utils import get_demo_file # test directory testdir = os.path.join(cfg.PATHS['test_dir'], 'tmp_border{}'.format(border)) if not invert_with_sliding: testdir += '_withoutslide' if not invert_with_rectangular: testdir += '_withoutrectangular' if not os.path.exists(testdir): os.makedirs(testdir) reset = True # Init cfg.initialize() cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif') cfg.PATHS['climate_file'] = get_demo_file('histalp_merged_hef.nc') cfg.PARAMS['border'] = border cfg.PARAMS['use_optimized_inversion_params'] = True hef_file = get_demo_file('Hintereisferner_RGI5.shp') entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0] gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=reset) if not gdir.has_file('inversion_params'): reset = True gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=reset) if not reset: return gdir gis.define_glacier_region(gdir, entity=entity) gis.glacier_masks(gdir) centerlines.compute_centerlines(gdir) centerlines.compute_downstream_lines(gdir) geometry.initialize_flowlines(gdir) centerlines.compute_downstream_bedshape(gdir) geometry.catchment_area(gdir) geometry.catchment_intersections(gdir) geometry.catchment_width_geom(gdir) geometry.catchment_width_correction(gdir) climate.process_histalp_nonparallel([gdir]) climate.mu_candidates(gdir, div_id=0) mbdf = gdir.get_ref_mb_data()['ANNUAL_BALANCE'] res = climate.t_star_from_refmb(gdir, mbdf) climate.local_mustar_apparent_mb(gdir, tstar=res['t_star'][-1], bias=res['bias'][-1], prcp_fac=res['prcp_fac']) inversion.prepare_for_inversion(gdir, add_debug_var=True, invert_with_rectangular=invert_with_rectangular) ref_v = 0.573 * 1e9 if invert_with_sliding: def to_optimize(x): # For backwards compat _fd = 1.9e-24 * x[0] glen_a = (cfg.N+2) * _fd / 2. fs = 5.7e-20 * x[1] v, _ = inversion.mass_conservation_inversion(gdir, fs=fs, glen_a=glen_a) return (v - ref_v)**2 out = optimization.minimize(to_optimize, [1, 1], bounds=((0.01, 10), (0.01, 10)), tol=1e-4)['x'] _fd = 1.9e-24 * out[0] glen_a = (cfg.N+2) * _fd / 2. fs = 5.7e-20 * out[1] v, _ = inversion.mass_conservation_inversion(gdir, fs=fs, glen_a=glen_a, write=True) else: def to_optimize(x): glen_a = cfg.A * x[0] v, _ = inversion.mass_conservation_inversion(gdir, fs=0., glen_a=glen_a) return (v - ref_v)**2 out = optimization.minimize(to_optimize, [1], bounds=((0.01, 10),), tol=1e-4)['x'] glen_a = cfg.A * out[0] fs = 0. v, _ = inversion.mass_conservation_inversion(gdir, fs=fs, glen_a=glen_a, write=True) d = dict(fs=fs, glen_a=glen_a) d['factor_glen_a'] = out[0] try: d['factor_fs'] = out[1] except IndexError: d['factor_fs'] = 0. gdir.write_pickle(d, 'inversion_params') # filter inversion.filter_inversion_output(gdir) inversion.distribute_thickness(gdir, how='per_altitude', add_nc_name=True) inversion.distribute_thickness(gdir, how='per_interpolation', add_slope=False, smooth=False, add_nc_name=True) return gdir
def compare(rgi_id, glacier_name): """ :param rgi_id: :param glacier_name: :return: """ # --------------------- # PREPROCESSING TASKS # --------------------- # create test directory wdir = os.path.join(os.path.abspath('.'), 'comparison_wdir') if not os.path.exists(wdir): os.makedirs(wdir) shutil.rmtree(wdir) os.makedirs(wdir) # load default parameter file cfg.initialize() # RGI entity # get/downlaod the rgi entity including the outline shapefile rgi_df = utils.get_rgi_glacier_entities([rgi_id]) # set name, since not delivered with RGI if rgi_df.loc[int(rgi_id[-5:])-1, 'Name'] is None: rgi_df.loc[int(rgi_id[-5:])-1, 'Name'] = glacier_name # select single entry rgi_entity = rgi_df.iloc[0] # GlacierDirectory # specify the working directory and define the glacier directory cfg.PATHS['working_dir'] = wdir gdir = oggm.GlacierDirectory(rgi_entity) # DEM and GIS tasks # get the path to the DEM file (will download if necessary) dem = utils.get_topo_file(gdir.cenlon, gdir.cenlat) # set path in config file cfg.PATHS['dem_file'] = dem[0][0] cfg.PARAMS['border'] = 10 cfg.PARAMS['use_intersects'] = False # run GIS tasks gis.define_glacier_region(gdir, entity=rgi_entity) gis.glacier_masks(gdir) # Climate data # using HistAlp cfg.PARAMS['baseline_climate'] = 'HISTALP' # climate records before 1850 are hardly reliable, which is not so drastic for # qualitative experiments (could be driven with random climate anyway) # cfg.PARAMS['baseline_y0'] = 1850 # change hyper parameters for HistAlp cfg.PARAMS['prcp_scaling_factor'] = 1.75 cfg.PARAMS['temp_melt'] = -1.75 # run climate task climate.process_histalp_data(gdir) # run center line preprocessing tasks centerlines.compute_centerlines(gdir) centerlines.initialize_flowlines(gdir) centerlines.compute_downstream_line(gdir) centerlines.compute_downstream_bedshape(gdir) centerlines.catchment_area(gdir) centerlines.catchment_intersections(gdir) centerlines.catchment_width_geom(gdir) centerlines.catchment_width_correction(gdir) # -------------------- # SCALING MODEL # -------------------- # compute local t* and the corresponding mu* vascaling.local_t_star(gdir) # instance the mass balance models vas_mb_mod = vascaling.VAScalingMassBalance(gdir) # get reference area a0 = gdir.rgi_area_m2 # get reference year y0 = gdir.read_pickle('climate_info')['baseline_hydro_yr_0'] y1 = gdir.read_pickle('climate_info')['baseline_hydro_yr_1'] # get min and max glacier surface elevation h0, h1 = vascaling.get_min_max_elevation(gdir) # instance VAS model vas_model = vascaling.VAScalingModel(year_0=y0, area_m2_0=a0, min_hgt=h0, max_hgt=h1, mb_model=vas_mb_mod) # run model over all HistAlp climate period vas_df = vas_model.run_and_store(y1, reset=True) # get relevant parameters years_vas = vas_df.index.values length_m_vas = vas_df.length_m.values area_m2_vas = vas_df.area_m2.values volume_m3_vas = vas_df.volume_m3.values # ------ # OGGM # ------ # compute local t* and the corresponding mu* climate.local_t_star(gdir) climate.mu_star_calibration(gdir) # instance the mass balance models mb_mod = massbalance.PastMassBalance(gdir) # run inversion tasks inversion.prepare_for_inversion(gdir) inversion.mass_conservation_inversion(gdir) inversion.filter_inversion_output(gdir) # initialize present time glacier flowline.init_present_time_glacier(gdir) # instance flowline model fls = gdir.read_pickle('model_flowlines') y0 = gdir.read_pickle('climate_info')['baseline_hydro_yr_0'] y1 = gdir.read_pickle('climate_info')['baseline_hydro_yr_1'] fl_mod = flowline.FluxBasedModel(flowlines=fls, mb_model=mb_mod, y0=y0) # run model and store output as xarray data set _, oggm_ds = fl_mod.run_until_and_store(y1) years_oggm = oggm_ds.hydro_year.values # annual index must be equal np.testing.assert_array_equal(years_oggm, years_vas) length_m_oggm = oggm_ds.length_m.values area_m2_oggm = oggm_ds.area_m2.values volume_m3_oggm = oggm_ds.volume_m3.values # define column names for DataFrame names = ['length_vas', 'length_oggm', 'area_vas', 'area_oggm', 'volume_vas', 'volume_oggm'] # combine glacier geometries into DataFrame df = pd.DataFrame(np.array([length_m_vas, length_m_oggm, area_m2_vas, area_m2_oggm, volume_m3_vas, volume_m3_oggm]).T, index=years_vas, columns=names) # save to file store = True if store: # define path and file names folder = '/Users/oberrauch/work/master/data/' df.to_csv(folder+'run_comparison.csv') def plot_both(vas_df, oggm_df, ref=None, correct_bias=False, title='', ylabel='', file_path='', exp=0): """ Plot geometric parameters of both models. If a `file_path` is given, the figure will be saved. :param vas_df: (pandas.Series) geometric glacier parameter of the VAS model :param oggm_df: (pandas.Series) geometric glacier parameter of the OGGM :param ref: (pandas.Series) measured glacier parameter, optional :param title: (string) figure title, optional :param ylabel: (string) label for y-axis, optional :param file_path: (string) where to store the figure, optional :param exp: (int) exponent for labels in scientific notation, optional """ beamer = True if beamer: mpl.rc('axes', titlesize=18) mpl.rc('axes', labelsize=14) mpl.rc('xtick', labelsize=14) mpl.rc('ytick', labelsize=14) mpl.rc('legend', fontsize=10) # create figure and first axes fig = plt.figure(figsize=[6, 4]) ax = fig.add_axes([0.15, 0.1, 0.8, 0.8]) # define colors c1 = 'C0' c2 = 'C1' c3 = 'C3' # plot vas and OGGM parameters ax.plot(oggm_df.index, oggm_df.values, c=c2, label='OGGM') ax.plot(vas_df.index, vas_df.values, c=c1, label='VAS') if ref: # plot reference parameter if given ax.plot(ref.index, ref.values, c=c3, label='measurements') if correct_bias: # plot bias corrected vas df_ = pd.DataFrame([oggm_df, vas_df]).T bias = vas_df.values - df_.mean().diff().iloc[1] ax.plot(vas_df.index, bias, c=c1, ls='--', label='VAS, bias corrected') # add RMSD as text ax.text(0.05, 0.05, 'RMSD: {:.1e}'.format(utils.rmsd(oggm_df, bias)), transform=plt.gca().transAxes) # add correlation coefficient as text ax.text(0.05, 0.11, 'Corr. Coef.: {:.2f}'.format( utils.corrcoef(oggm_df, vas_df)), transform=plt.gca().transAxes) # add title, labels, legend ax.set_title(title) ax.set_ylabel(ylabel) ax.legend() import matplotlib.ticker class OOMFormatter(matplotlib.ticker.ScalarFormatter): def __init__(self, order=0, fformat="%1.1f", offset=False, mathText=False): self.oom = order self.fformat = fformat matplotlib.ticker.ScalarFormatter.__init__(self, useOffset=offset, useMathText=mathText) def _set_orderOfMagnitude(self, nothing): self.orderOfMagnitude = self.oom def _set_format(self, vmin, vmax): self.format = self.fformat if self._useMathText: self.format = '$%s$' % matplotlib.ticker._mathdefault(self.format) # use scientific notation with fixed exponent according ax.yaxis.set_major_formatter(OOMFormatter(exp, "%1.2f")) # store to file if file_path: plt.savefig(file_path, bbox_inches='tight', format=file_path.split('.')[-1]) # specify plot directory folder = '/Users/oberrauch/work/master/plots/' # plot length plot_both(df.length_vas, df.length_oggm, correct_bias=True, title='Glacier length - {}'.format(glacier_name), ylabel=r'Length [m]', file_path=os.path.join(folder, '{}_length.pdf'.format(rgi_id)), exp=3) # plot area plot_both(df.area_vas, df.area_oggm, correct_bias=True, title='Surface area - {}'.format(glacier_name), ylabel=r'Area [m$^2$]', file_path=os.path.join(folder, '{}_area.pdf'.format(rgi_id)), exp=6) # plot volume plot_both(df.volume_vas, df.volume_oggm, correct_bias=True, title='Glacier volume - {}'.format(glacier_name), ylabel=r'Volume [m$^3$]', file_path=os.path.join(folder, '{}_volume.pdf'.format(rgi_id)), exp=9)
def init_hef(reset=False, border=40, invert_with_sliding=True): # test directory testdir = TESTDIR_BASE + '_border{}'.format(border) if not invert_with_sliding: testdir += '_withoutslide' if not os.path.exists(testdir): os.makedirs(testdir) reset = True if not os.path.exists(os.path.join(testdir, 'RGI40-11.00897')): reset = True if not os.path.exists(os.path.join(testdir, 'RGI40-11.00897', 'inversion_params.pkl')): reset = True # Init cfg.initialize() cfg.set_divides_db(get_demo_file('HEF_divided.shp')) cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif') cfg.PATHS['climate_file'] = get_demo_file('histalp_merged_hef.nc') cfg.PARAMS['border'] = border # loop because for some reason indexing wont work hef_file = get_demo_file('Hintereisferner.shp') rgidf = gpd.GeoDataFrame.from_file(hef_file) for index, entity in rgidf.iterrows(): gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=reset) if not reset: return gdir gis.define_glacier_region(gdir, entity=entity) gis.glacier_masks(gdir) centerlines.compute_centerlines(gdir) centerlines.compute_downstream_lines(gdir) geometry.initialize_flowlines(gdir) geometry.catchment_area(gdir) geometry.catchment_width_geom(gdir) geometry.catchment_width_correction(gdir) climate.distribute_climate_data([gdir]) climate.mu_candidates(gdir, div_id=0) hef_file = get_demo_file('mbdata_RGI40-11.00897.csv') mbdf = pd.read_csv(hef_file).set_index('YEAR') t_star, bias = climate.t_star_from_refmb(gdir, mbdf['ANNUAL_BALANCE']) climate.local_mustar_apparent_mb(gdir, tstar=t_star[-1], bias=bias[-1]) inversion.prepare_for_inversion(gdir) ref_v = 0.573 * 1e9 if invert_with_sliding: def to_optimize(x): # For backwards compat _fd = 1.9e-24 * x[0] glen_a = (cfg.N+2) * _fd / 2. fs = 5.7e-20 * x[1] v, _ = inversion.invert_parabolic_bed(gdir, fs=fs, glen_a=glen_a) return (v - ref_v)**2 import scipy.optimize as optimization out = optimization.minimize(to_optimize, [1, 1], bounds=((0.01, 10), (0.01, 10)), tol=1e-4)['x'] _fd = 1.9e-24 * out[0] glen_a = (cfg.N+2) * _fd / 2. fs = 5.7e-20 * out[1] v, _ = inversion.invert_parabolic_bed(gdir, fs=fs, glen_a=glen_a, write=True) else: def to_optimize(x): glen_a = cfg.A * x[0] v, _ = inversion.invert_parabolic_bed(gdir, fs=0., glen_a=glen_a) return (v - ref_v)**2 import scipy.optimize as optimization out = optimization.minimize(to_optimize, [1], bounds=((0.01, 10),), tol=1e-4)['x'] glen_a = cfg.A * out[0] fs = 0. v, _ = inversion.invert_parabolic_bed(gdir, fs=fs, glen_a=glen_a, write=True) d = dict(fs=fs, glen_a=glen_a) d['factor_glen_a'] = out[0] try: d['factor_fs'] = out[1] except IndexError: d['factor_fs'] = 0. gdir.write_pickle(d, 'inversion_params') inversion.distribute_thickness(gdir, how='per_altitude', add_nc_name=True) inversion.distribute_thickness(gdir, how='per_interpolation', add_slope=False, smooth=False, add_nc_name=True) return gdir
def init_hef(reset=False, border=40, invert_with_sliding=True): from oggm.core.preprocessing import gis, centerlines, geometry from oggm.core.preprocessing import climate, inversion import oggm import oggm.cfg as cfg from oggm.utils import get_demo_file # test directory testdir = TESTDIR_BASE + '_border{}'.format(border) if not invert_with_sliding: testdir += '_withoutslide' if not os.path.exists(testdir): os.makedirs(testdir) reset = True if not os.path.exists(os.path.join(testdir, 'RGI40-11.00897')): reset = True if not os.path.exists(os.path.join(testdir, 'RGI40-11.00897', 'inversion_params.pkl')): reset = True # Init cfg.initialize() cfg.PATHS['dem_file'] = get_demo_file('hef_srtm.tif') cfg.PATHS['climate_file'] = get_demo_file('histalp_merged_hef.nc') cfg.PARAMS['border'] = border hef_file = get_demo_file('Hintereisferner.shp') entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0] gdir = oggm.GlacierDirectory(entity, base_dir=testdir, reset=reset) if not reset: return gdir gis.define_glacier_region(gdir, entity=entity) gis.glacier_masks(gdir) centerlines.compute_centerlines(gdir) centerlines.compute_downstream_lines(gdir) geometry.initialize_flowlines(gdir) geometry.catchment_area(gdir) geometry.catchment_width_geom(gdir) geometry.catchment_width_correction(gdir) climate.process_histalp_nonparallel([gdir]) climate.mu_candidates(gdir, div_id=0) mbdf = gdir.get_ref_mb_data()['ANNUAL_BALANCE'] res = climate.t_star_from_refmb(gdir, mbdf) climate.local_mustar_apparent_mb(gdir, tstar=res['t_star'][-1], bias=res['bias'][-1], prcp_fac=res['prcp_fac']) inversion.prepare_for_inversion(gdir) ref_v = 0.573 * 1e9 if invert_with_sliding: def to_optimize(x): # For backwards compat _fd = 1.9e-24 * x[0] glen_a = (cfg.N+2) * _fd / 2. fs = 5.7e-20 * x[1] v, _ = inversion.invert_parabolic_bed(gdir, fs=fs, glen_a=glen_a) return (v - ref_v)**2 out = optimization.minimize(to_optimize, [1, 1], bounds=((0.01, 10), (0.01, 10)), tol=1e-4)['x'] _fd = 1.9e-24 * out[0] glen_a = (cfg.N+2) * _fd / 2. fs = 5.7e-20 * out[1] v, _ = inversion.invert_parabolic_bed(gdir, fs=fs, glen_a=glen_a, write=True) else: def to_optimize(x): glen_a = cfg.A * x[0] v, _ = inversion.invert_parabolic_bed(gdir, fs=0., glen_a=glen_a) return (v - ref_v)**2 out = optimization.minimize(to_optimize, [1], bounds=((0.01, 10),), tol=1e-4)['x'] glen_a = cfg.A * out[0] fs = 0. v, _ = inversion.invert_parabolic_bed(gdir, fs=fs, glen_a=glen_a, write=True) d = dict(fs=fs, glen_a=glen_a) d['factor_glen_a'] = out[0] try: d['factor_fs'] = out[1] except IndexError: d['factor_fs'] = 0. gdir.write_pickle(d, 'inversion_params') inversion.distribute_thickness(gdir, how='per_altitude', add_nc_name=True) inversion.distribute_thickness(gdir, how='per_interpolation', add_slope=False, smooth=False, add_nc_name=True) return gdir
def run_benchmark(rgi_version=None, rgi_reg=None, border=None, output_folder='', working_dir='', is_test=False, test_rgidf=None, test_intersects_file=None, test_topofile=None, test_crudir=None): """Does the actual job. Parameters ---------- rgi_version : str the RGI version to use (defaults to cfg.PARAMS) rgi_reg : str the RGI region to process border : int the number of pixels at the maps border output_folder : str path to the output folder (where to put the preprocessed tar files) working_dir : str path to the OGGM working directory is_test : bool to test on a couple of glaciers only! test_rgidf : shapefile for testing purposes only test_intersects_file : shapefile for testing purposes only test_topofile : str for testing purposes only test_crudir : str for testing purposes only """ # TODO: temporarily silence Fiona deprecation warnings import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) # Module logger log = logging.getLogger(__name__) # Initialize OGGM and set up the run parameters cfg.initialize(logging_level='WORKFLOW') # Local paths utils.mkdir(working_dir) cfg.PATHS['working_dir'] = working_dir # Use multiprocessing? cfg.PARAMS['use_multiprocessing'] = True # How many grid points around the glacier? # Make it large if you expect your glaciers to grow large cfg.PARAMS['border'] = border # Set to True for operational runs cfg.PARAMS['continue_on_error'] = True # For statistics odf = pd.DataFrame() if rgi_version is None: rgi_version = cfg.PARAMS['rgi_version'] base_dir = os.path.join(output_folder) # Add a package version file utils.mkdir(base_dir) opath = os.path.join(base_dir, 'package_versions.txt') with open(opath, 'w') as vfile: vfile.write(utils.show_versions(logger=log)) # Read RGI start = time.time() if test_rgidf is None: # Get the RGI file rgidf = gpd.read_file(utils.get_rgi_region_file(rgi_reg, version=rgi_version)) # We use intersects rgif = utils.get_rgi_intersects_region_file(rgi_reg, version=rgi_version) cfg.set_intersects_db(rgif) else: rgidf = test_rgidf cfg.set_intersects_db(test_intersects_file) if is_test: # Just for fun rgidf = rgidf.sample(2) _add_time_to_df(odf, 'Read RGI', time.time()-start) # Sort for more efficient parallel computing rgidf = rgidf.sort_values('Area', ascending=False) log.workflow('Starting prepro run for RGI reg: {} ' 'and border: {}'.format(rgi_reg, border)) log.workflow('Number of glaciers: {}'.format(len(rgidf))) # Input if test_topofile: cfg.PATHS['dem_file'] = test_topofile # Initialize working directories start = time.time() gdirs = workflow.init_glacier_regions(rgidf, reset=True, force=True) _add_time_to_df(odf, 'init_glacier_regions', time.time()-start) # Pre-download other files just in case if test_crudir is None: _ = utils.get_cru_file(var='tmp') _ = utils.get_cru_file(var='pre') else: cfg.PATHS['cru_dir'] = test_crudir # Tasks task_list = [ tasks.process_cru_data, tasks.glacier_masks, tasks.compute_centerlines, tasks.initialize_flowlines, tasks.compute_downstream_line, tasks.compute_downstream_bedshape, tasks.catchment_area, tasks.catchment_intersections, tasks.catchment_width_geom, tasks.catchment_width_correction, tasks.local_t_star, tasks.mu_star_calibration, tasks.prepare_for_inversion, tasks.mass_conservation_inversion, tasks.filter_inversion_output, tasks.init_present_time_glacier, ] for task in task_list: start = time.time() workflow.execute_entity_task(task, gdirs) _add_time_to_df(odf, task.__name__, time.time()-start) # Runs start = time.time() workflow.execute_entity_task(tasks.run_random_climate, gdirs, nyears=250, bias=0, seed=0, output_filesuffix='_tstar') _add_time_to_df(odf, 'run_random_climate_tstar_250', time.time()-start) start = time.time() workflow.execute_entity_task(tasks.run_random_climate, gdirs, nyears=250, y0=1995, seed=0, output_filesuffix='_commit') _add_time_to_df(odf, 'run_random_climate_commit_250', time.time()-start) # Compile results start = time.time() utils.compile_glacier_statistics(gdirs) _add_time_to_df(odf, 'compile_glacier_statistics', time.time()-start) start = time.time() utils.compile_climate_statistics(gdirs, add_climate_period=[1920, 1960, 2000]) _add_time_to_df(odf, 'compile_climate_statistics', time.time()-start) start = time.time() utils.compile_run_output(gdirs, filesuffix='_tstar') _add_time_to_df(odf, 'compile_run_output_tstar', time.time()-start) start = time.time() utils.compile_run_output(gdirs, filesuffix='_commit') _add_time_to_df(odf, 'compile_run_output_commit', time.time()-start) # Log opath = os.path.join(base_dir, 'benchmarks_b{:03d}.csv'.format(border)) odf.index.name = 'Task' odf.to_csv(opath) log.workflow('OGGM benchmarks is done!')
import geopandas as gpd import matplotlib.pyplot as plt import salem # Locals import oggm.cfg as cfg from oggm import workflow from oggm import tasks from oggm.workflow import execute_entity_task from oggm import graphics, utils from oggm.core.models import flowline # Initialize OGGM and set up the run parameters # --------------------------------------------- cfg.initialize() # Local paths (where to write the OGGM run output) WORKING_DIR = '/home/mowglie/disk/OGGM_Runs/EXAMPLE_GLACIERS' PLOTS_DIR = os.path.join(WORKING_DIR, 'plots') utils.mkdir(WORKING_DIR) cfg.PATHS['working_dir'] = WORKING_DIR # Use multiprocessing? cfg.PARAMS['use_multiprocessing'] = True # How many grid points around the glacier? # Make it large if you expect your glaciers to grow large cfg.PARAMS['border'] = 60 # This is the default in OGGM
def run_prepro_levels(rgi_version=None, rgi_reg=None, border=None, output_folder='', working_dir='', dem_source='', is_test=False, test_ids=None, demo=False, test_rgidf=None, test_intersects_file=None, test_topofile=None, disable_mp=False, params_file=None, elev_bands=False, match_geodetic_mb=False, centerlines_only=False, add_consensus=False, start_level=None, start_base_url=None, max_level=5, logging_level='WORKFLOW', disable_dl_verify=False): """Does the actual job. Parameters ---------- rgi_version : str the RGI version to use (defaults to cfg.PARAMS) rgi_reg : str the RGI region to process border : int the number of pixels at the maps border output_folder : str path to the output folder (where to put the preprocessed tar files) dem_source : str which DEM source to use: default, SOURCE_NAME or ALL working_dir : str path to the OGGM working directory params_file : str path to the OGGM parameter file (to override defaults) is_test : bool to test on a couple of glaciers only! test_ids : list if is_test: list of ids to process demo : bool to run the prepro for the list of demo glaciers test_rgidf : shapefile for testing purposes only test_intersects_file : shapefile for testing purposes only test_topofile : str for testing purposes only test_crudir : str for testing purposes only disable_mp : bool disable multiprocessing elev_bands : bool compute all flowlines based on the Huss&Hock 2015 method instead of the OGGM default, which is a mix of elev_bands and centerlines. centerlines_only : bool compute all flowlines based on the OGGM centerline(s) method instead of the OGGM default, which is a mix of elev_bands and centerlines. match_geodetic_mb : bool match the regional mass-balance estimates at the regional level (currently Hugonnet et al., 2020). add_consensus : bool adds (reprojects) the consensus estimates thickness to the glacier directories. With elev_bands=True, the data will also be binned. start_level : int the pre-processed level to start from (default is to start from scratch). If set, you'll need to indicate start_base_url as well. start_base_url : str the pre-processed base-url to fetch the data from. max_level : int the maximum pre-processing level before stopping logging_level : str the logging level to use (DEBUG, INFO, WARNING, WORKFLOW) disable_dl_verify : bool disable the hash verification of OGGM downloads """ # TODO: temporarily silence Fiona and other deprecation warnings import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) # Input check if max_level not in [1, 2, 3, 4, 5]: raise InvalidParamsError('max_level should be one of [1, 2, 3, 4, 5]') if start_level is not None: if start_level not in [0, 1, 2]: raise InvalidParamsError('start_level should be one of [0, 1, 2]') if start_level > 0 and start_base_url is None: raise InvalidParamsError('With start_level, please also indicate ' 'start_base_url') else: start_level = 0 # Time start = time.time() def _time_log(): # Log util m, s = divmod(time.time() - start, 60) h, m = divmod(m, 60) log.workflow('OGGM prepro_levels is done! Time needed: ' '{:02d}:{:02d}:{:02d}'.format(int(h), int(m), int(s))) # Config Override Params params = {} # Local paths utils.mkdir(working_dir) params['working_dir'] = working_dir # Initialize OGGM and set up the run parameters cfg.initialize(file=params_file, params=params, logging_level=logging_level, future=True) # Use multiprocessing? cfg.PARAMS['use_multiprocessing'] = not disable_mp # How many grid points around the glacier? # Make it large if you expect your glaciers to grow large cfg.PARAMS['border'] = border # Set to True for operational runs cfg.PARAMS['continue_on_error'] = True # Check for the integrity of the files OGGM downloads at run time # For large files (e.g. using a 1 tif DEM like ALASKA) calculating the hash # takes a long time, so deactivating this can make sense cfg.PARAMS['dl_verify'] = not disable_dl_verify # Log the parameters msg = '# OGGM Run parameters:' for k, v in cfg.PARAMS.items(): if type(v) in [pd.DataFrame, dict]: continue msg += '\n {}: {}'.format(k, v) log.workflow(msg) if rgi_version is None: rgi_version = cfg.PARAMS['rgi_version'] output_base_dir = os.path.join(output_folder, 'RGI{}'.format(rgi_version), 'b_{:03d}'.format(border)) # Add a package version file utils.mkdir(output_base_dir) opath = os.path.join(output_base_dir, 'package_versions.txt') with open(opath, 'w') as vfile: vfile.write(utils.show_versions(logger=log)) if demo: rgidf = utils.get_rgi_glacier_entities(cfg.DATA['demo_glaciers'].index) elif test_rgidf is None: # Get the RGI file rgidf = gpd.read_file( utils.get_rgi_region_file(rgi_reg, version=rgi_version)) # We use intersects rgif = utils.get_rgi_intersects_region_file(rgi_reg, version=rgi_version) cfg.set_intersects_db(rgif) # Some RGI input quality checks - this is based on visual checks # of large glaciers in the RGI ids_to_ice_cap = [ 'RGI60-05.10315', # huge Greenland ice cap 'RGI60-03.01466', # strange thing next to Devon 'RGI60-09.00918', # Academy of sciences Ice cap 'RGI60-09.00969', 'RGI60-09.00958', 'RGI60-09.00957', ] rgidf.loc[rgidf.RGIId.isin(ids_to_ice_cap), 'Form'] = '1' # In AA almost all large ice bodies are actually ice caps if rgi_reg == '19': rgidf.loc[rgidf.Area > 100, 'Form'] = '1' # For greenland we omit connectivity level 2 if rgi_reg == '05': rgidf = rgidf.loc[rgidf['Connect'] != 2] else: rgidf = test_rgidf cfg.set_intersects_db(test_intersects_file) if is_test: if test_ids is not None: rgidf = rgidf.loc[rgidf.RGIId.isin(test_ids)] else: rgidf = rgidf.sample(4) log.workflow('Starting prepro run for RGI reg: {} ' 'and border: {}'.format(rgi_reg, border)) log.workflow('Number of glaciers: {}'.format(len(rgidf))) # L0 - go if start_level == 0: gdirs = workflow.init_glacier_directories(rgidf, reset=True, force=True) # Glacier stats sum_dir = os.path.join(output_base_dir, 'L0', 'summary') utils.mkdir(sum_dir) opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg)) utils.compile_glacier_statistics(gdirs, path=opath) # L0 OK - compress all in output directory log.workflow('L0 done. Writing to tar...') level_base_dir = os.path.join(output_base_dir, 'L0') workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False, base_dir=level_base_dir) utils.base_dir_to_tar(level_base_dir) if max_level == 0: _time_log() return else: gdirs = workflow.init_glacier_directories( rgidf, reset=True, force=True, from_prepro_level=start_level, prepro_border=border, prepro_rgi_version=rgi_version, prepro_base_url=start_base_url) # L1 - Add dem files if start_level == 0: if test_topofile: cfg.PATHS['dem_file'] = test_topofile # Which DEM source? if dem_source.upper() == 'ALL': # This is the complex one, just do the job and leave log.workflow('Running prepro on ALL sources') for i, s in enumerate(utils.DEM_SOURCES): rs = i == 0 log.workflow('Running prepro on sources: {}'.format(s)) gdirs = workflow.init_glacier_directories(rgidf, reset=rs, force=rs) workflow.execute_entity_task(tasks.define_glacier_region, gdirs, source=s) workflow.execute_entity_task(_rename_dem_folder, gdirs, source=s) # make a GeoTiff mask of the glacier, choose any source workflow.execute_entity_task(gis.rasterio_glacier_mask, gdirs, source='ALL') # Compress all in output directory level_base_dir = os.path.join(output_base_dir, 'L1') workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False, base_dir=level_base_dir) utils.base_dir_to_tar(level_base_dir) _time_log() return # Force a given source source = dem_source.upper() if dem_source else None # L1 - go workflow.execute_entity_task(tasks.define_glacier_region, gdirs, source=source) # Glacier stats sum_dir = os.path.join(output_base_dir, 'L1', 'summary') utils.mkdir(sum_dir) opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg)) utils.compile_glacier_statistics(gdirs, path=opath) # L1 OK - compress all in output directory log.workflow('L1 done. Writing to tar...') level_base_dir = os.path.join(output_base_dir, 'L1') workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False, base_dir=level_base_dir) utils.base_dir_to_tar(level_base_dir) if max_level == 1: _time_log() return # L2 - Tasks if start_level <= 1: # Check which glaciers will be processed as what if elev_bands: gdirs_band = gdirs gdirs_cent = [] elif centerlines_only: gdirs_band = [] gdirs_cent = gdirs else: # Default is to centerlines_only, but it used to be a mix # (e.g. bands for ice caps, etc) # I still keep this logic here in case we want to mix again gdirs_band = [] gdirs_cent = gdirs log.workflow('Start flowline processing with: ' 'N centerline type: {}, ' 'N elev bands type: {}.' ''.format(len(gdirs_cent), len(gdirs_band))) # HH2015 method workflow.execute_entity_task(tasks.simple_glacier_masks, gdirs_band) # Centerlines OGGM workflow.execute_entity_task(tasks.glacier_masks, gdirs_cent) if add_consensus: from oggm.shop.bedtopo import add_consensus_thickness workflow.execute_entity_task(add_consensus_thickness, gdirs_band) workflow.execute_entity_task(add_consensus_thickness, gdirs_cent) # Elev bands with var data vn = 'consensus_ice_thickness' workflow.execute_entity_task(tasks.elevation_band_flowline, gdirs_band, bin_variables=vn) workflow.execute_entity_task( tasks.fixed_dx_elevation_band_flowline, gdirs_band, bin_variables=vn) else: # HH2015 method without it task_list = [ tasks.elevation_band_flowline, tasks.fixed_dx_elevation_band_flowline, ] for task in task_list: workflow.execute_entity_task(task, gdirs_band) # Centerlines OGGM task_list = [ tasks.compute_centerlines, tasks.initialize_flowlines, tasks.catchment_area, tasks.catchment_intersections, tasks.catchment_width_geom, tasks.catchment_width_correction, ] for task in task_list: workflow.execute_entity_task(task, gdirs_cent) # Same for all glaciers if border >= 20: task_list = [ tasks.compute_downstream_line, tasks.compute_downstream_bedshape, ] for task in task_list: workflow.execute_entity_task(task, gdirs) else: log.workflow('L2: for map border values < 20, wont compute ' 'downstream lines.') # Glacier stats sum_dir = os.path.join(output_base_dir, 'L2', 'summary') utils.mkdir(sum_dir) opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg)) utils.compile_glacier_statistics(gdirs, path=opath) # L2 OK - compress all in output directory log.workflow('L2 done. Writing to tar...') level_base_dir = os.path.join(output_base_dir, 'L2') workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False, base_dir=level_base_dir) utils.base_dir_to_tar(level_base_dir) if max_level == 2: _time_log() return # L3 - Tasks # Climate workflow.execute_entity_task(tasks.process_climate_data, gdirs) if cfg.PARAMS['climate_qc_months'] > 0: workflow.execute_entity_task(tasks.historical_climate_qc, gdirs) workflow.execute_entity_task(tasks.local_t_star, gdirs) workflow.execute_entity_task(tasks.mu_star_calibration, gdirs) # Inversion: we match the consensus workflow.calibrate_inversion_from_consensus(gdirs, apply_fs_on_mismatch=True, error_on_mismatch=False) # Do we want to match geodetic estimates? # This affects only the bias so we can actually do this *after* # the inversion, but we really want to take calving into account here if match_geodetic_mb: opath = os.path.join( sum_dir, 'fixed_geometry_mass_balance_' 'before_match_{}.csv'.format(rgi_reg)) utils.compile_fixed_geometry_mass_balance(gdirs, path=opath) workflow.match_regional_geodetic_mb(gdirs, rgi_reg) # We get ready for modelling if border >= 20: workflow.execute_entity_task(tasks.init_present_time_glacier, gdirs) else: log.workflow('L3: for map border values < 20, wont initalize glaciers ' 'for the run.') # Glacier stats sum_dir = os.path.join(output_base_dir, 'L3', 'summary') utils.mkdir(sum_dir) opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg)) utils.compile_glacier_statistics(gdirs, path=opath) opath = os.path.join(sum_dir, 'climate_statistics_{}.csv'.format(rgi_reg)) utils.compile_climate_statistics(gdirs, path=opath) opath = os.path.join(sum_dir, 'fixed_geometry_mass_balance_{}.csv'.format(rgi_reg)) utils.compile_fixed_geometry_mass_balance(gdirs, path=opath) # L3 OK - compress all in output directory log.workflow('L3 done. Writing to tar...') level_base_dir = os.path.join(output_base_dir, 'L3') workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False, base_dir=level_base_dir) utils.base_dir_to_tar(level_base_dir) if max_level == 3: _time_log() return if border < 20: log.workflow('L3: for map border values < 20, wont compute L4 and L5.') _time_log() return # L4 - No tasks: add some stats for consistency and make the dirs small sum_dir_L3 = sum_dir sum_dir = os.path.join(output_base_dir, 'L4', 'summary') utils.mkdir(sum_dir) # Copy L3 files for consistency for bn in [ 'glacier_statistics', 'climate_statistics', 'fixed_geometry_mass_balance' ]: ipath = os.path.join(sum_dir_L3, bn + '_{}.csv'.format(rgi_reg)) opath = os.path.join(sum_dir, bn + '_{}.csv'.format(rgi_reg)) shutil.copyfile(ipath, opath) # Copy mini data to new dir mini_base_dir = os.path.join(working_dir, 'mini_perglacier') mini_gdirs = workflow.execute_entity_task(tasks.copy_to_basedir, gdirs, base_dir=mini_base_dir) # L4 OK - compress all in output directory log.workflow('L4 done. Writing to tar...') level_base_dir = os.path.join(output_base_dir, 'L4') workflow.execute_entity_task(utils.gdir_to_tar, mini_gdirs, delete=False, base_dir=level_base_dir) utils.base_dir_to_tar(level_base_dir) if max_level == 4: _time_log() return # L5 - spinup run in mini gdirs gdirs = mini_gdirs # Get end date. The first gdir might have blown up, try some others i = 0 while True: if i >= len(gdirs): raise RuntimeError('Found no valid glaciers!') try: y0 = gdirs[i].get_climate_info()['baseline_hydro_yr_0'] # One adds 1 because the run ends at the end of the year ye = gdirs[i].get_climate_info()['baseline_hydro_yr_1'] + 1 break except BaseException: i += 1 # OK - run workflow.execute_entity_task(tasks.run_from_climate_data, gdirs, min_ys=y0, ye=ye, output_filesuffix='_historical') # Now compile the output sum_dir = os.path.join(output_base_dir, 'L5', 'summary') utils.mkdir(sum_dir) opath = os.path.join(sum_dir, 'historical_run_output_{}.nc'.format(rgi_reg)) utils.compile_run_output(gdirs, path=opath, input_filesuffix='_historical') # Glacier statistics we recompute here for error analysis opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg)) utils.compile_glacier_statistics(gdirs, path=opath) # Other stats for consistency for bn in ['climate_statistics', 'fixed_geometry_mass_balance']: ipath = os.path.join(sum_dir_L3, bn + '_{}.csv'.format(rgi_reg)) opath = os.path.join(sum_dir, bn + '_{}.csv'.format(rgi_reg)) shutil.copyfile(ipath, opath) # Add the extended files pf = os.path.join(sum_dir, 'historical_run_output_{}.nc'.format(rgi_reg)) mf = os.path.join(sum_dir, 'fixed_geometry_mass_balance_{}.csv'.format(rgi_reg)) # This is crucial - extending calving only possible with L3 data!!! sf = os.path.join(sum_dir_L3, 'glacier_statistics_{}.csv'.format(rgi_reg)) opath = os.path.join( sum_dir, 'historical_run_output_extended_{}.nc'.format(rgi_reg)) utils.extend_past_climate_run(past_run_file=pf, fixed_geometry_mb_file=mf, glacier_statistics_file=sf, path=opath) # L5 OK - compress all in output directory log.workflow('L5 done. Writing to tar...') level_base_dir = os.path.join(output_base_dir, 'L5') workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False, base_dir=level_base_dir) utils.base_dir_to_tar(level_base_dir) _time_log()