def initialize(file=None, logging_level='INFO', params=None, future=False): """Read the configuration file containing the run's parameters. This should be the first call, before using any of the other OGGM modules for most (all?) OGGM simulations. Parameters ---------- file : str path to the configuration file (default: OGGM params.cfg) logging_level : str set a logging level. See :func:`set_logging_config` for options. params : dict overrides for specific parameters from the config file future : bool use the new behavior of logging='WORKFLOW'. """ global PARAMS global DATA initialize_minimal(file=file, logging_level=logging_level, params=params, future=future) # Do not spam PARAMS.do_log = False # Make sure we have a proper cache dir from oggm.utils import download_oggm_files, get_demo_file download_oggm_files() # Read in the demo glaciers file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'demo_glaciers.csv') DATA['demo_glaciers'] = pd.read_csv(file, index_col=0) # Add other things if 'dem_grids' not in DATA: grids = {} for grid_json in [ 'gimpdem_90m_v01.1.json', 'arcticdem_mosaic_100m_v3.0.json', 'Alaska_albers_V3.json', 'AntarcticDEM_wgs84.json', 'REMA_100m_dem.json' ]: if grid_json not in grids: fp = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', grid_json) try: grids[grid_json] = salem.Grid.from_json(fp) except NameError: pass DATA['dem_grids'] = grids # Trigger a one time check of the hash file from oggm.utils import get_dl_verify_data get_dl_verify_data('dummy_section') # OK PARAMS.do_log = True
def init_glacier_directories(rgidf=None, *, reset=False, force=False, from_prepro_level=None, prepro_border=None, prepro_rgi_version=None, prepro_base_url=None, from_tar=False, delete_tar=False, use_demo_glaciers=None): """Initializes the list of Glacier Directories for this run. This is the very first task to do (always). If the directories are already available in the working directory, use them. If not, create new ones. Parameters ---------- rgidf : GeoDataFrame or list of ids, optional for pre-computed runs the RGI glacier outlines. If unavailable, OGGM will parse the information from the glacier directories found in the working directory. It is required for new runs. reset : bool delete the existing glacier directories if found. force : bool setting `reset=True` will trigger a yes/no question to the user. Set `force=True` to avoid this. from_prepro_level : int get the gdir data from the official pre-processed pool. See the documentation for more information prepro_border : int for `from_prepro_level` only: if you want to override the default behavior which is to use `cfg.PARAMS['border']` prepro_rgi_version : str for `from_prepro_level` only: if you want to override the default behavior which is to use `cfg.PARAMS['rgi_version']` prepro_base_url : str for `from_prepro_level` only: if you want to override the default URL from which to download the gdirs. Default currently is https://cluster.klima.uni-bremen.de/~fmaussion/gdirs/oggm_v1.1/ use_demo_glaciers : bool whether to check the demo glaciers for download (faster than the standard prepro downloads). The default is to decide whether or not to check based on simple criteria such as glacier list size. from_tar : bool, default=False extract the gdir data from a tar file. If set to `True`, will check for a tar file at the expected location in `base_dir`. delete_tar : bool, default=False delete the original tar file after extraction. Returns ------- gdirs : list of :py:class:`oggm.GlacierDirectory` objects the initialised glacier directories Notes ----- This task is very similar to init_glacier_regions, with one main difference: it does not process the DEMs for this glacier. Eventually, init_glacier_regions will be deprecated and removed from the codebase. """ _check_duplicates(rgidf) if reset and not force: reset = utils.query_yes_no('Delete all glacier directories?') if from_prepro_level: url = utils.get_prepro_base_url(base_url=prepro_base_url, border=prepro_border, prepro_level=from_prepro_level, rgi_version=prepro_rgi_version) if cfg.PARAMS['has_internet'] and not utils.url_exists(url): raise InvalidParamsError("base url seems unreachable with these " "parameters: {}".format(url)) # if reset delete also the log directory if reset: fpath = os.path.join(cfg.PATHS['working_dir'], 'log') if os.path.exists(fpath): rmtree(fpath) if rgidf is None: # Infer the glacier directories from folders available in working dir if reset: raise ValueError('Cannot use reset without setting rgidf') log.workflow('init_glacier_directories by parsing all available ' 'folders (this takes time: if possible, provide rgidf ' 'instead).') # The dirs should be there already gl_dir = os.path.join(cfg.PATHS['working_dir'], 'per_glacier') gdirs = [] for root, _, files in os.walk(gl_dir): if files and ('outlines.shp' in files or 'outlines.tar.gz' in files): gdirs.append(oggm.GlacierDirectory(os.path.basename(root))) else: # Create glacier directories from input # Check if dataframe or list of str try: entities = [] for _, entity in rgidf.iterrows(): entities.append(entity) except AttributeError: entities = utils.tolist(rgidf) # Check demo if use_demo_glaciers is None: use_demo_glaciers = len(entities) < 100 if from_prepro_level is not None: log.workflow('init_glacier_directories from prepro level {} on ' '{} glaciers.'.format(from_prepro_level, len(entities))) # Read the hash dictionary before we use multiproc if cfg.PARAMS['dl_verify']: utils.get_dl_verify_data('cluster.klima.uni-bremen.de') gdirs = execute_entity_task(gdir_from_prepro, entities, from_prepro_level=from_prepro_level, prepro_border=prepro_border, prepro_rgi_version=prepro_rgi_version, check_demo_glacier=use_demo_glaciers, base_url=prepro_base_url) else: # We can set the intersects file automatically here if (cfg.PARAMS['use_intersects'] and len(cfg.PARAMS['intersects_gdf']) == 0): try: rgi_ids = np.unique(np.sort([entity.RGIId for entity in entities])) rgi_version = rgi_ids[0].split('-')[0][-2:] fp = utils.get_rgi_intersects_entities(rgi_ids, version=rgi_version) cfg.set_intersects_db(fp) except AttributeError: # List of str pass gdirs = execute_entity_task(utils.GlacierDirectory, entities, reset=reset, from_tar=from_tar, delete_tar=delete_tar) return gdirs
def init_glacier_regions(rgidf=None, *, reset=False, force=False, from_prepro_level=None, prepro_border=None, prepro_rgi_version=None, prepro_base_url=None, from_tar=False, delete_tar=False, use_demo_glaciers=None): """DEPRECATED: Initializes the list of Glacier Directories for this run. This is the very first task to do (always). If the directories are already available in the working directory, use them. If not, create new ones. Parameters ---------- rgidf : GeoDataFrame or list of ids, optional for pre-computed runs the RGI glacier outlines. If unavailable, OGGM will parse the information from the glacier directories found in the working directory. It is required for new runs. reset : bool delete the existing glacier directories if found. force : bool setting `reset=True` will trigger a yes/no question to the user. Set `force=True` to avoid this. from_prepro_level : int get the gdir data from the official pre-processed pool. See the documentation for more information prepro_border : int for `from_prepro_level` only: if you want to override the default behavior which is to use `cfg.PARAMS['border']` prepro_rgi_version : str for `from_prepro_level` only: if you want to override the default behavior which is to use `cfg.PARAMS['rgi_version']` prepro_base_url : str for `from_prepro_level` only: if you want to override the default URL from which to download the gdirs. Default currently is https://cluster.klima.uni-bremen.de/~oggm/gdirs/oggm_v1.1/ use_demo_glaciers : bool whether to check the demo glaciers for download (faster than the standard prepro downloads). The default is to decide whether or not to check based on simple criteria such as glacier list size. from_tar : bool, default=False extract the gdir data from a tar file. If set to `True`, will check for a tar file at the expected location in `base_dir`. delete_tar : bool, default=False delete the original tar file after extraction. delete_tar : bool, default=False delete the original tar file after extraction. Returns ------- gdirs : list of :py:class:`oggm.GlacierDirectory` objects the initialised glacier directories Notes ----- This task is deprecated in favor of the more explicit init_glacier_directories. Indeed, init_glacier_directories is very similar to init_glacier_regions, but it does not process the DEMs: a glacier directory is valid also without DEM. """ _check_duplicates(rgidf) if reset and not force: reset = utils.query_yes_no('Delete all glacier directories?') if prepro_border is None: prepro_border = int(cfg.PARAMS['border']) if from_prepro_level and prepro_border not in [10, 80, 160, 250]: if 'test' not in utils._downloads.GDIR_URL: raise InvalidParamsError("prepro_border or cfg.PARAMS['border'] " "should be one of: 10, 80, 160, 250.") # if reset delete also the log directory if reset: fpath = os.path.join(cfg.PATHS['working_dir'], 'log') if os.path.exists(fpath): rmtree(fpath) gdirs = [] new_gdirs = [] if rgidf is None: if reset: raise ValueError('Cannot use reset without setting rgidf') log.workflow('init_glacier_regions by parsing available folders ' '(can be slow).') # The dirs should be there already gl_dir = os.path.join(cfg.PATHS['working_dir'], 'per_glacier') for root, _, files in os.walk(gl_dir): if files and ('dem.tif' in files): gdirs.append(oggm.GlacierDirectory(os.path.basename(root))) else: # Check if dataframe or list of strs try: entities = [] for _, entity in rgidf.iterrows(): entities.append(entity) except AttributeError: entities = utils.tolist(rgidf) # Check demo if use_demo_glaciers is None: use_demo_glaciers = len(entities) < 100 if from_prepro_level is not None: log.workflow('init_glacier_regions from prepro level {} on ' '{} glaciers.'.format(from_prepro_level, len(entities))) # Read the hash dictionary before we use multiproc if cfg.PARAMS['dl_verify']: utils.get_dl_verify_data('cluster.klima.uni-bremen.de') gdirs = execute_entity_task(gdir_from_prepro, entities, from_prepro_level=from_prepro_level, prepro_border=prepro_border, prepro_rgi_version=prepro_rgi_version, check_demo_glacier=use_demo_glaciers, base_url=prepro_base_url) else: # We can set the intersects file automatically here if (cfg.PARAMS['use_intersects'] and len(cfg.PARAMS['intersects_gdf']) == 0): rgi_ids = np.unique(np.sort([entity.RGIId for entity in entities])) rgi_version = rgi_ids[0].split('-')[0][-2:] fp = utils.get_rgi_intersects_entities(rgi_ids, version=rgi_version) cfg.set_intersects_db(fp) gdirs = execute_entity_task(utils.GlacierDirectory, entities, reset=reset, from_tar=from_tar, delete_tar=delete_tar) for gdir in gdirs: if not os.path.exists(gdir.get_filepath('dem')): new_gdirs.append(gdir) if len(new_gdirs) > 0: # If not initialized, run the task in parallel execute_entity_task(tasks.define_glacier_region, new_gdirs) return gdirs
def initialize(file=None, logging_level='INFO'): """Read the configuration file containing the run's parameters. This should be the first call, before using any of the other OGGM modules for most (all?) OGGM simulations. Parameters ---------- file : str path to the configuration file (default: OGGM params.cfg) logging_level : str set a logging level. See :func:`set_logging_config` for options. """ global PARAMS global DATA initialize_minimal(file=file, logging_level=logging_level) # Do not spam PARAMS.do_log = False # Make sure we have a proper cache dir from oggm.utils import download_oggm_files, get_demo_file download_oggm_files() # Read-in the reference t* data for all available models types (oggm, vas) model_prefixes = ['oggm_', 'vas_'] for prefix in model_prefixes: fns = [ 'ref_tstars_rgi5_cru4', 'ref_tstars_rgi6_cru4', 'ref_tstars_rgi5_histalp', 'ref_tstars_rgi6_histalp' ] for fn in fns: fpath = get_demo_file(prefix + fn + '.csv') PARAMS[prefix + fn] = pd.read_csv(fpath) fpath = get_demo_file(prefix + fn + '_calib_params.json') with open(fpath, 'r') as fp: mbpar = json.load(fp) PARAMS[prefix + fn + '_calib_params'] = mbpar # Read in the demo glaciers file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'demo_glaciers.csv') DATA['demo_glaciers'] = pd.read_csv(file, index_col=0) # Add other things if 'dem_grids' not in DATA: grids = {} for grid_json in [ 'gimpdem_90m_v01.1.json', 'arcticdem_mosaic_100m_v3.0.json', 'Alaska_albers_V3.json', 'AntarcticDEM_wgs84.json', 'REMA_100m_dem.json' ]: if grid_json not in grids: fp = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', grid_json) try: grids[grid_json] = salem.Grid.from_json(fp) except NameError: pass DATA['dem_grids'] = grids # Trigger a one time check of the hash file from oggm.utils import get_dl_verify_data get_dl_verify_data('dummy_section') # OK PARAMS.do_log = True