def init_glacier_regions(rgidf, reset=False, force=False):
"""Very first task to do (always).
Set reset=True in order to delete the content of the directories.
"""
if reset and not force:
reset = utils.query_yes_no('Delete all glacier directories?')
# if reset delete also the log directory
if reset:
fpath = os.path.join(cfg.PATHS['working_dir'], 'log')
if os.path.exists(fpath):
rmtree(fpath)
gdirs = []
new_gdirs = []
for _, entity in rgidf.iterrows():
gdir = oggm.GlacierDirectory(entity, reset=reset)
if not os.path.exists(gdir.get_filepath('dem')):
new_gdirs.append((gdir, dict(entity=entity)))
gdirs.append(gdir)
execute_entity_task(tasks.define_glacier_region, new_gdirs)
return gdirs
def init_glacier_regions(rgidf, reset=False, force=False):
"""Very first task to do (always).
Set reset=True in order to delete the content of the directories.
"""
if reset and not force:
reset = utils.query_yes_no('Delete all glacier directories?')
# if reset delete also the log directory
if reset:
fpath = os.path.join(cfg.PATHS['working_dir'], 'log')
if os.path.exists(fpath):
rmtree(fpath)
gdirs = []
new_gdirs = []
for _, entity in rgidf.iterrows():
gdir = oggm.GlacierDirectory(entity, reset=reset)
if not os.path.exists(gdir.get_filepath('dem')):
new_gdirs.append((gdir, dict(entity=entity)))
gdirs.append(gdir)
execute_entity_task(tasks.define_glacier_region, new_gdirs)
return gdirs
def init_glacier_regions(rgidf, reset=False, force=False):
"""Very first task to do (allways). Set reset=False
in order not to delete the content"""
if reset and not force:
reset = utils.query_yes_no('Delete all glacier directories?')
gdirs = []
for _, entity in rgidf.iterrows():
gdir = cfg.GlacierDir(entity, reset=reset)
if not os.path.exists(gdir.get_filepath('dem')):
gis.define_glacier_region(gdir, entity)
gdirs.append(gdir)
return gdirs
def synthetic_experiments_parallel(gdirs, t0, te):
"""
creates searched and observed glacier to test the method, need only to
be run once
:param gdirs: list of oggm.GlacierDirectories
:return:
"""
reset = True
if os.path.isfile(gdirs[0].get_filepath('synthetic_experiment')):
reset = utils.query_yes_no('Running the function synthetic_experiments'
' will reset the previous results. Are you '
' sure you like to continue?')
if not reset:
return
pool = mp.Pool()
pool.map(partial(_run_parallel_experiment, t0=t0, te=te), gdirs)
pool.close()
pool.join()
def init_glacier_regions(rgidf=None, reset=False, force=False):
"""Very first task to do (always).
Set reset=True in order to delete the content of the directories.
"""
if reset and not force:
reset = utils.query_yes_no('Delete all glacier directories?')
# if reset delete also the log directory
if reset:
fpath = os.path.join(cfg.PATHS['working_dir'], 'log')
if os.path.exists(fpath):
rmtree(fpath)
gdirs = []
new_gdirs = []
if rgidf is None:
if reset:
raise ValueError('Cannot use reset without a rgi file')
# The dirs should be there already
gl_dir = os.path.join(cfg.PATHS['working_dir'], 'per_glacier')
for root, _, files in os.walk(gl_dir):
if files and ('dem.tif' in files):
gdirs.append(oggm.GlacierDirectory(os.path.basename(root)))
else:
for _, entity in rgidf.iterrows():
gdir = oggm.GlacierDirectory(entity, reset=reset)
if not os.path.exists(gdir.get_filepath('dem')):
new_gdirs.append((gdir, dict(entity=entity)))
gdirs.append(gdir)
# If not initialized, run the task in parallel
execute_entity_task(tasks.define_glacier_region, new_gdirs)
return gdirs
def init_glacier_directories(rgidf=None, *, reset=False, force=False,
from_prepro_level=None, prepro_border=None,
prepro_rgi_version=None, prepro_base_url=None,
from_tar=False, delete_tar=False,
use_demo_glaciers=None):
"""Initializes the list of Glacier Directories for this run.
This is the very first task to do (always). If the directories are already
available in the working directory, use them. If not, create new ones.
Parameters
----------
rgidf : GeoDataFrame or list of ids, optional for pre-computed runs
the RGI glacier outlines. If unavailable, OGGM will parse the
information from the glacier directories found in the working
directory. It is required for new runs.
reset : bool
delete the existing glacier directories if found.
force : bool
setting `reset=True` will trigger a yes/no question to the user. Set
`force=True` to avoid this.
from_prepro_level : int
get the gdir data from the official pre-processed pool. See the
documentation for more information
prepro_border : int
for `from_prepro_level` only: if you want to override the default
behavior which is to use `cfg.PARAMS['border']`
prepro_rgi_version : str
for `from_prepro_level` only: if you want to override the default
behavior which is to use `cfg.PARAMS['rgi_version']`
prepro_base_url : str
for `from_prepro_level` only: if you want to override the default
URL from which to download the gdirs. Default currently is
https://cluster.klima.uni-bremen.de/~fmaussion/gdirs/oggm_v1.1/
use_demo_glaciers : bool
whether to check the demo glaciers for download (faster than the
standard prepro downloads). The default is to decide whether or
not to check based on simple criteria such as glacier list size.
from_tar : bool, default=False
extract the gdir data from a tar file. If set to `True`,
will check for a tar file at the expected location in `base_dir`.
delete_tar : bool, default=False
delete the original tar file after extraction.
Returns
-------
gdirs : list of :py:class:`oggm.GlacierDirectory` objects
the initialised glacier directories
Notes
-----
This task is very similar to init_glacier_regions, with one main
difference: it does not process the DEMs for this glacier.
Eventually, init_glacier_regions will be deprecated and removed from the
codebase.
"""
_check_duplicates(rgidf)
if reset and not force:
reset = utils.query_yes_no('Delete all glacier directories?')
if from_prepro_level:
url = utils.get_prepro_base_url(base_url=prepro_base_url,
border=prepro_border,
prepro_level=from_prepro_level,
rgi_version=prepro_rgi_version)
if cfg.PARAMS['has_internet'] and not utils.url_exists(url):
raise InvalidParamsError("base url seems unreachable with these "
"parameters: {}".format(url))
# if reset delete also the log directory
if reset:
fpath = os.path.join(cfg.PATHS['working_dir'], 'log')
if os.path.exists(fpath):
rmtree(fpath)
if rgidf is None:
# Infer the glacier directories from folders available in working dir
if reset:
raise ValueError('Cannot use reset without setting rgidf')
log.workflow('init_glacier_directories by parsing all available '
'folders (this takes time: if possible, provide rgidf '
'instead).')
# The dirs should be there already
gl_dir = os.path.join(cfg.PATHS['working_dir'], 'per_glacier')
gdirs = []
for root, _, files in os.walk(gl_dir):
if files and ('outlines.shp' in files or
'outlines.tar.gz' in files):
gdirs.append(oggm.GlacierDirectory(os.path.basename(root)))
else:
# Create glacier directories from input
# Check if dataframe or list of str
try:
entities = []
for _, entity in rgidf.iterrows():
entities.append(entity)
except AttributeError:
entities = utils.tolist(rgidf)
# Check demo
if use_demo_glaciers is None:
use_demo_glaciers = len(entities) < 100
if from_prepro_level is not None:
log.workflow('init_glacier_directories from prepro level {} on '
'{} glaciers.'.format(from_prepro_level,
len(entities)))
# Read the hash dictionary before we use multiproc
if cfg.PARAMS['dl_verify']:
utils.get_dl_verify_data('cluster.klima.uni-bremen.de')
gdirs = execute_entity_task(gdir_from_prepro, entities,
from_prepro_level=from_prepro_level,
prepro_border=prepro_border,
prepro_rgi_version=prepro_rgi_version,
check_demo_glacier=use_demo_glaciers,
base_url=prepro_base_url)
else:
# We can set the intersects file automatically here
if (cfg.PARAMS['use_intersects'] and
len(cfg.PARAMS['intersects_gdf']) == 0):
try:
rgi_ids = np.unique(np.sort([entity.RGIId for entity in
entities]))
rgi_version = rgi_ids[0].split('-')[0][-2:]
fp = utils.get_rgi_intersects_entities(rgi_ids,
version=rgi_version)
cfg.set_intersects_db(fp)
except AttributeError:
# List of str
pass
gdirs = execute_entity_task(utils.GlacierDirectory, entities,
reset=reset,
from_tar=from_tar,
delete_tar=delete_tar)
return gdirs
def init_glacier_regions(rgidf=None, *, reset=False, force=False,
from_prepro_level=None, prepro_border=None,
prepro_rgi_version=None, prepro_base_url=None,
from_tar=False, delete_tar=False,
use_demo_glaciers=None):
"""DEPRECATED: Initializes the list of Glacier Directories for this run.
This is the very first task to do (always). If the directories are already
available in the working directory, use them. If not, create new ones.
Parameters
----------
rgidf : GeoDataFrame or list of ids, optional for pre-computed runs
the RGI glacier outlines. If unavailable, OGGM will parse the
information from the glacier directories found in the working
directory. It is required for new runs.
reset : bool
delete the existing glacier directories if found.
force : bool
setting `reset=True` will trigger a yes/no question to the user. Set
`force=True` to avoid this.
from_prepro_level : int
get the gdir data from the official pre-processed pool. See the
documentation for more information
prepro_border : int
for `from_prepro_level` only: if you want to override the default
behavior which is to use `cfg.PARAMS['border']`
prepro_rgi_version : str
for `from_prepro_level` only: if you want to override the default
behavior which is to use `cfg.PARAMS['rgi_version']`
prepro_base_url : str
for `from_prepro_level` only: if you want to override the default
URL from which to download the gdirs. Default currently is
https://cluster.klima.uni-bremen.de/~oggm/gdirs/oggm_v1.1/
use_demo_glaciers : bool
whether to check the demo glaciers for download (faster than the
standard prepro downloads). The default is to decide whether or
not to check based on simple criteria such as glacier list size.
from_tar : bool, default=False
extract the gdir data from a tar file. If set to `True`,
will check for a tar file at the expected location in `base_dir`.
delete_tar : bool, default=False
delete the original tar file after extraction.
delete_tar : bool, default=False
delete the original tar file after extraction.
Returns
-------
gdirs : list of :py:class:`oggm.GlacierDirectory` objects
the initialised glacier directories
Notes
-----
This task is deprecated in favor of the more explicit
init_glacier_directories. Indeed, init_glacier_directories is very
similar to init_glacier_regions, but it does not process the DEMs:
a glacier directory is valid also without DEM.
"""
_check_duplicates(rgidf)
if reset and not force:
reset = utils.query_yes_no('Delete all glacier directories?')
if prepro_border is None:
prepro_border = int(cfg.PARAMS['border'])
if from_prepro_level and prepro_border not in [10, 80, 160, 250]:
if 'test' not in utils._downloads.GDIR_URL:
raise InvalidParamsError("prepro_border or cfg.PARAMS['border'] "
"should be one of: 10, 80, 160, 250.")
# if reset delete also the log directory
if reset:
fpath = os.path.join(cfg.PATHS['working_dir'], 'log')
if os.path.exists(fpath):
rmtree(fpath)
gdirs = []
new_gdirs = []
if rgidf is None:
if reset:
raise ValueError('Cannot use reset without setting rgidf')
log.workflow('init_glacier_regions by parsing available folders '
'(can be slow).')
# The dirs should be there already
gl_dir = os.path.join(cfg.PATHS['working_dir'], 'per_glacier')
for root, _, files in os.walk(gl_dir):
if files and ('dem.tif' in files):
gdirs.append(oggm.GlacierDirectory(os.path.basename(root)))
else:
# Check if dataframe or list of strs
try:
entities = []
for _, entity in rgidf.iterrows():
entities.append(entity)
except AttributeError:
entities = utils.tolist(rgidf)
# Check demo
if use_demo_glaciers is None:
use_demo_glaciers = len(entities) < 100
if from_prepro_level is not None:
log.workflow('init_glacier_regions from prepro level {} on '
'{} glaciers.'.format(from_prepro_level,
len(entities)))
# Read the hash dictionary before we use multiproc
if cfg.PARAMS['dl_verify']:
utils.get_dl_verify_data('cluster.klima.uni-bremen.de')
gdirs = execute_entity_task(gdir_from_prepro, entities,
from_prepro_level=from_prepro_level,
prepro_border=prepro_border,
prepro_rgi_version=prepro_rgi_version,
check_demo_glacier=use_demo_glaciers,
base_url=prepro_base_url)
else:
# We can set the intersects file automatically here
if (cfg.PARAMS['use_intersects'] and
len(cfg.PARAMS['intersects_gdf']) == 0):
rgi_ids = np.unique(np.sort([entity.RGIId for entity in
entities]))
rgi_version = rgi_ids[0].split('-')[0][-2:]
fp = utils.get_rgi_intersects_entities(rgi_ids,
version=rgi_version)
cfg.set_intersects_db(fp)
gdirs = execute_entity_task(utils.GlacierDirectory, entities,
reset=reset,
from_tar=from_tar,
delete_tar=delete_tar)
for gdir in gdirs:
if not os.path.exists(gdir.get_filepath('dem')):
new_gdirs.append(gdir)
if len(new_gdirs) > 0:
# If not initialized, run the task in parallel
execute_entity_task(tasks.define_glacier_region, new_gdirs)
return gdirs
def init_glacier_regions(rgidf=None, reset=False, force=False):
"""Initializes the list of Glacier Directories for this run.
This is the very first task to do (always). If the directories are already
available in the working directory, use them. If not, create new ones.
Parameters
----------
rgidf : GeoDataFrame, optional for pre-computed runs
the RGI glacier outlines. If unavailable, OGGM will parse the
information from the glacier directories found in the working
directory. It is required for new runs.
reset : bool
delete the existing glacier directories if found.
force : bool
setting `reset=True` will trigger a yes/no question to the user. Set
`force=True` to avoid this.
Returns
-------
a list of GlacierDirectory objects
"""
if reset and not force:
reset = utils.query_yes_no('Delete all glacier directories?')
# if reset delete also the log directory
if reset:
fpath = os.path.join(cfg.PATHS['working_dir'], 'log')
if os.path.exists(fpath):
rmtree(fpath)
gdirs = []
new_gdirs = []
if rgidf is None:
if reset:
raise ValueError('Cannot use reset without a rgi file')
# The dirs should be there already
gl_dir = os.path.join(cfg.PATHS['working_dir'], 'per_glacier')
for root, _, files in os.walk(gl_dir):
if files and ('dem.tif' in files):
gdirs.append(oggm.GlacierDirectory(os.path.basename(root)))
else:
for _, entity in rgidf.iterrows():
gdir = oggm.GlacierDirectory(entity, reset=reset)
if not os.path.exists(gdir.get_filepath('dem')):
new_gdirs.append((gdir, dict(entity=entity)))
gdirs.append(gdir)
# We can set the intersects file automatically here
if (cfg.PARAMS['use_intersects'] and new_gdirs
and (len(cfg.PARAMS['intersects_gdf']) == 0)):
rgi_ids = np.unique(np.sort([t[0].rgi_id for t in new_gdirs]))
rgi_version = new_gdirs[0][0].rgi_version
fp = utils.get_rgi_intersects_entities(rgi_ids, version=rgi_version)
cfg.set_intersects_db(fp)
# If not initialized, run the task in parallel
execute_entity_task(tasks.define_glacier_region, new_gdirs)
return gdirs
def init_glacier_regions(rgidf=None,
*,
reset=False,
force=False,
from_prepro_level=None,
prepro_border=None,
prepro_rgi_version=None,
from_tar=False,
delete_tar=False,
use_demo_glaciers=None):
"""Initializes the list of Glacier Directories for this run.
This is the very first task to do (always). If the directories are already
available in the working directory, use them. If not, create new ones.
Parameters
----------
rgidf : GeoDataFrame or list of ids, optional for pre-computed runs
the RGI glacier outlines. If unavailable, OGGM will parse the
information from the glacier directories found in the working
directory. It is required for new runs.
reset : bool
delete the existing glacier directories if found.
force : bool
setting `reset=True` will trigger a yes/no question to the user. Set
`force=True` to avoid this.
from_prepro_level : int
get the gdir data from the official pre-processed pool. See the
documentation for more information
prepro_border : int
for `from_prepro_level` only: if you want to override the default
behavior which is to use `cfg.PARAMS['border']`
prepro_rgi_version : str
for `from_prepro_level` only: if you want to override the default
behavior which is to use `cfg.PARAMS['rgi_version']`
use_demo_glaciers : bool
whether to check the demo glaciers for download (faster than the
standard prepro downloads). The default is to decide whether or
not to check based on simple crietria such as glacier list size.
from_tar : bool, default=False
extract the gdir data from a tar file. If set to `True`,
will check for a tar file at the expected location in `base_dir`.
delete_tar : bool, default=False
delete the original tar file after extraction.
delete_tar : bool, default=False
delete the original tar file after extraction.
Returns
-------
gdirs : list of :py:class:`oggm.GlacierDirectory` objects
the initialised glacier directories
"""
if reset and not force:
reset = utils.query_yes_no('Delete all glacier directories?')
# if reset delete also the log directory
if reset:
fpath = os.path.join(cfg.PATHS['working_dir'], 'log')
if os.path.exists(fpath):
rmtree(fpath)
gdirs = []
new_gdirs = []
if rgidf is None:
if reset:
raise ValueError('Cannot use reset without setting rgidf')
log.workflow('init_glacier_regions by parsing available folders '
'(can be slow).')
# The dirs should be there already
gl_dir = os.path.join(cfg.PATHS['working_dir'], 'per_glacier')
for root, _, files in os.walk(gl_dir):
if files and ('dem.tif' in files):
gdirs.append(oggm.GlacierDirectory(os.path.basename(root)))
else:
# Check if dataframe or list of strs
try:
entities = []
for _, entity in rgidf.iterrows():
entities.append(entity)
except AttributeError:
entities = utils.tolist(rgidf)
# Check demo
if use_demo_glaciers is None:
use_demo_glaciers = len(entities) < 100
if from_prepro_level is not None:
log.workflow('init_glacier_regions from prepro level {} on '
'{} glaciers.'.format(from_prepro_level,
len(entities)))
gdirs = execute_entity_task(gdir_from_prepro,
entities,
from_prepro_level=from_prepro_level,
prepro_border=prepro_border,
prepro_rgi_version=prepro_rgi_version,
check_demo_glacier=use_demo_glaciers)
else:
# TODO: if necessary this could use multiprocessing as well
for entity in entities:
gdir = oggm.GlacierDirectory(entity,
reset=reset,
from_tar=from_tar,
delete_tar=delete_tar)
if not os.path.exists(gdir.get_filepath('dem')):
new_gdirs.append((gdir, dict(entity=entity)))
gdirs.append(gdir)
# We can set the intersects file automatically here
if (cfg.PARAMS['use_intersects'] and new_gdirs
and (len(cfg.PARAMS['intersects_gdf']) == 0)):
rgi_ids = np.unique(np.sort([t[0].rgi_id for t in new_gdirs]))
rgi_version = new_gdirs[0][0].rgi_version
fp = utils.get_rgi_intersects_entities(rgi_ids, version=rgi_version)
cfg.set_intersects_db(fp)
# If not initialized, run the task in parallel
execute_entity_task(tasks.define_glacier_region, new_gdirs)
return gdirs
def init_glacier_regions(rgidf=None, *, reset=False, force=False,
from_prepro_level=None, prepro_border=None,
prepro_rgi_version=None,
from_tar=False, delete_tar=False,
use_demo_glaciers=None):
"""Initializes the list of Glacier Directories for this run.
This is the very first task to do (always). If the directories are already
available in the working directory, use them. If not, create new ones.
Parameters
----------
rgidf : GeoDataFrame or list of ids, optional for pre-computed runs
the RGI glacier outlines. If unavailable, OGGM will parse the
information from the glacier directories found in the working
directory. It is required for new runs.
reset : bool
delete the existing glacier directories if found.
force : bool
setting `reset=True` will trigger a yes/no question to the user. Set
`force=True` to avoid this.
from_prepro_level : int
get the gdir data from the official pre-processed pool. See the
documentation for more information
prepro_border : int
for `from_prepro_level` only: if you want to override the default
behavior which is to use `cfg.PARAMS['border']`
prepro_rgi_version : str
for `from_prepro_level` only: if you want to override the default
behavior which is to use `cfg.PARAMS['rgi_version']`
use_demo_glaciers : bool
whether to check the demo glaciers for download (faster than the
standard prepro downloads). The default is to decide whether or
not to check based on simple crietria such as glacier list size.
from_tar : bool, default=False
extract the gdir data from a tar file. If set to `True`,
will check for a tar file at the expected location in `base_dir`.
delete_tar : bool, default=False
delete the original tar file after extraction.
delete_tar : bool, default=False
delete the original tar file after extraction.
Returns
-------
gdirs : list of :py:class:`oggm.GlacierDirectory` objects
the initialised glacier directories
"""
if reset and not force:
reset = utils.query_yes_no('Delete all glacier directories?')
if prepro_border is None:
prepro_border = int(cfg.PARAMS['border'])
if from_prepro_level and prepro_border not in [10, 80, 160, 250]:
if 'test' not in utils._downloads.GDIR_URL:
raise InvalidParamsError("prepro_border or cfg.PARAMS['border'] "
"should be one of: 10, 80, 160, 250.")
# if reset delete also the log directory
if reset:
fpath = os.path.join(cfg.PATHS['working_dir'], 'log')
if os.path.exists(fpath):
rmtree(fpath)
gdirs = []
new_gdirs = []
if rgidf is None:
if reset:
raise ValueError('Cannot use reset without setting rgidf')
log.workflow('init_glacier_regions by parsing available folders '
'(can be slow).')
# The dirs should be there already
gl_dir = os.path.join(cfg.PATHS['working_dir'], 'per_glacier')
for root, _, files in os.walk(gl_dir):
if files and ('dem.tif' in files):
gdirs.append(oggm.GlacierDirectory(os.path.basename(root)))
else:
# Check if dataframe or list of strs
try:
entities = []
for _, entity in rgidf.iterrows():
entities.append(entity)
except AttributeError:
entities = utils.tolist(rgidf)
# Check demo
if use_demo_glaciers is None:
use_demo_glaciers = len(entities) < 100
if from_prepro_level is not None:
log.workflow('init_glacier_regions from prepro level {} on '
'{} glaciers.'.format(from_prepro_level,
len(entities)))
gdirs = execute_entity_task(gdir_from_prepro, entities,
from_prepro_level=from_prepro_level,
prepro_border=prepro_border,
prepro_rgi_version=prepro_rgi_version,
check_demo_glacier=use_demo_glaciers)
else:
# TODO: if necessary this could use multiprocessing as well
for entity in entities:
gdir = oggm.GlacierDirectory(entity, reset=reset,
from_tar=from_tar,
delete_tar=delete_tar)
if not os.path.exists(gdir.get_filepath('dem')):
new_gdirs.append((gdir, dict(entity=entity)))
gdirs.append(gdir)
# We can set the intersects file automatically here
if (cfg.PARAMS['use_intersects'] and new_gdirs and
(len(cfg.PARAMS['intersects_gdf']) == 0)):
rgi_ids = np.unique(np.sort([t[0].rgi_id for t in new_gdirs]))
rgi_version = new_gdirs[0][0].rgi_version
fp = utils.get_rgi_intersects_entities(rgi_ids, version=rgi_version)
cfg.set_intersects_db(fp)
# If not initialized, run the task in parallel
execute_entity_task(tasks.define_glacier_region, new_gdirs)
return gdirs