def execute_entity_task(task, gdirs, **kwargs):
"""Execute a task on gdirs.
If you asked for multiprocessing, it will do it.
Parameters
----------
task: function
the entity task to apply
gdirs: list
the list of oggm.GlacierDirectory to process
optionally, each list element can be a tuple, with the first element being
the oggm.GlacierDirectory, and the second element a dict that will be passed
to the task function as **kwargs.
"""
if task.__dict__.get('global_task', False):
return task(gdirs, **kwargs)
pc = _pickle_copier(task, kwargs)
if _have_ogmpi:
if ogmpi.OGGM_MPI_COMM is not None:
ogmpi.mpi_master_spin_tasks(pc, gdirs)
return
if cfg.PARAMS['use_multiprocessing']:
mppool = _init_pool()
mppool.map(pc, gdirs, chunksize=1)
else:
for gdir in gdirs:
pc(gdir)
def execute_entity_task(task, gdirs, **kwargs):
"""Execute a task on gdirs.
If you asked for multiprocessing, it will do it.
Parameters
----------
task : function
the entity task to apply
gdirs : list
the list of oggm.GlacierDirectory to process.
"""
if task.__dict__.get('global_task', False):
return task(gdirs, **kwargs)
pc = _pickle_copier(task, kwargs)
if _have_ogmpi:
if ogmpi.OGGM_MPI_COMM is not None:
ogmpi.mpi_master_spin_tasks(pc, gdirs)
return
if cfg.PARAMS['use_multiprocessing']:
mppool = init_mp_pool(cfg.CONFIG_MODIFIED)
mppool.map(pc, gdirs, chunksize=1)
else:
for gdir in gdirs:
pc(gdir)
def execute_entity_task(task, gdirs, **kwargs):
"""Execute a task on gdirs.
If you asked for multiprocessing, it will do it.
Parameters
----------
task: function
the entity task to apply
gdirs: list
the list of oggm.GlacierDirectory to process
optionally, each list element can be a tuple, with the first element being
the oggm.GlacierDirectory, and the second element a dict that will be passed
to the task function as **kwargs.
"""
if task.__dict__.get('global_task', False):
return task(gdirs, **kwargs)
pc = _pickle_copier(task, kwargs)
if _have_ogmpi:
if ogmpi.OGGM_MPI_COMM is not None:
ogmpi.mpi_master_spin_tasks(pc, gdirs)
return
if cfg.PARAMS['use_multiprocessing']:
mppool = _init_pool()
mppool.map(pc, gdirs, chunksize=1)
else:
for gdir in gdirs:
pc(gdir)
def execute_parallel_tasks(gdir, tasks):
"""Execute a list of task on a single gdir (experimental!).
This is useful when running a non-sequential list of task on a gdir,
mostly for e.g. different experiments with different output files.
Parameters
----------
gdir : :py:class:`oggm.GlacierDirectory`
the directory to process.
tasks : list
the the list of entity tasks to apply.
Optionally, each list element can be a tuple, with the first element
being the task, and the second element a dict that
will be passed to the task function as ``**kwargs``.
"""
pc = _pickle_copier(None, {})
_tasks = []
for task in tasks:
kwargs = {}
if isinstance(task, Sequence):
task, kwargs = task
_tasks.append((task, (gdir, kwargs)))
if _have_ogmpi:
if ogmpi.OGGM_MPI_COMM is not None:
ogmpi.mpi_master_spin_tasks(pc, _tasks)
return
if cfg.PARAMS['use_multiprocessing']:
mppool = init_mp_pool(cfg.CONFIG_MODIFIED)
mppool.map(pc, _tasks, chunksize=1)
else:
for task, (gd, kw) in _tasks:
task(gd, **kw)
def execute_parallel_tasks(gdir, tasks):
"""Execute a list of task on a single gdir (experimental!).
This is useful when running a non-sequential list of task on a gdir,
mostly for e.g. different experiments with different output files.
Parameters
----------
gdir : :py:class:`oggm.GlacierDirectory`
the directory to process.
tasks : list
the the list of entity tasks to apply.
Optionally, each list element can be a tuple, with the first element
being the task, and the second element a dict that
will be passed to the task function as ``**kwargs``.
"""
pc = _pickle_copier(None, {})
_tasks = []
for task in tasks:
kwargs = {}
if isinstance(task, Sequence):
task, kwargs = task
_tasks.append((task, (gdir, kwargs)))
if _have_ogmpi:
if ogmpi.OGGM_MPI_COMM is not None:
ogmpi.mpi_master_spin_tasks(pc, _tasks)
return
if cfg.PARAMS['use_multiprocessing']:
mppool = init_mp_pool(cfg.CONFIG_MODIFIED)
mppool.map(pc, _tasks, chunksize=1)
else:
for task in _tasks:
task()
def execute_entity_task(task, gdirs, **kwargs):
"""Execute a task on gdirs.
If you asked for multiprocessing, it will do it.
If ``task`` has more arguments than `gdir` they have to be keyword
arguments.
Parameters
----------
task : function
the entity task to apply
gdirs : list of :py:class:`oggm.GlacierDirectory` objects
the glacier directories to process
"""
if task.__dict__.get('is_global_task', False):
raise InvalidWorkflowError('execute_entity_task cannot be used on '
'global tasks.')
# Should be iterable
gdirs = utils.tolist(gdirs)
ng = len(gdirs)
if ng == 0:
log.workflow('Called entity task %s on 0 glaciers. Returning...',
task.__name__)
return
log.workflow('Execute entity task %s on %d glaciers', task.__name__, ng)
pc = _pickle_copier(task, kwargs)
if _have_ogmpi:
if ogmpi.OGGM_MPI_COMM is not None:
return ogmpi.mpi_master_spin_tasks(pc, gdirs)
if cfg.PARAMS['use_multiprocessing'] and ng > 1:
mppool = init_mp_pool(cfg.CONFIG_MODIFIED)
out = mppool.map(pc, gdirs, chunksize=1)
else:
if ng > 3:
log.workflow(
'WARNING: you are trying to run an entity task on '
'%d glaciers with multiprocessing turned off. OGGM '
'will run faster with multiprocessing turned on.', ng)
out = [pc(gdir) for gdir in gdirs]
return out
def execute_entity_task(task, gdirs, **kwargs):
"""Execute a task on gdirs.
If you asked for multiprocessing, it will do it.
If ``task`` has more arguments than `gdir` they have to be keyword
arguments.
Parameters
----------
task : function
the entity task to apply
gdirs : list of :py:class:`oggm.GlacierDirectory` objects
the glacier directories to process
"""
# If not iterable it's ok
try:
len(gdirs)
except TypeError:
gdirs = [gdirs]
if len(gdirs) == 0:
return
log.workflow('Execute entity task %s on %d glaciers', task.__name__,
len(gdirs))
if task.__dict__.get('global_task', False):
return task(gdirs, **kwargs)
pc = _pickle_copier(task, kwargs)
if _have_ogmpi:
if ogmpi.OGGM_MPI_COMM is not None:
return ogmpi.mpi_master_spin_tasks(pc, gdirs)
if cfg.PARAMS['use_multiprocessing']:
mppool = init_mp_pool(cfg.CONFIG_MODIFIED)
out = mppool.map(pc, gdirs, chunksize=1)
else:
out = [pc(gdir) for gdir in gdirs]
return out
def execute_entity_task(task, gdirs, **kwargs):
"""Execute a task on gdirs.
If you asked for multiprocessing, it will do it.
If ``task`` has more arguments than `gdir` they have to be keyword
arguments.
Parameters
----------
task : function
the entity task to apply
gdirs : list of :py:class:`oggm.GlacierDirectory` objects
the glacier directories to process
"""
# If not iterable it's ok
try:
len(gdirs)
except TypeError:
gdirs = [gdirs]
if len(gdirs) == 0:
return
log.workflow('Execute entity task %s on %d glaciers',
task.__name__, len(gdirs))
if task.__dict__.get('global_task', False):
return task(gdirs, **kwargs)
pc = _pickle_copier(task, kwargs)
if _have_ogmpi:
if ogmpi.OGGM_MPI_COMM is not None:
return ogmpi.mpi_master_spin_tasks(pc, gdirs)
if cfg.PARAMS['use_multiprocessing']:
mppool = init_mp_pool(cfg.CONFIG_MODIFIED)
out = mppool.map(pc, gdirs, chunksize=1)
else:
out = [pc(gdir) for gdir in gdirs]
return out
def execute_entity_task(task, gdirs, **kwargs):
"""Execute a task on gdirs.
If you asked for multiprocessing, it will do it.
If ``task`` has more arguments than `gdir` they have to be keyword
arguments.
Parameters
----------
task : function or sequence of functions
The entity task(s) to apply.
Can be None, in which case each gdir is expected to be a tuple of (task, gdir).
When passing a sequence, each item can also optionally be a tuple of (task, dictionary).
In this case the dictionary items will be passed to the task as kwargs.
gdirs : list of :py:class:`oggm.GlacierDirectory` objects
The glacier directories to process.
Each individual gdir can optionally be a tuple of (gdir, dictionary).
In this case, the values in the dictionary will be passed to the task as
keyword arguments for that specific gdir.
Returns
-------
List of results from task. Last task if a list of tasks was given.
"""
# Normalize task into list of tuples for simplicity
if not isinstance(task, Sequence):
task = [task]
tasks = []
for t in task:
if isinstance(t, tuple):
tasks.append(t)
else:
tasks.append((t, {}))
# Reject global tasks
for t in tasks:
if t[0].__dict__.get('is_global_task', False):
raise InvalidWorkflowError('execute_entity_task cannot be used on '
'global tasks.')
# Should be iterable
gdirs = utils.tolist(gdirs)
ng = len(gdirs)
if ng == 0:
log.workflow('Called execute_entity_task on 0 glaciers. Returning...')
return
log.workflow('Execute entity tasks [%s] on %d glaciers',
', '.join([t[0].__name__ for t in tasks]), ng)
pc = _pickle_copier(tasks, kwargs)
if _have_ogmpi:
if ogmpi.OGGM_MPI_COMM is not None:
return ogmpi.mpi_master_spin_tasks(pc, gdirs)
if cfg.PARAMS['use_multiprocessing'] and ng > 1:
mppool = init_mp_pool(cfg.CONFIG_MODIFIED)
out = mppool.map(pc, gdirs, chunksize=1)
else:
if ng > 3:
log.workflow(
'WARNING: you are trying to run an entity task on '
'%d glaciers with multiprocessing turned off. OGGM '
'will run faster with multiprocessing turned on.', ng)
out = [pc(gdir) for gdir in gdirs]
return out
