def execute_entity_task(task, gdirs, **kwargs): """Execute a task on gdirs. If you asked for multiprocessing, it will do it. Parameters ---------- task: function the entity task to apply gdirs: list the list of oggm.GlacierDirectory to process optionally, each list element can be a tuple, with the first element being the oggm.GlacierDirectory, and the second element a dict that will be passed to the task function as **kwargs. """ if task.__dict__.get('global_task', False): return task(gdirs, **kwargs) pc = _pickle_copier(task, kwargs) if _have_ogmpi: if ogmpi.OGGM_MPI_COMM is not None: ogmpi.mpi_master_spin_tasks(pc, gdirs) return if cfg.PARAMS['use_multiprocessing']: mppool = _init_pool() mppool.map(pc, gdirs, chunksize=1) else: for gdir in gdirs: pc(gdir)
def execute_entity_task(task, gdirs, **kwargs): """Execute a task on gdirs. If you asked for multiprocessing, it will do it. Parameters ---------- task : function the entity task to apply gdirs : list the list of oggm.GlacierDirectory to process. """ if task.__dict__.get('global_task', False): return task(gdirs, **kwargs) pc = _pickle_copier(task, kwargs) if _have_ogmpi: if ogmpi.OGGM_MPI_COMM is not None: ogmpi.mpi_master_spin_tasks(pc, gdirs) return if cfg.PARAMS['use_multiprocessing']: mppool = init_mp_pool(cfg.CONFIG_MODIFIED) mppool.map(pc, gdirs, chunksize=1) else: for gdir in gdirs: pc(gdir)
def execute_parallel_tasks(gdir, tasks): """Execute a list of task on a single gdir (experimental!). This is useful when running a non-sequential list of task on a gdir, mostly for e.g. different experiments with different output files. Parameters ---------- gdir : :py:class:`oggm.GlacierDirectory` the directory to process. tasks : list the the list of entity tasks to apply. Optionally, each list element can be a tuple, with the first element being the task, and the second element a dict that will be passed to the task function as ``**kwargs``. """ pc = _pickle_copier(None, {}) _tasks = [] for task in tasks: kwargs = {} if isinstance(task, Sequence): task, kwargs = task _tasks.append((task, (gdir, kwargs))) if _have_ogmpi: if ogmpi.OGGM_MPI_COMM is not None: ogmpi.mpi_master_spin_tasks(pc, _tasks) return if cfg.PARAMS['use_multiprocessing']: mppool = init_mp_pool(cfg.CONFIG_MODIFIED) mppool.map(pc, _tasks, chunksize=1) else: for task, (gd, kw) in _tasks: task(gd, **kw)
def execute_parallel_tasks(gdir, tasks): """Execute a list of task on a single gdir (experimental!). This is useful when running a non-sequential list of task on a gdir, mostly for e.g. different experiments with different output files. Parameters ---------- gdir : :py:class:`oggm.GlacierDirectory` the directory to process. tasks : list the the list of entity tasks to apply. Optionally, each list element can be a tuple, with the first element being the task, and the second element a dict that will be passed to the task function as ``**kwargs``. """ pc = _pickle_copier(None, {}) _tasks = [] for task in tasks: kwargs = {} if isinstance(task, Sequence): task, kwargs = task _tasks.append((task, (gdir, kwargs))) if _have_ogmpi: if ogmpi.OGGM_MPI_COMM is not None: ogmpi.mpi_master_spin_tasks(pc, _tasks) return if cfg.PARAMS['use_multiprocessing']: mppool = init_mp_pool(cfg.CONFIG_MODIFIED) mppool.map(pc, _tasks, chunksize=1) else: for task in _tasks: task()
def execute_entity_task(task, gdirs, **kwargs): """Execute a task on gdirs. If you asked for multiprocessing, it will do it. If ``task`` has more arguments than `gdir` they have to be keyword arguments. Parameters ---------- task : function the entity task to apply gdirs : list of :py:class:`oggm.GlacierDirectory` objects the glacier directories to process """ if task.__dict__.get('is_global_task', False): raise InvalidWorkflowError('execute_entity_task cannot be used on ' 'global tasks.') # Should be iterable gdirs = utils.tolist(gdirs) ng = len(gdirs) if ng == 0: log.workflow('Called entity task %s on 0 glaciers. Returning...', task.__name__) return log.workflow('Execute entity task %s on %d glaciers', task.__name__, ng) pc = _pickle_copier(task, kwargs) if _have_ogmpi: if ogmpi.OGGM_MPI_COMM is not None: return ogmpi.mpi_master_spin_tasks(pc, gdirs) if cfg.PARAMS['use_multiprocessing'] and ng > 1: mppool = init_mp_pool(cfg.CONFIG_MODIFIED) out = mppool.map(pc, gdirs, chunksize=1) else: if ng > 3: log.workflow( 'WARNING: you are trying to run an entity task on ' '%d glaciers with multiprocessing turned off. OGGM ' 'will run faster with multiprocessing turned on.', ng) out = [pc(gdir) for gdir in gdirs] return out
def execute_entity_task(task, gdirs, **kwargs): """Execute a task on gdirs. If you asked for multiprocessing, it will do it. If ``task`` has more arguments than `gdir` they have to be keyword arguments. Parameters ---------- task : function the entity task to apply gdirs : list of :py:class:`oggm.GlacierDirectory` objects the glacier directories to process """ # If not iterable it's ok try: len(gdirs) except TypeError: gdirs = [gdirs] if len(gdirs) == 0: return log.workflow('Execute entity task %s on %d glaciers', task.__name__, len(gdirs)) if task.__dict__.get('global_task', False): return task(gdirs, **kwargs) pc = _pickle_copier(task, kwargs) if _have_ogmpi: if ogmpi.OGGM_MPI_COMM is not None: return ogmpi.mpi_master_spin_tasks(pc, gdirs) if cfg.PARAMS['use_multiprocessing']: mppool = init_mp_pool(cfg.CONFIG_MODIFIED) out = mppool.map(pc, gdirs, chunksize=1) else: out = [pc(gdir) for gdir in gdirs] return out
def execute_entity_task(task, gdirs, **kwargs): """Execute a task on gdirs. If you asked for multiprocessing, it will do it. If ``task`` has more arguments than `gdir` they have to be keyword arguments. Parameters ---------- task : function or sequence of functions The entity task(s) to apply. Can be None, in which case each gdir is expected to be a tuple of (task, gdir). When passing a sequence, each item can also optionally be a tuple of (task, dictionary). In this case the dictionary items will be passed to the task as kwargs. gdirs : list of :py:class:`oggm.GlacierDirectory` objects The glacier directories to process. Each individual gdir can optionally be a tuple of (gdir, dictionary). In this case, the values in the dictionary will be passed to the task as keyword arguments for that specific gdir. Returns ------- List of results from task. Last task if a list of tasks was given. """ # Normalize task into list of tuples for simplicity if not isinstance(task, Sequence): task = [task] tasks = [] for t in task: if isinstance(t, tuple): tasks.append(t) else: tasks.append((t, {})) # Reject global tasks for t in tasks: if t[0].__dict__.get('is_global_task', False): raise InvalidWorkflowError('execute_entity_task cannot be used on ' 'global tasks.') # Should be iterable gdirs = utils.tolist(gdirs) ng = len(gdirs) if ng == 0: log.workflow('Called execute_entity_task on 0 glaciers. Returning...') return log.workflow('Execute entity tasks [%s] on %d glaciers', ', '.join([t[0].__name__ for t in tasks]), ng) pc = _pickle_copier(tasks, kwargs) if _have_ogmpi: if ogmpi.OGGM_MPI_COMM is not None: return ogmpi.mpi_master_spin_tasks(pc, gdirs) if cfg.PARAMS['use_multiprocessing'] and ng > 1: mppool = init_mp_pool(cfg.CONFIG_MODIFIED) out = mppool.map(pc, gdirs, chunksize=1) else: if ng > 3: log.workflow( 'WARNING: you are trying to run an entity task on ' '%d glaciers with multiprocessing turned off. OGGM ' 'will run faster with multiprocessing turned on.', ng) out = [pc(gdir) for gdir in gdirs] return out