def make_targets_comm(targets): world = get_comm_world() world_rank = world.rank if len(targets) > world.size: raise ValueError("The number of engines (%s) is less than the number" " of targets you want (%s)." % (world.size - 1, len(targets))) targets = targets or list(range(world.size - 1)) # get a universal name for the out comm if world_rank == 0: comm_name = uid() else: comm_name = '' comm_name = world.bcast(comm_name) # create a mapping from the targets to world ranks all_ranks = range(1, world.size) all_targets = range(world.size - 1) target_to_rank_map = {t: r for t, r in zip(all_targets, all_ranks)} # map the targets to the world ranks mapped_targets = [target_to_rank_map[t] for t in targets] # create the targets comm targets_group = world.group.Incl(mapped_targets) targets_comm = world.Create(targets_group) return _set_on_main(comm_name, targets_comm)
def _setup_context_key(self): """ Create a dict on the engines which will hold everything from this context. """ context_key = uid() cmd = ("import types, sys;" "%s = types.ModuleType('%s');") cmd %= (context_key, context_key) self._execute(cmd, self.all_targets) return context_key
def _setup_context_key(self): """ Create a dict on the engines which will hold everything from this context. """ context_key = uid() cmd = ("import types, sys;" "%s = types.ModuleType('%s');") cmd %= (context_key, context_key) self._execute(cmd, targets=range(len(self.view))) return context_key
def apply(self, func, args=None, kwargs=None, targets=None, return_proxy=False): """ Analogous to IPython.parallel.view.apply_sync Parameters ---------- func : function args : tuple positional arguments to func kwargs : dict key word arguments to func targets : sequence of integers engines func is to be run on. return_proxy : bool if False (default) return result. if True, return the name (as a str) of the result on the engines. Returns ------- if return_proxy: return a list of the results on the each engine. else: the name of the result on all the engines. """ def func_wrapper(func, result_name, args, kwargs): """ Function which calls the applied function after grabbing all the arguments on the engines that are passed in as names of the form `__distarray__<some uuid>`. """ main = __import__('__main__') prefix = main.distarray.utils.DISTARRAY_BASE_NAME # convert args args = list(args) for i, a in enumerate(args): if (isinstance(a, str) and a.startswith(prefix)): args[i] = main.reduce(getattr, [main] + a.split('.')) args = tuple(args) # convert kwargs for k in kwargs.keys(): val = kwargs[k] if (isinstance(val, str) and val.startswith(prefix)): kwargs[k] = main.reduce(getattr, [main] + val.split('.')) if result_name is not None: setattr(main, result_name, func(*args, **kwargs)) return result_name else: return func(*args, **kwargs) # default arguments result_name = None if not return_proxy else uid() args = () if args is None else args kwargs = {} if kwargs is None else kwargs wrapped_args = (func, result_name, args, kwargs) targets = self.targets if targets is None else targets result = self.view._really_apply(func_wrapper, args=wrapped_args, targets=targets, block=True) if return_proxy: # result is a list of the same name 4 times, so just return 1. return result[0] else: return result
def _generate_key(self): """ Generate a unique key name for this context. """ key = "%s.%s" % (self.context_key, uid()) return key
def _generate_key(self): """ Generate a unique key name for this context. """ key = "%s_%s" % (self.context_key, uid()) return key
def apply(self, func, args=None, kwargs=None, targets=None): """ Analogous to IPython.parallel.view.apply_sync Parameters ---------- func : function args : tuple positional arguments to func kwargs : dict key word arguments to func targets : sequence of integers engines func is to be run on. Returns ------- return a list of the results on the each engine. """ def func_wrapper(func, apply_nonce, context_key, args, kwargs): """ Function which calls the applied function after grabbing all the arguments on the engines that are passed in as names of the form `__distarray__<some uuid>`. """ from importlib import import_module import types main = import_module('__main__') prefix = main.distarray.utils.DISTARRAY_BASE_NAME main.proxyize.set_state(apply_nonce) # Modify func to change the namespace it executes in. # but builtins don't have __code__, __globals__, etc. if not isinstance(func, types.BuiltinFunctionType): # get func's building blocks first func_code = func.__code__ func_globals = func.__globals__ # noqa we don't need these. func_name = func.__name__ func_defaults = func.__defaults__ func_closure = func.__closure__ # build the func's new execution environment main.__dict__.update({'context_key': context_key}) new_func_globals = main.__dict__ # create the new func func = types.FunctionType(func_code, new_func_globals, func_name, func_defaults, func_closure) # convert args args = list(args) for i, a in enumerate(args): if (isinstance(a, str) and a.startswith(prefix)): args[i] = main.reduce(getattr, [main] + a.split('.')) args = tuple(args) # convert kwargs for k in kwargs.keys(): val = kwargs[k] if (isinstance(val, str) and val.startswith(prefix)): kwargs[k] = main.reduce(getattr, [main] + val.split('.')) return func(*args, **kwargs) # default arguments args = () if args is None else args kwargs = {} if kwargs is None else kwargs apply_nonce = uid()[13:] wrapped_args = (func, apply_nonce, self.context_key, args, kwargs) targets = self.targets if targets is None else targets return self.view._really_apply(func_wrapper, args=wrapped_args, targets=targets, block=True)