def make_targets_comm(targets):
world = get_comm_world()
world_rank = world.rank
if len(targets) > world.size:
raise ValueError("The number of engines (%s) is less than the number"
" of targets you want (%s)." % (world.size - 1,
len(targets)))
targets = targets or list(range(world.size - 1))
# get a universal name for the out comm
if world_rank == 0:
comm_name = uid()
else:
comm_name = ''
comm_name = world.bcast(comm_name)
# create a mapping from the targets to world ranks
all_ranks = range(1, world.size)
all_targets = range(world.size - 1)
target_to_rank_map = {t: r for t, r in zip(all_targets, all_ranks)}
# map the targets to the world ranks
mapped_targets = [target_to_rank_map[t] for t in targets]
# create the targets comm
targets_group = world.group.Incl(mapped_targets)
targets_comm = world.Create(targets_group)
return _set_on_main(comm_name, targets_comm)
def _setup_context_key(self):
"""
Create a dict on the engines which will hold everything from
this context.
"""
context_key = uid()
cmd = ("import types, sys;" "%s = types.ModuleType('%s');")
cmd %= (context_key, context_key)
self._execute(cmd, self.all_targets)
return context_key
def _setup_context_key(self):
"""
Create a dict on the engines which will hold everything from
this context.
"""
context_key = uid()
cmd = ("import types, sys;"
"%s = types.ModuleType('%s');")
cmd %= (context_key, context_key)
self._execute(cmd, targets=range(len(self.view)))
return context_key
def apply(self, func, args=None, kwargs=None, targets=None,
return_proxy=False):
"""
Analogous to IPython.parallel.view.apply_sync
Parameters
----------
func : function
args : tuple
positional arguments to func
kwargs : dict
key word arguments to func
targets : sequence of integers
engines func is to be run on.
return_proxy : bool
if False (default) return result.
if True, return the name (as a str) of the result on the engines.
Returns
-------
if return_proxy:
return a list of the results on the each engine.
else:
the name of the result on all the engines.
"""
def func_wrapper(func, result_name, args, kwargs):
"""
Function which calls the applied function after grabbing all the
arguments on the engines that are passed in as names of the form
`__distarray__<some uuid>`.
"""
main = __import__('__main__')
prefix = main.distarray.utils.DISTARRAY_BASE_NAME
# convert args
args = list(args)
for i, a in enumerate(args):
if (isinstance(a, str) and a.startswith(prefix)):
args[i] = main.reduce(getattr, [main] + a.split('.'))
args = tuple(args)
# convert kwargs
for k in kwargs.keys():
val = kwargs[k]
if (isinstance(val, str) and val.startswith(prefix)):
kwargs[k] = main.reduce(getattr, [main] + val.split('.'))
if result_name is not None:
setattr(main, result_name, func(*args, **kwargs))
return result_name
else:
return func(*args, **kwargs)
# default arguments
result_name = None if not return_proxy else uid()
args = () if args is None else args
kwargs = {} if kwargs is None else kwargs
wrapped_args = (func, result_name, args, kwargs)
targets = self.targets if targets is None else targets
result = self.view._really_apply(func_wrapper, args=wrapped_args,
targets=targets, block=True)
if return_proxy:
# result is a list of the same name 4 times, so just return 1.
return result[0]
else:
return result
def _generate_key(self):
""" Generate a unique key name for this context. """
key = "%s.%s" % (self.context_key, uid())
return key
def _generate_key(self):
""" Generate a unique key name for this context. """
key = "%s_%s" % (self.context_key, uid())
return key
def apply(self, func, args=None, kwargs=None, targets=None):
"""
Analogous to IPython.parallel.view.apply_sync
Parameters
----------
func : function
args : tuple
positional arguments to func
kwargs : dict
key word arguments to func
targets : sequence of integers
engines func is to be run on.
Returns
-------
return a list of the results on the each engine.
"""
def func_wrapper(func, apply_nonce, context_key, args, kwargs):
"""
Function which calls the applied function after grabbing all the
arguments on the engines that are passed in as names of the form
`__distarray__<some uuid>`.
"""
from importlib import import_module
import types
main = import_module('__main__')
prefix = main.distarray.utils.DISTARRAY_BASE_NAME
main.proxyize.set_state(apply_nonce)
# Modify func to change the namespace it executes in.
# but builtins don't have __code__, __globals__, etc.
if not isinstance(func, types.BuiltinFunctionType):
# get func's building blocks first
func_code = func.__code__
func_globals = func.__globals__ # noqa we don't need these.
func_name = func.__name__
func_defaults = func.__defaults__
func_closure = func.__closure__
# build the func's new execution environment
main.__dict__.update({'context_key': context_key})
new_func_globals = main.__dict__
# create the new func
func = types.FunctionType(func_code, new_func_globals,
func_name, func_defaults,
func_closure)
# convert args
args = list(args)
for i, a in enumerate(args):
if (isinstance(a, str) and a.startswith(prefix)):
args[i] = main.reduce(getattr, [main] + a.split('.'))
args = tuple(args)
# convert kwargs
for k in kwargs.keys():
val = kwargs[k]
if (isinstance(val, str) and val.startswith(prefix)):
kwargs[k] = main.reduce(getattr, [main] + val.split('.'))
return func(*args, **kwargs)
# default arguments
args = () if args is None else args
kwargs = {} if kwargs is None else kwargs
apply_nonce = uid()[13:]
wrapped_args = (func, apply_nonce, self.context_key, args, kwargs)
targets = self.targets if targets is None else targets
return self.view._really_apply(func_wrapper, args=wrapped_args,
targets=targets, block=True)
