def init(profile="mpi"): """Initialize pyDive. :param str profile: The name of the cluster profile of *IPython.parallel*. Has to be an MPI-profile.\ Defaults to 'mpi'. """ # init direct view global view client = Client(profile=profile) client.clear() view = client[:] view.block = True view.execute( """\ import numpy as np from mpi4py import MPI import h5py as h5 import os, sys import psutil import math os.environ["onTarget"] = 'True' from pyDive import structured from pyDive import algorithm from pyDive.distribution import interengine try: import pyDive.arrays.local.h5_ndarray except ImportError: pass try: import pyDive.arrays.local.ad_ndarray except ImportError: pass try: import pyDive.arrays.local.gpu_ndarray import pycuda.autoinit except ImportError: pass """ ) # get number of processes per node (ppn) def hostname(): import socket return socket.gethostname() hostnames = view.apply(interactive(hostname)) global ppn ppn = max(Counter(hostnames).values()) # mpi ranks get_rank = interactive(lambda: MPI.COMM_WORLD.Get_rank()) all_ranks = view.apply(get_rank) view["target2rank"] = all_ranks
def init(profile='mpi'): """Initialize pyDive. :param str profile: The name of the cluster profile of *IPython.parallel*. Has to be an MPI-profile.\ Defaults to 'mpi'. """ # init direct view global view client = Client(profile=profile) client.clear() view = client[:] view.block = True view.execute('''\ import numpy as np from mpi4py import MPI import h5py as h5 import os, sys import psutil import math os.environ["onTarget"] = 'True' from pyDive import structured from pyDive import algorithm from pyDive.distribution import interengine try: import pyDive.arrays.local.h5_ndarray except ImportError: pass try: import pyDive.arrays.local.ad_ndarray except ImportError: pass try: import pyDive.arrays.local.gpu_ndarray import pycuda.autoinit except ImportError: pass ''') # get number of processes per node (ppn) def hostname(): import socket return socket.gethostname() hostnames = view.apply(interactive(hostname)) global ppn ppn = max(Counter(hostnames).values()) # mpi ranks get_rank = interactive(lambda: MPI.COMM_WORLD.Get_rank()) all_ranks = view.apply(get_rank) view['target2rank'] = all_ranks
def __mp_mc_setup( generator, kernel, M = 100, **kwargs ) : ## The parallel computing part: use synchronous computations cli = mp.Client( ) clu = cli.direct_view( ) clu.clear( block = True ) ## Make the workers import the necessary dependencies clu.execute( 'import numpy as np', block = True ) ## The crossing tree toolkit clu.execute( 'from crossing_tree import xtree_build', block = True ) ## The generators clu.execute( 'from Weierstrass import synth_Weier', block = True ) clu.execute( 'from synthfbmcircul import synth_fbm, synth_fgn', block = True ) clu.execute( 'from hsssi_processes import synth_Rosenblatt, synth_Hermite3, synth_Hermite4', block = True ) # Distribute the workload evenly among the members of the cluster clu.scatter( 'local_replications', range( M ), block = True ) ## The main problem is that it is possible that each worker reads ## its seed from the same source and at the same time, which would ## produces dangerously correlated results, even meaningless. Therefore ## before starting the kernels, let the parent process generate ## some entropy for each child process. ## http://stackoverflow.com/questions/2396209/best-seed-for-parallel-process ## Generate 32bit values uniformly at random. seeds = np.random.randint( 0x7FFFFFFF, size = len( cli ) ) ## Alternative way for UNIX systems is to read from /dev/random. ## with open( "/dev/random", "rb" ) as dev : ## seeds = [ struct.unpack( 'I', dev.read( 4 ) )[ 0 ] for c in cli ] ## Dispatch individual seed values to each worker. for c, seed in zip( cli, seeds ) : c.push( { 'seed' : seed } ) ## Pass the necesary environment to the cluster. clu.push({ 'generator' : generator, 'local_kwargs' : kwargs, 'local_M': M, 'kernel': interactive( kernel ) }, block = True ) return clu
def reduce(array, op): """Perform a tree-like reduction over all axes of *array*. :param array: *pyDive.ndarray*, *pyDive.h5_ndarray* or *pyDive.cloned_ndarray* to be reduced :param numpy-ufunc op: reduce operation, e.g. *numpy.add*. If the hdf5 data exceeds the memory limit (currently 25% of the combined main memory of all cluster nodes)\ the data will be read block-wise so that a block fits into memory. """ def reduce_wrapper(array_name, op_name): array = globals()[array_name] op = eval("np." + op_name) return algorithm.__tree_reduce(array, axis=None, op=op) # reduction over all axes view = com.getView() tmp_targets = view.targets # save current target list if type(array) == VirtualArrayOfStructs: view.targets = array.firstArray.target_ranks else: view.targets = array.target_ranks result = None if (hasattr(array, "arraytype") and array.arraytype in hdd_arraytypes) or type(array) in hdd_arraytypes: for chunk in fragment(array): array_name = repr(chunk) targets_results = view.apply(interactive(reduce_wrapper), array_name, op.__name__) chunk_result = op.reduce(targets_results) # reduce over targets' results if result is None: result = chunk_result else: result = op(result, chunk_result) else: array_name = repr(array) targets_results = view.apply(interactive(reduce_wrapper), array_name, op.__name__) result = op.reduce(targets_results) # reduce over targets' results view.targets = tmp_targets # restore target list return result
def init(): #init direct view global view view = Client(profile='mpi')[:] view.block = True view.execute('from numpy import *') view.execute('from mpi4py import MPI') view.execute('import h5py as h5') view.execute('import os') view.run('ndarray/interengine.py') get_rank = interactive(lambda: MPI.COMM_WORLD.Get_rank()) all_ranks = view.apply(get_rank) view['target2rank'] = all_ranks
def solveBellman_parallel(Para,Vf): ''' Solves the bellman equation for the states computed need to run computePosteriors before this. ''' cl = Client() v = cl[:]#get the view for the client v.block = True X = getX(Para) Vs_old = Vf(X).flatten() diff = 100. diff_old = diff a = 0.05 n_reset = 10 #Setup the bellman Map on the engines v.execute('from bayesian_log import BellmanMap') v.execute('T = BellmanMap(Para)') while diff > 1e-4: #send the value function to the engines and create new value function v['Vf'] = Vf v.execute('Vnew = T(Vf)') #Now apply Vnew on engines to each element of Para.domain f = interactive(lambda state: Vnew(state)[0]) Vs = hstack(v.map(f,Para.domain)) #Now fit the new value function c_old = Vf.f.get_c() Vf.fit(X,Vs) #mix between old coefficients and new ones Vf.f.set_c(a*Vf.f.get_c()+(1-a)*c_old) diff = max(abs((Vs-Vs_old)/Vs_old))/a if diff > diff_old and n_reset >9: a /= 2. n_reset = 0 diff_old = diff n_reset += 1 print diff Vs_old = Vs return Vf
def __bestStepSize(arrays, axis, memory_limit): view = com.getView() # minimum amount of memory available and memory needed, both per engine get_mem_av_node = interactive(lambda: psutil.virtual_memory().available) tmp_targets = view.targets view.targets = 'all' mem_av = min(view.apply(get_mem_av_node)) / com.getPPN() mem_needed = sum(a.nbytes for a in arrays) / len(view) view.targets = tmp_targets # edge length of the whole array edge_length = arrays[0].shape[axis] # maximum edge length on one engine according to the available memory step_size = memory_limit * edge_length * mem_av / mem_needed if step_size >= edge_length: return edge_length # round 'step_size' down to nearest power of two return pow(2, int(math.log(step_size, 2)))
def map(f, *arrays, **kwargs): """Applies *f* on :term:`engine` on local arrays related to *arrays*. Example: :: cluster_array = pyDive.ones(shape=[100], distaxes=0) cluster_array *= 2.0 # equivalent to pyDive.map(lambda a: a *= 2.0, cluster_array) # a is the local numpy-array of *cluster_array* Or, as a decorator: :: @pyDive.map def twice(a): a *= 2.0 twice(cluster_array) :param callable f: function to be called on :term:`engine`. Has to accept *numpy-arrays* and *kwargs* :param arrays: list of arrays including *pyDive.ndarrays*, *pyDive.h5_ndarrays* or *pyDive.cloned_ndarrays* :param kwargs: user-specified keyword arguments passed to *f* :raises AssertionError: if the *shapes* of *pyDive.ndarrays* and *pyDive.h5_ndarrays* do not match :raises AssertionError: if the *distaxes* attributes of *pyDive.ndarrays* and *pyDive.h5_ndarrays* do not match Notes: - If the hdf5 data exceeds the memory limit (currently 25% of the combined main memory of all cluster nodes)\ the data will be read block-wise so that a block fits into memory. - *map* chooses the list of *engines* from the **first** element of *arrays*. On these engines *f* is called.\ If the first array is a *pyDive.h5_ndarray* all engines will be used. - *map* is not writing data back to a *pyDive.h5_ndarray* yet. - *map* does not equalize the element distribution of *pyDive.ndarrays* before execution. """ if not arrays: # decorator mode def map_deco(*arrays, **kwargs): map(f, *arrays, **kwargs) return map_deco def map_wrapper(f, array_names, **kwargs): arrays = [globals()[array_name] for array_name in array_names] f(*arrays, **kwargs) view = com.getView() tmp_targets = view.targets # save current target list if type(arrays[0]) == VirtualArrayOfStructs: view.targets = arrays[0].firstArray.target_ranks else: view.targets = arrays[0].target_ranks hdd_arrays = [a for a in arrays if (hasattr(a, "arraytype") and a.arraytype in hdd_arraytypes) or type(a) in hdd_arraytypes] if hdd_arrays: cloned_arrays = [a for a in arrays if (hasattr(a, "arraytype") and a.arraytype is cloned_ndarray) or type(a) is cloned_ndarray] other_arrays = [a for a in arrays if not ((hasattr(a, "arraytype") and a.arraytype is cloned_ndarray) or type(a) is cloned_ndarray)] cloned_arrays_ids = [id(a) for a in cloned_arrays] other_arrays_ids = [id(a) for a in other_arrays] for fragments in fragment(*other_arrays): it_other_arrays = iter(other_arrays) it_cloned_arrays = iter(cloned_arrays) array_names = [] for a in arrays: if id(a) in cloned_arrays_ids: array_names.append(repr(it_cloned_arrays.next())) continue if id(a) in other_arrays_ids: array_names.append(repr(it_other_arrays.next())) continue view.apply(interactive(map_wrapper), interactive(f), array_names, **kwargs) else: array_names = [repr(a) for a in arrays] view.apply(interactive(map_wrapper), interactive(f), array_names, **kwargs) view.targets = tmp_targets # restore target list
def mapReduce(map_func, reduce_op, *arrays, **kwargs): """Applies *map_func* on :term:`engine` on local arrays related to *arrays* and reduces its result in a tree-like fashion over all axes. Example: :: cluster_array = pyDive.ones(shape=[100], distaxes=0) s = pyDive.mapReduce(lambda a: a**2, np.add, cluster_array) # a is the local numpy-array of *cluster_array* assert s == 100 :param callable f: function to be called on :term:`engine`. Has to accept *numpy-arrays* and *kwargs* :param numpy-ufunc reduce_op: reduce operation, e.g. *numpy.add*. :param arrays: list of arrays including *pyDive.ndarrays*, *pyDive.h5_ndarrays* or *pyDive.cloned_ndarrays* :param kwargs: user-specified keyword arguments passed to *f* :raises AssertionError: if the *shapes* of *pyDive.ndarrays* and *pyDive.h5_ndarrays* do not match :raises AssertionError: if the *distaxes* attributes of *pyDive.ndarrays* and *pyDive.h5_ndarrays* do not match Notes: - If the hdf5 data exceeds the memory limit (currently 25% of the combined main memory of all cluster nodes)\ the data will be read block-wise so that a block fits into memory. - *mapReduce* chooses the list of *engines* from the **first** element of *arrays*. On these engines the mapReduce will be executed.\ If the first array is a *pyDive.h5_ndarray* all engines will be used. - *mapReduce* is not writing data back to a *pyDive.h5_ndarray* yet. - *mapReduce* does not equalize the element distribution of *pyDive.ndarrays* before execution. """ def mapReduce_wrapper(map_func, reduce_op_name, array_names, **kwargs): arrays = [globals()[array_name] for array_name in array_names] reduce_op = eval("np." + reduce_op_name) return algorithm.__tree_reduce(map_func(*arrays, **kwargs), axis=None, op=reduce_op) view = com.getView() tmp_targets = view.targets # save current target list if type(arrays[0]) == VirtualArrayOfStructs: view.targets = arrays[0].firstArray.target_ranks else: view.targets = arrays[0].target_ranks result = None hdd_arrays = [a for a in arrays if (hasattr(a, "arraytype") and a.arraytype in hdd_arraytypes) or type(a) in hdd_arraytypes] if hdd_arrays: cloned_arrays = [a for a in arrays if (hasattr(a, "arraytype") and a.arraytype is cloned_ndarray) or type(a) is cloned_ndarray] other_arrays = [a for a in arrays if not ((hasattr(a, "arraytype") and a.arraytype is cloned_ndarray) or type(a) is cloned_ndarray)] cloned_arrays_ids = [id(a) for a in cloned_arrays] other_arrays_ids = [id(a) for a in other_arrays] for fragments in fragment(*other_arrays): it_other_arrays = iter(other_arrays) it_cloned_arrays = iter(cloned_arrays) array_names = [] for a in arrays: if id(a) in cloned_arrays_ids: array_names.append(repr(it_cloned_arrays.next())) continue if id(a) in other_arrays_ids: array_names.append(repr(it_other_arrays.next())) continue targets_results = view.apply(interactive(mapReduce_wrapper),\ interactive(map_func), reduce_op.__name__, array_names, **kwargs) fragment_result = reduce_op.reduce(targets_results) # reduce over targets' results if result is None: result = fragment_result else: result = reduce_op(result, fragment_result) else: array_names = [repr(a) for a in arrays] targets_results = view.apply(interactive(mapReduce_wrapper),\ interactive(map_func), reduce_op.__name__, array_names, **kwargs) result = reduce_op.reduce(targets_results) # reduce over targets' results view.targets = tmp_targets # restore target list return result