def init(profile="mpi"):
"""Initialize pyDive.
:param str profile: The name of the cluster profile of *IPython.parallel*. Has to be an MPI-profile.\
Defaults to 'mpi'.
"""
# init direct view
global view
client = Client(profile=profile)
client.clear()
view = client[:]
view.block = True
view.execute(
"""\
import numpy as np
from mpi4py import MPI
import h5py as h5
import os, sys
import psutil
import math
os.environ["onTarget"] = 'True'
from pyDive import structured
from pyDive import algorithm
from pyDive.distribution import interengine
try:
import pyDive.arrays.local.h5_ndarray
except ImportError:
pass
try:
import pyDive.arrays.local.ad_ndarray
except ImportError:
pass
try:
import pyDive.arrays.local.gpu_ndarray
import pycuda.autoinit
except ImportError:
pass
"""
)
# get number of processes per node (ppn)
def hostname():
import socket
return socket.gethostname()
hostnames = view.apply(interactive(hostname))
global ppn
ppn = max(Counter(hostnames).values())
# mpi ranks
get_rank = interactive(lambda: MPI.COMM_WORLD.Get_rank())
all_ranks = view.apply(get_rank)
view["target2rank"] = all_ranks
def init(profile='mpi'):
"""Initialize pyDive.
:param str profile: The name of the cluster profile of *IPython.parallel*. Has to be an MPI-profile.\
Defaults to 'mpi'.
"""
# init direct view
global view
client = Client(profile=profile)
client.clear()
view = client[:]
view.block = True
view.execute('''\
import numpy as np
from mpi4py import MPI
import h5py as h5
import os, sys
import psutil
import math
os.environ["onTarget"] = 'True'
from pyDive import structured
from pyDive import algorithm
from pyDive.distribution import interengine
try:
import pyDive.arrays.local.h5_ndarray
except ImportError:
pass
try:
import pyDive.arrays.local.ad_ndarray
except ImportError:
pass
try:
import pyDive.arrays.local.gpu_ndarray
import pycuda.autoinit
except ImportError:
pass
''')
# get number of processes per node (ppn)
def hostname():
import socket
return socket.gethostname()
hostnames = view.apply(interactive(hostname))
global ppn
ppn = max(Counter(hostnames).values())
# mpi ranks
get_rank = interactive(lambda: MPI.COMM_WORLD.Get_rank())
all_ranks = view.apply(get_rank)
view['target2rank'] = all_ranks
def __mp_mc_setup( generator, kernel, M = 100, **kwargs ) :
## The parallel computing part: use synchronous computations
cli = mp.Client( )
clu = cli.direct_view( )
clu.clear( block = True )
## Make the workers import the necessary dependencies
clu.execute( 'import numpy as np', block = True )
## The crossing tree toolkit
clu.execute( 'from crossing_tree import xtree_build', block = True )
## The generators
clu.execute( 'from Weierstrass import synth_Weier', block = True )
clu.execute( 'from synthfbmcircul import synth_fbm, synth_fgn', block = True )
clu.execute( 'from hsssi_processes import synth_Rosenblatt, synth_Hermite3, synth_Hermite4', block = True )
# Distribute the workload evenly among the members of the cluster
clu.scatter( 'local_replications', range( M ), block = True )
## The main problem is that it is possible that each worker reads
## its seed from the same source and at the same time, which would
## produces dangerously correlated results, even meaningless. Therefore
## before starting the kernels, let the parent process generate
## some entropy for each child process.
## http://stackoverflow.com/questions/2396209/best-seed-for-parallel-process
## Generate 32bit values uniformly at random.
seeds = np.random.randint( 0x7FFFFFFF, size = len( cli ) )
## Alternative way for UNIX systems is to read from /dev/random.
## with open( "/dev/random", "rb" ) as dev :
## seeds = [ struct.unpack( 'I', dev.read( 4 ) )[ 0 ] for c in cli ]
## Dispatch individual seed values to each worker.
for c, seed in zip( cli, seeds ) :
c.push( { 'seed' : seed } )
## Pass the necesary environment to the cluster.
clu.push({ 'generator' : generator, 'local_kwargs' : kwargs, 'local_M': M,
'kernel': interactive( kernel ) }, block = True )
return clu
def reduce(array, op):
"""Perform a tree-like reduction over all axes of *array*.
:param array: *pyDive.ndarray*, *pyDive.h5_ndarray* or *pyDive.cloned_ndarray* to be reduced
:param numpy-ufunc op: reduce operation, e.g. *numpy.add*.
If the hdf5 data exceeds the memory limit (currently 25% of the combined main memory of all cluster nodes)\
the data will be read block-wise so that a block fits into memory.
"""
def reduce_wrapper(array_name, op_name):
array = globals()[array_name]
op = eval("np." + op_name)
return algorithm.__tree_reduce(array, axis=None, op=op) # reduction over all axes
view = com.getView()
tmp_targets = view.targets # save current target list
if type(array) == VirtualArrayOfStructs:
view.targets = array.firstArray.target_ranks
else:
view.targets = array.target_ranks
result = None
if (hasattr(array, "arraytype") and array.arraytype in hdd_arraytypes) or type(array) in hdd_arraytypes:
for chunk in fragment(array):
array_name = repr(chunk)
targets_results = view.apply(interactive(reduce_wrapper), array_name, op.__name__)
chunk_result = op.reduce(targets_results) # reduce over targets' results
if result is None:
result = chunk_result
else:
result = op(result, chunk_result)
else:
array_name = repr(array)
targets_results = view.apply(interactive(reduce_wrapper), array_name, op.__name__)
result = op.reduce(targets_results) # reduce over targets' results
view.targets = tmp_targets # restore target list
return result
def init():
#init direct view
global view
view = Client(profile='mpi')[:]
view.block = True
view.execute('from numpy import *')
view.execute('from mpi4py import MPI')
view.execute('import h5py as h5')
view.execute('import os')
view.run('ndarray/interengine.py')
get_rank = interactive(lambda: MPI.COMM_WORLD.Get_rank())
all_ranks = view.apply(get_rank)
view['target2rank'] = all_ranks
def solveBellman_parallel(Para,Vf):
'''
Solves the bellman equation for the states computed need to run
computePosteriors before this.
'''
cl = Client()
v = cl[:]#get the view for the client
v.block = True
X = getX(Para)
Vs_old = Vf(X).flatten()
diff = 100.
diff_old = diff
a = 0.05
n_reset = 10
#Setup the bellman Map on the engines
v.execute('from bayesian_log import BellmanMap')
v.execute('T = BellmanMap(Para)')
while diff > 1e-4:
#send the value function to the engines and create new value function
v['Vf'] = Vf
v.execute('Vnew = T(Vf)')
#Now apply Vnew on engines to each element of Para.domain
f = interactive(lambda state: Vnew(state)[0])
Vs = hstack(v.map(f,Para.domain))
#Now fit the new value function
c_old = Vf.f.get_c()
Vf.fit(X,Vs)
#mix between old coefficients and new ones
Vf.f.set_c(a*Vf.f.get_c()+(1-a)*c_old)
diff = max(abs((Vs-Vs_old)/Vs_old))/a
if diff > diff_old and n_reset >9:
a /= 2.
n_reset = 0
diff_old = diff
n_reset += 1
print diff
Vs_old = Vs
return Vf
def __bestStepSize(arrays, axis, memory_limit):
view = com.getView()
# minimum amount of memory available and memory needed, both per engine
get_mem_av_node = interactive(lambda: psutil.virtual_memory().available)
tmp_targets = view.targets
view.targets = 'all'
mem_av = min(view.apply(get_mem_av_node)) / com.getPPN()
mem_needed = sum(a.nbytes for a in arrays) / len(view)
view.targets = tmp_targets
# edge length of the whole array
edge_length = arrays[0].shape[axis]
# maximum edge length on one engine according to the available memory
step_size = memory_limit * edge_length * mem_av / mem_needed
if step_size >= edge_length:
return edge_length
# round 'step_size' down to nearest power of two
return pow(2, int(math.log(step_size, 2)))
def __bestStepSize(arrays, axis, memory_limit):
view = com.getView()
# minimum amount of memory available and memory needed, both per engine
get_mem_av_node = interactive(lambda: psutil.virtual_memory().available)
tmp_targets = view.targets
view.targets = 'all'
mem_av = min(view.apply(get_mem_av_node)) / com.getPPN()
mem_needed = sum(a.nbytes for a in arrays) / len(view)
view.targets = tmp_targets
# edge length of the whole array
edge_length = arrays[0].shape[axis]
# maximum edge length on one engine according to the available memory
step_size = memory_limit * edge_length * mem_av / mem_needed
if step_size >= edge_length:
return edge_length
# round 'step_size' down to nearest power of two
return pow(2, int(math.log(step_size, 2)))
def map(f, *arrays, **kwargs):
"""Applies *f* on :term:`engine` on local arrays related to *arrays*.
Example: ::
cluster_array = pyDive.ones(shape=[100], distaxes=0)
cluster_array *= 2.0
# equivalent to
pyDive.map(lambda a: a *= 2.0, cluster_array) # a is the local numpy-array of *cluster_array*
Or, as a decorator: ::
@pyDive.map
def twice(a):
a *= 2.0
twice(cluster_array)
:param callable f: function to be called on :term:`engine`. Has to accept *numpy-arrays* and *kwargs*
:param arrays: list of arrays including *pyDive.ndarrays*, *pyDive.h5_ndarrays* or *pyDive.cloned_ndarrays*
:param kwargs: user-specified keyword arguments passed to *f*
:raises AssertionError: if the *shapes* of *pyDive.ndarrays* and *pyDive.h5_ndarrays* do not match
:raises AssertionError: if the *distaxes* attributes of *pyDive.ndarrays* and *pyDive.h5_ndarrays* do not match
Notes:
- If the hdf5 data exceeds the memory limit (currently 25% of the combined main memory of all cluster nodes)\
the data will be read block-wise so that a block fits into memory.
- *map* chooses the list of *engines* from the **first** element of *arrays*. On these engines *f* is called.\
If the first array is a *pyDive.h5_ndarray* all engines will be used.
- *map* is not writing data back to a *pyDive.h5_ndarray* yet.
- *map* does not equalize the element distribution of *pyDive.ndarrays* before execution.
"""
if not arrays:
# decorator mode
def map_deco(*arrays, **kwargs):
map(f, *arrays, **kwargs)
return map_deco
def map_wrapper(f, array_names, **kwargs):
arrays = [globals()[array_name] for array_name in array_names]
f(*arrays, **kwargs)
view = com.getView()
tmp_targets = view.targets # save current target list
if type(arrays[0]) == VirtualArrayOfStructs:
view.targets = arrays[0].firstArray.target_ranks
else:
view.targets = arrays[0].target_ranks
hdd_arrays = [a for a in arrays if (hasattr(a, "arraytype") and a.arraytype in hdd_arraytypes) or type(a) in hdd_arraytypes]
if hdd_arrays:
cloned_arrays = [a for a in arrays if (hasattr(a, "arraytype") and a.arraytype is cloned_ndarray) or type(a) is cloned_ndarray]
other_arrays = [a for a in arrays if not ((hasattr(a, "arraytype") and a.arraytype is cloned_ndarray) or type(a) is cloned_ndarray)]
cloned_arrays_ids = [id(a) for a in cloned_arrays]
other_arrays_ids = [id(a) for a in other_arrays]
for fragments in fragment(*other_arrays):
it_other_arrays = iter(other_arrays)
it_cloned_arrays = iter(cloned_arrays)
array_names = []
for a in arrays:
if id(a) in cloned_arrays_ids:
array_names.append(repr(it_cloned_arrays.next()))
continue
if id(a) in other_arrays_ids:
array_names.append(repr(it_other_arrays.next()))
continue
view.apply(interactive(map_wrapper), interactive(f), array_names, **kwargs)
else:
array_names = [repr(a) for a in arrays]
view.apply(interactive(map_wrapper), interactive(f), array_names, **kwargs)
view.targets = tmp_targets # restore target list
def mapReduce(map_func, reduce_op, *arrays, **kwargs):
"""Applies *map_func* on :term:`engine` on local arrays related to *arrays*
and reduces its result in a tree-like fashion over all axes.
Example: ::
cluster_array = pyDive.ones(shape=[100], distaxes=0)
s = pyDive.mapReduce(lambda a: a**2, np.add, cluster_array) # a is the local numpy-array of *cluster_array*
assert s == 100
:param callable f: function to be called on :term:`engine`. Has to accept *numpy-arrays* and *kwargs*
:param numpy-ufunc reduce_op: reduce operation, e.g. *numpy.add*.
:param arrays: list of arrays including *pyDive.ndarrays*, *pyDive.h5_ndarrays* or *pyDive.cloned_ndarrays*
:param kwargs: user-specified keyword arguments passed to *f*
:raises AssertionError: if the *shapes* of *pyDive.ndarrays* and *pyDive.h5_ndarrays* do not match
:raises AssertionError: if the *distaxes* attributes of *pyDive.ndarrays* and *pyDive.h5_ndarrays* do not match
Notes:
- If the hdf5 data exceeds the memory limit (currently 25% of the combined main memory of all cluster nodes)\
the data will be read block-wise so that a block fits into memory.
- *mapReduce* chooses the list of *engines* from the **first** element of *arrays*. On these engines the mapReduce will be executed.\
If the first array is a *pyDive.h5_ndarray* all engines will be used.
- *mapReduce* is not writing data back to a *pyDive.h5_ndarray* yet.
- *mapReduce* does not equalize the element distribution of *pyDive.ndarrays* before execution.
"""
def mapReduce_wrapper(map_func, reduce_op_name, array_names, **kwargs):
arrays = [globals()[array_name] for array_name in array_names]
reduce_op = eval("np." + reduce_op_name)
return algorithm.__tree_reduce(map_func(*arrays, **kwargs), axis=None, op=reduce_op)
view = com.getView()
tmp_targets = view.targets # save current target list
if type(arrays[0]) == VirtualArrayOfStructs:
view.targets = arrays[0].firstArray.target_ranks
else:
view.targets = arrays[0].target_ranks
result = None
hdd_arrays = [a for a in arrays if (hasattr(a, "arraytype") and a.arraytype in hdd_arraytypes) or type(a) in hdd_arraytypes]
if hdd_arrays:
cloned_arrays = [a for a in arrays if (hasattr(a, "arraytype") and a.arraytype is cloned_ndarray) or type(a) is cloned_ndarray]
other_arrays = [a for a in arrays if not ((hasattr(a, "arraytype") and a.arraytype is cloned_ndarray) or type(a) is cloned_ndarray)]
cloned_arrays_ids = [id(a) for a in cloned_arrays]
other_arrays_ids = [id(a) for a in other_arrays]
for fragments in fragment(*other_arrays):
it_other_arrays = iter(other_arrays)
it_cloned_arrays = iter(cloned_arrays)
array_names = []
for a in arrays:
if id(a) in cloned_arrays_ids:
array_names.append(repr(it_cloned_arrays.next()))
continue
if id(a) in other_arrays_ids:
array_names.append(repr(it_other_arrays.next()))
continue
targets_results = view.apply(interactive(mapReduce_wrapper),\
interactive(map_func), reduce_op.__name__, array_names, **kwargs)
fragment_result = reduce_op.reduce(targets_results) # reduce over targets' results
if result is None:
result = fragment_result
else:
result = reduce_op(result, fragment_result)
else:
array_names = [repr(a) for a in arrays]
targets_results = view.apply(interactive(mapReduce_wrapper),\
interactive(map_func), reduce_op.__name__, array_names, **kwargs)
result = reduce_op.reduce(targets_results) # reduce over targets' results
view.targets = tmp_targets # restore target list
return result
