def map(self, fcn, incols, outcols, dview=None):
"""Evaluates a function for each particle, based on that particle and its
neighbors.
:param fcn: function to be called with ``data[i]``, ``data[nn_i]`` as arguments,
where ``data[nn_i]`` is a 2D array
:param incols: list of column names in the original DataFrame, corresponding
to the columns passed to ``fcn``.
:param outcols: list of new column names, in which the value(s) returned by ``fcn``
will be inserted.
:param dview: optional IPython parallel direct view.
Returns a copy of the original tracks DataFrame, with new columns from ``outcols``.
"""
# Design notes:
# - Things go the fastest and work the best when we send numpy arrays to the engines.
# In other words, avoid pickling at all costs.
# - Pandas objects are slow. The algorithm should work with plain numpy arrays;
# we convert back to DataFrames at the end.
alldata = self.frametracks[incols].values
allresults = np.ones((alldata.shape[0], len(outcols))) * np.nan
def worker(fcn, data, loopindices, coords, nncutoff, n_output_cols):
# This runs only once on each engine so it's ok to have all this setup code
import numpy as np
import scipy.spatial.ckdtree
tree = scipy.spatial.ckdtree.cKDTree(coords, 5)
results = np.ones((len(loopindices), n_output_cols)) * np.nan
neighborlist = tree.query_ball_point(coords[loopindices], nncutoff)
for i, (pindex,
neighbors) in enumerate(zip(loopindices, neighborlist)):
neighbors.remove(pindex)
results[i] = fcn(data[pindex], data[neighbors])
return results
if dview is None:
allresults[self.loopindices] = worker(fcn, alldata,
self.loopindices,
self.coords, self.nncutoff,
len(outcols))
else:
from IPython.parallel.util import interactive
dview.execute('''import numpy as np''')
# To send function to engines, its parent namespace must be the global namespace.
dview['worker'] = interactive(worker)
dview['fcn'] = interactive(fcn)
dview['data'] = alldata
dview.scatter('loopindices', self.loopindices)
dview['coords'] = self.coords
dview['nncutoff'] = self.nncutoff
dview['n_output_cols'] = len(outcols)
dview.execute(
'''results = worker(fcn, data, loopindices, coords, nncutoff, n_output_cols)'''
)
allresults[self.loopindices] = dview.gather('results', block=True)
rtr = self.frametracks.copy()
for i, name in enumerate(outcols):
rtr[name] = allresults[:, i]
return rtr
def map(self, fcn, incols, outcols, dview=None):
"""Evaluates a function for each particle, based on that particle and its
neighbors.
:param fcn: function to be called with ``data[i]``, ``data[nn_i]`` as arguments,
where ``data[nn_i]`` is a 2D array
:param incols: list of column names in the original DataFrame, corresponding
to the columns passed to ``fcn``.
:param outcols: list of new column names, in which the value(s) returned by ``fcn``
will be inserted.
:param dview: optional IPython parallel direct view.
Returns a copy of the original tracks DataFrame, with new columns from ``outcols``.
"""
# Design notes:
# - Things go the fastest and work the best when we send numpy arrays to the engines.
# In other words, avoid pickling at all costs.
# - Pandas objects are slow. The algorithm should work with plain numpy arrays;
# we convert back to DataFrames at the end.
alldata = self.frametracks[incols].values
allresults = np.ones((alldata.shape[0], len(outcols))) * np.nan
def worker(fcn, data, loopindices, coords, nncutoff, n_output_cols):
# This runs only once on each engine so it's ok to have all this setup code
import numpy as np
import scipy.spatial.ckdtree
tree = scipy.spatial.ckdtree.cKDTree(coords, 5)
results = np.ones((len(loopindices), n_output_cols)) * np.nan
neighborlist = tree.query_ball_point(coords[loopindices], nncutoff)
for i, (pindex, neighbors) in enumerate(zip(loopindices, neighborlist)):
neighbors.remove(pindex)
results[i] = fcn(data[pindex], data[neighbors])
return results
if dview is None:
allresults[self.loopindices] = worker(fcn, alldata, self.loopindices, self.coords, self.nncutoff, len(outcols))
else:
from IPython.parallel.util import interactive
dview.execute('''import numpy as np''')
# To send function to engines, its parent namespace must be the global namespace.
dview['worker'] = interactive(worker)
dview['fcn'] = interactive(fcn)
dview['data'] = alldata
dview.scatter('loopindices', self.loopindices)
dview['coords'] = self.coords
dview['nncutoff'] = self.nncutoff
dview['n_output_cols'] = len(outcols)
dview.execute('''results = worker(fcn, data, loopindices, coords, nncutoff, n_output_cols)''')
allresults[self.loopindices] = dview.gather('results', block=True)
rtr = self.frametracks.copy()
for i, name in enumerate(outcols):
rtr[name] = allresults[:,i]
return rtr
def compute_parallel(cache,
func,
keys,
save_every=4,
func_args=None,
func_kwargs=None,
parallel=True,
client=None):
"""Do a parallel computation of a function"""
keys = [key for key in keys]
results = dict(cache.items())
print(50 * '=')
print("Starting parallel run of {0} results".format(len(keys)))
print(" - parallel={0}".format(parallel))
if results:
print(" - found {0} previous results in {1}"
"".format(len(results), cache.filename))
keys_to_compute = [key for key in keys if key not in results]
# default arguments
def iter_function(key,
func=func,
func_args=func_args,
func_kwargs=func_kwargs):
func_args = func_args or ()
func_kwargs = func_kwargs or {}
return func(key, *func_args, **func_kwargs)
print(" - computing {0} results".format(len(keys_to_compute)))
# Set up the iterator over results
if parallel:
# Use interactive to prevent namespace issues
from IPython.parallel.util import interactive
iter_function = interactive(iter_function)
if client is None:
from IPython.parallel import Client
client = Client()
lbv = client.load_balanced_view()
results_iter = lbv.imap(iter_function, keys_to_compute, ordered=False)
else:
results_iter = imap(iter_function, keys_to_compute)
# Do the iteration, saving the results occasionally
print(datetime.now())
for i, (key, result) in enumerate(results_iter):
print('{0}/{1}: {2}'.format(i + 1, len(keys_to_compute), result))
print(' {0}'.format(datetime.now()))
cache.add_row(key, result, save=((i + 1) % save_every == 0))
cache.save()
return np.array([cache.get_row(key) for key in keys])
def _affine_field(self, d2min_scale=1.0, dview=None):
# Highly-optimized affine field computation, using a direct line to FORTRAN (LAPACK).
# The prototype for this design is the map() method.
def worker(data, loopindices, coords, nncutoff):
# This runs only once on each engine so it's ok to have all this setup code
tree = scipy.spatial.cKDTree(coords, 5) # 5 levels in the tree. YMMV.
# Laughing in the face of danger, we use the Fortran linear system solver directly.
solver, = scipy.linalg.lapack.get_lapack_funcs(('gelss',), (data, data))
results = np.ones((len(loopindices), 5)) * np.nan # 5 output columns
neighborlist = tree.query_ball_point(coords[loopindices], nncutoff)
for i, (pindex, neighbors) in enumerate(zip(loopindices, neighborlist)):
neighbors.remove(pindex)
if len(neighbors) < 2: continue # Minimum to satisfy DOF
r = data[neighbors] - np.tile(data[pindex], (len(neighbors), 1))
# The rest of the loop body is computation-specific.
try:
solvret = solver(r[:,0:2], r[:,2:4]) # v, x, s, rank, work, info
assert solvret[5] == 0 # "info"
except:
continue # Did not converge or not enough data; results will be NaN
results[i,:4] = solvret[1][:2].flat
# NORMALIZE by number of particles
results[i,4] = (solvret[1][2:]**2).sum() / len(neighbors)
# Uncommenting the assert below checks that D2min is equal to the residual.
#assert np.allclose(results[i,4], np.sum((r[:,0:2].dot(gelret[1][:2]) \
#- r[:,2:4])**2) / len(neighbors))
return results
alldata = self.frametracks[['x', 'y', 'x0', 'y0']].values
allresults = np.ones((alldata.shape[0], 5)) * np.nan
if dview is None:
# Single-threaded
allresults[self.loopindices] = worker(alldata, self.loopindices, self.coords, self.nncutoff)
else: # IPython parallel computing
from IPython.parallel.util import interactive
dview.execute('''import scipy.linalg.lapack, scipy.spatial''')
dview.execute('''import numpy as np''')
# To send function to engines, its parent namespace must be the global namespace.
dview['worker'] = interactive(worker)
dview['data'] = alldata
# Each engine considers a different set of particles
dview.scatter('loopindices', self.loopindices)
dview['coords'] = self.coords
dview['nncutoff'] = self.nncutoff
dview.execute('''results = worker(data, loopindices, coords, nncutoff)''')
allresults[self.loopindices] = dview.gather('results', block=True)
rtr = self.frametracks.copy()
for i, name in enumerate(['xdil', 'vstrain', 'hstrain', 'ydil', 'd2min']):
rtr[name] = allresults[:,i]
# FURTHER NORMALIZE by interparticle distance, if provided.
rtr['d2min'] = rtr.d2min / d2min_scale**2
return rtr
def _affine_field(self, d2min_scale=1.0, dview=None):
# Highly-optimized affine field computation, using a direct line to FORTRAN (LAPACK).
# The prototype for this design is the map() method.
def worker(data, loopindices, coords, nncutoff):
# This runs only once on each engine so it's ok to have all this setup code
tree = scipy.spatial.cKDTree(coords, 5) # 5 levels in the tree. YMMV.
# Laughing in the face of danger, we use the Fortran linear system solver directly.
solver, = scipy.linalg.lapack.get_lapack_funcs(('gelss',), (data, data))
results = np.ones((len(loopindices), 5)) * np.nan # 5 output columns
neighborlist = tree.query_ball_point(coords[loopindices], nncutoff)
for i, (pindex, neighbors) in enumerate(zip(loopindices, neighborlist)):
neighbors.remove(pindex)
if len(neighbors) < 2: continue # Minimum to satisfy DOF
r = data[neighbors] - np.tile(data[pindex], (len(neighbors), 1))
# The rest of the loop body is computation-specific.
try:
solvret = solver(r[:,0:2], r[:,2:4]) # v, x, s, rank, work, info
assert solvret[5] == 0 # "info"
except:
continue # Did not converge or not enough data; results will be NaN
results[i,:4] = solvret[1][:2].flat
# NORMALIZE by number of particles
results[i,4] = (solvret[1][2:]**2).sum() / len(neighbors)
# Uncommenting the assert below checks that D2min is equal to the residual.
#assert np.allclose(results[i,4], np.sum((r[:,0:2].dot(gelret[1][:2]) \
#- r[:,2:4])**2) / len(neighbors))
return results
alldata = self.frametracks[['x', 'y', 'x0', 'y0']].values
allresults = np.ones((alldata.shape[0], 5)) * np.nan
if dview is None:
# Single-threaded
allresults[self.loopindices] = worker(alldata, self.loopindices, self.coords, self.nncutoff)
else: # IPython parallel computing
from IPython.parallel.util import interactive
dview.execute('''import scipy.linalg.lapack, scipy.spatial''')
dview.execute('''import numpy as np''')
# To send function to engines, its parent namespace must be the global namespace.
dview['worker'] = interactive(worker)
dview['data'] = alldata
# Each engine considers a different set of particles
dview.scatter('loopindices', self.loopindices)
dview['coords'] = self.coords
dview['nncutoff'] = self.nncutoff
dview.execute('''results = worker(data, loopindices, coords, nncutoff)''')
allresults[self.loopindices] = dview.gather('results', block=True)
rtr = self.frametracks.copy()
for i, name in enumerate(['xdil', 'vstrain', 'hstrain', 'ydil', 'd2min']):
rtr[name] = allresults[:,i]
# FURTHER NORMALIZE by interparticle distance, if provided.
rtr['d2min'] = rtr.d2min / d2min_scale**2
return rtr
def compute_parallel(cache, func, keys, save_every=4,
func_args=None, func_kwargs=None,
parallel=True, client=None):
"""Do a parallel computation of a function"""
keys = [key for key in keys]
results = dict(cache.items())
print(50 * '=')
print("Starting parallel run of {0} results".format(len(keys)))
print(" - parallel={0}".format(parallel))
if results:
print(" - found {0} previous results in {1}"
"".format(len(results), cache.filename))
keys_to_compute = [key for key in keys if key not in results]
# default arguments
def iter_function(key, func=func, func_args=func_args,
func_kwargs=func_kwargs):
func_args = func_args or ()
func_kwargs = func_kwargs or {}
return func(key, *func_args, **func_kwargs)
print(" - computing {0} results".format(len(keys_to_compute)))
# Set up the iterator over results
if parallel:
# Use interactive to prevent namespace issues
from IPython.parallel.util import interactive
iter_function = interactive(iter_function)
if client is None:
from IPython.parallel import Client
client = Client()
lbv = client.load_balanced_view()
results_iter = lbv.imap(iter_function, keys_to_compute,
ordered=False)
else:
results_iter = imap(iter_function, keys_to_compute)
# Do the iteration, saving the results occasionally
print(datetime.now())
for i, (key, result) in enumerate(results_iter):
print('{0}/{1}: {2}'.format(i + 1, len(keys_to_compute), result))
print(' {0}'.format(datetime.now()))
cache.add_row(key, result, save=((i + 1) % save_every == 0))
cache.save()
return np.array([cache.get_row(key) for key in keys])
def line_pmap(self, function_name, args, kernel_name=None):
"""
%pmap FUNCTION [ARGS1,ARGS2,...] - ("parallel map") call a FUNCTION on args
This line magic will apply a function name to all of the
arguments given one at a time using a dynamic load balancing scheduler.
Currently, the args are provided as a Python expression (with no spaces).
You must first setup a cluster using the %parallel magic.
Examples:
%pmap function-name-in-language range(10)
%pmap function-name-in-language [1,2,3,4]
%pmap run_experiment range(1,100,5)
%pmap run_experiment ["test1","test2","test3"]
%pmap f [(1,4,7),(2,3,5),(7,2,2)]
The function name must be a function that is available on all
nodes in the cluster. For example, you could:
%%px
(define myfunc
(lambda (n)
(+ n 1)))
to define myfunc on all machines (use %%px -e to also define
it in the running notebook or console). Then you can apply it
to a list of arguments:
%%pmap myfunc range(100)
The load balancer will run myfunc on the next available node
in the cluster.
Note: not all languages may support running a function via this magic.
"""
if kernel_name is None:
kernel_name = self.kernel_name
# To make sure we can find `kernels`:
from IPython.parallel.util import interactive
f = interactive(lambda arg, kname=kernel_name, fname=function_name: \
kernels[kname].do_function_direct(fname, arg))
self.retval = self.view_load_balanced.map_async(f, eval(args))
def line_pmap(self, function_name, args, kernel_name=None):
"""
%pmap FUNCTION [ARGS1,ARGS2,...] - ("parallel map") call a FUNCTION on args
This line magic will apply a function name to all of the
arguments given one at a time using a dynamic load balancing scheduler.
Currently, the args are provided as a Python expression (with no spaces).
You must first setup a cluster using the %parallel magic.
Examples:
%pmap function-name-in-language range(10)
%pmap function-name-in-language [1,2,3,4]
%pmap run_experiment range(1,100,5)
%pmap run_experiment ["test1","test2","test3"]
%pmap f [(1,4,7),(2,3,5),(7,2,2)]
The function name must be a function that is available on all
nodes in the cluster. For example, you could:
%%px
(define myfunc
(lambda (n)
(+ n 1)))
to define myfunc on all machines (use %%px -e to also define
it in the running notebook or console). Then you can apply it
to a list of arguments:
%%pmap myfunc range(100)
The load balancer will run myfunc on the next available node
in the cluster.
Note: not all languages may support running a function via this magic.
"""
if kernel_name is None:
kernel_name = self.kernel_name
# To make sure we can find `kernels`:
from IPython.parallel.util import interactive
f = interactive(lambda arg, kname=kernel_name, fname=function_name: \
kernels[kname].do_function_direct(fname, arg))
self.retval = self.view_load_balanced.map_async(f, eval(args))
def test_push_pull_recarray(self):
"""push/pull recarrays"""
import numpy
from numpy.testing.utils import assert_array_equal
view = self.client[-1]
R = numpy.array(
[(1, "hi", 0.0), (2 ** 30, "there", 2.5), (-99999, "world", -12345.6789)],
[("n", int), ("s", "|S10"), ("f", float)],
)
view["RR"] = R
R2 = view["RR"]
r_dtype, r_shape = view.apply_sync(interactive(lambda: (RR.dtype, RR.shape)))
self.assertEqual(r_dtype, R.dtype)
self.assertEqual(r_shape, R.shape)
self.assertEqual(R2.dtype, R.dtype)
self.assertEqual(R2.shape, R.shape)
assert_array_equal(R2, R)
def test_push_pull_recarray(self):
"""push/pull recarrays"""
import numpy
from numpy.testing.utils import assert_array_equal
view = self.client[-1]
R = numpy.array([
(1, 'hi', 0.),
(2**30, 'there', 2.5),
(-99999, 'world', -12345.6789),
], [('n', int), ('s', '|S10'), ('f', float)])
view['RR'] = R
R2 = view['RR']
r_dtype, r_shape = view.apply_sync(interactive(lambda : (RR.dtype, RR.shape)))
self.assertEqual(r_dtype, R.dtype)
self.assertEqual(r_shape, R.shape)
self.assertEqual(R2.dtype, R.dtype)
self.assertEqual(R2.shape, R.shape)
assert_array_equal(R2, R)
def test_push_pull_recarray(self):
"""push/pull recarrays"""
import numpy
from numpy.testing.utils import assert_array_equal
view = self.client[-1]
R = numpy.array([
(1, 'hi', 0.),
(2**30, 'there', 2.5),
(-99999, 'world', -12345.6789),
], [('n', int), ('s', '|S10'), ('f', float)])
view['RR'] = R
R2 = view['RR']
r_dtype, r_shape = view.apply_sync(interactive(lambda : (RR.dtype, RR.shape)))
self.assertEqual(r_dtype, R.dtype)
self.assertEqual(r_shape, R.shape)
self.assertEqual(R2.dtype, R.dtype)
self.assertEqual(R2.shape, R.shape)
assert_array_equal(R2, R)