def map(self, fcn, incols, outcols, dview=None): """Evaluates a function for each particle, based on that particle and its neighbors. :param fcn: function to be called with ``data[i]``, ``data[nn_i]`` as arguments, where ``data[nn_i]`` is a 2D array :param incols: list of column names in the original DataFrame, corresponding to the columns passed to ``fcn``. :param outcols: list of new column names, in which the value(s) returned by ``fcn`` will be inserted. :param dview: optional IPython parallel direct view. Returns a copy of the original tracks DataFrame, with new columns from ``outcols``. """ # Design notes: # - Things go the fastest and work the best when we send numpy arrays to the engines. # In other words, avoid pickling at all costs. # - Pandas objects are slow. The algorithm should work with plain numpy arrays; # we convert back to DataFrames at the end. alldata = self.frametracks[incols].values allresults = np.ones((alldata.shape[0], len(outcols))) * np.nan def worker(fcn, data, loopindices, coords, nncutoff, n_output_cols): # This runs only once on each engine so it's ok to have all this setup code import numpy as np import scipy.spatial.ckdtree tree = scipy.spatial.ckdtree.cKDTree(coords, 5) results = np.ones((len(loopindices), n_output_cols)) * np.nan neighborlist = tree.query_ball_point(coords[loopindices], nncutoff) for i, (pindex, neighbors) in enumerate(zip(loopindices, neighborlist)): neighbors.remove(pindex) results[i] = fcn(data[pindex], data[neighbors]) return results if dview is None: allresults[self.loopindices] = worker(fcn, alldata, self.loopindices, self.coords, self.nncutoff, len(outcols)) else: from IPython.parallel.util import interactive dview.execute('''import numpy as np''') # To send function to engines, its parent namespace must be the global namespace. dview['worker'] = interactive(worker) dview['fcn'] = interactive(fcn) dview['data'] = alldata dview.scatter('loopindices', self.loopindices) dview['coords'] = self.coords dview['nncutoff'] = self.nncutoff dview['n_output_cols'] = len(outcols) dview.execute( '''results = worker(fcn, data, loopindices, coords, nncutoff, n_output_cols)''' ) allresults[self.loopindices] = dview.gather('results', block=True) rtr = self.frametracks.copy() for i, name in enumerate(outcols): rtr[name] = allresults[:, i] return rtr
def map(self, fcn, incols, outcols, dview=None): """Evaluates a function for each particle, based on that particle and its neighbors. :param fcn: function to be called with ``data[i]``, ``data[nn_i]`` as arguments, where ``data[nn_i]`` is a 2D array :param incols: list of column names in the original DataFrame, corresponding to the columns passed to ``fcn``. :param outcols: list of new column names, in which the value(s) returned by ``fcn`` will be inserted. :param dview: optional IPython parallel direct view. Returns a copy of the original tracks DataFrame, with new columns from ``outcols``. """ # Design notes: # - Things go the fastest and work the best when we send numpy arrays to the engines. # In other words, avoid pickling at all costs. # - Pandas objects are slow. The algorithm should work with plain numpy arrays; # we convert back to DataFrames at the end. alldata = self.frametracks[incols].values allresults = np.ones((alldata.shape[0], len(outcols))) * np.nan def worker(fcn, data, loopindices, coords, nncutoff, n_output_cols): # This runs only once on each engine so it's ok to have all this setup code import numpy as np import scipy.spatial.ckdtree tree = scipy.spatial.ckdtree.cKDTree(coords, 5) results = np.ones((len(loopindices), n_output_cols)) * np.nan neighborlist = tree.query_ball_point(coords[loopindices], nncutoff) for i, (pindex, neighbors) in enumerate(zip(loopindices, neighborlist)): neighbors.remove(pindex) results[i] = fcn(data[pindex], data[neighbors]) return results if dview is None: allresults[self.loopindices] = worker(fcn, alldata, self.loopindices, self.coords, self.nncutoff, len(outcols)) else: from IPython.parallel.util import interactive dview.execute('''import numpy as np''') # To send function to engines, its parent namespace must be the global namespace. dview['worker'] = interactive(worker) dview['fcn'] = interactive(fcn) dview['data'] = alldata dview.scatter('loopindices', self.loopindices) dview['coords'] = self.coords dview['nncutoff'] = self.nncutoff dview['n_output_cols'] = len(outcols) dview.execute('''results = worker(fcn, data, loopindices, coords, nncutoff, n_output_cols)''') allresults[self.loopindices] = dview.gather('results', block=True) rtr = self.frametracks.copy() for i, name in enumerate(outcols): rtr[name] = allresults[:,i] return rtr
def compute_parallel(cache, func, keys, save_every=4, func_args=None, func_kwargs=None, parallel=True, client=None): """Do a parallel computation of a function""" keys = [key for key in keys] results = dict(cache.items()) print(50 * '=') print("Starting parallel run of {0} results".format(len(keys))) print(" - parallel={0}".format(parallel)) if results: print(" - found {0} previous results in {1}" "".format(len(results), cache.filename)) keys_to_compute = [key for key in keys if key not in results] # default arguments def iter_function(key, func=func, func_args=func_args, func_kwargs=func_kwargs): func_args = func_args or () func_kwargs = func_kwargs or {} return func(key, *func_args, **func_kwargs) print(" - computing {0} results".format(len(keys_to_compute))) # Set up the iterator over results if parallel: # Use interactive to prevent namespace issues from IPython.parallel.util import interactive iter_function = interactive(iter_function) if client is None: from IPython.parallel import Client client = Client() lbv = client.load_balanced_view() results_iter = lbv.imap(iter_function, keys_to_compute, ordered=False) else: results_iter = imap(iter_function, keys_to_compute) # Do the iteration, saving the results occasionally print(datetime.now()) for i, (key, result) in enumerate(results_iter): print('{0}/{1}: {2}'.format(i + 1, len(keys_to_compute), result)) print(' {0}'.format(datetime.now())) cache.add_row(key, result, save=((i + 1) % save_every == 0)) cache.save() return np.array([cache.get_row(key) for key in keys])
def _affine_field(self, d2min_scale=1.0, dview=None): # Highly-optimized affine field computation, using a direct line to FORTRAN (LAPACK). # The prototype for this design is the map() method. def worker(data, loopindices, coords, nncutoff): # This runs only once on each engine so it's ok to have all this setup code tree = scipy.spatial.cKDTree(coords, 5) # 5 levels in the tree. YMMV. # Laughing in the face of danger, we use the Fortran linear system solver directly. solver, = scipy.linalg.lapack.get_lapack_funcs(('gelss',), (data, data)) results = np.ones((len(loopindices), 5)) * np.nan # 5 output columns neighborlist = tree.query_ball_point(coords[loopindices], nncutoff) for i, (pindex, neighbors) in enumerate(zip(loopindices, neighborlist)): neighbors.remove(pindex) if len(neighbors) < 2: continue # Minimum to satisfy DOF r = data[neighbors] - np.tile(data[pindex], (len(neighbors), 1)) # The rest of the loop body is computation-specific. try: solvret = solver(r[:,0:2], r[:,2:4]) # v, x, s, rank, work, info assert solvret[5] == 0 # "info" except: continue # Did not converge or not enough data; results will be NaN results[i,:4] = solvret[1][:2].flat # NORMALIZE by number of particles results[i,4] = (solvret[1][2:]**2).sum() / len(neighbors) # Uncommenting the assert below checks that D2min is equal to the residual. #assert np.allclose(results[i,4], np.sum((r[:,0:2].dot(gelret[1][:2]) \ #- r[:,2:4])**2) / len(neighbors)) return results alldata = self.frametracks[['x', 'y', 'x0', 'y0']].values allresults = np.ones((alldata.shape[0], 5)) * np.nan if dview is None: # Single-threaded allresults[self.loopindices] = worker(alldata, self.loopindices, self.coords, self.nncutoff) else: # IPython parallel computing from IPython.parallel.util import interactive dview.execute('''import scipy.linalg.lapack, scipy.spatial''') dview.execute('''import numpy as np''') # To send function to engines, its parent namespace must be the global namespace. dview['worker'] = interactive(worker) dview['data'] = alldata # Each engine considers a different set of particles dview.scatter('loopindices', self.loopindices) dview['coords'] = self.coords dview['nncutoff'] = self.nncutoff dview.execute('''results = worker(data, loopindices, coords, nncutoff)''') allresults[self.loopindices] = dview.gather('results', block=True) rtr = self.frametracks.copy() for i, name in enumerate(['xdil', 'vstrain', 'hstrain', 'ydil', 'd2min']): rtr[name] = allresults[:,i] # FURTHER NORMALIZE by interparticle distance, if provided. rtr['d2min'] = rtr.d2min / d2min_scale**2 return rtr
def line_pmap(self, function_name, args, kernel_name=None): """ %pmap FUNCTION [ARGS1,ARGS2,...] - ("parallel map") call a FUNCTION on args This line magic will apply a function name to all of the arguments given one at a time using a dynamic load balancing scheduler. Currently, the args are provided as a Python expression (with no spaces). You must first setup a cluster using the %parallel magic. Examples: %pmap function-name-in-language range(10) %pmap function-name-in-language [1,2,3,4] %pmap run_experiment range(1,100,5) %pmap run_experiment ["test1","test2","test3"] %pmap f [(1,4,7),(2,3,5),(7,2,2)] The function name must be a function that is available on all nodes in the cluster. For example, you could: %%px (define myfunc (lambda (n) (+ n 1))) to define myfunc on all machines (use %%px -e to also define it in the running notebook or console). Then you can apply it to a list of arguments: %%pmap myfunc range(100) The load balancer will run myfunc on the next available node in the cluster. Note: not all languages may support running a function via this magic. """ if kernel_name is None: kernel_name = self.kernel_name # To make sure we can find `kernels`: from IPython.parallel.util import interactive f = interactive(lambda arg, kname=kernel_name, fname=function_name: \ kernels[kname].do_function_direct(fname, arg)) self.retval = self.view_load_balanced.map_async(f, eval(args))
def test_push_pull_recarray(self): """push/pull recarrays""" import numpy from numpy.testing.utils import assert_array_equal view = self.client[-1] R = numpy.array( [(1, "hi", 0.0), (2 ** 30, "there", 2.5), (-99999, "world", -12345.6789)], [("n", int), ("s", "|S10"), ("f", float)], ) view["RR"] = R R2 = view["RR"] r_dtype, r_shape = view.apply_sync(interactive(lambda: (RR.dtype, RR.shape))) self.assertEqual(r_dtype, R.dtype) self.assertEqual(r_shape, R.shape) self.assertEqual(R2.dtype, R.dtype) self.assertEqual(R2.shape, R.shape) assert_array_equal(R2, R)
def test_push_pull_recarray(self): """push/pull recarrays""" import numpy from numpy.testing.utils import assert_array_equal view = self.client[-1] R = numpy.array([ (1, 'hi', 0.), (2**30, 'there', 2.5), (-99999, 'world', -12345.6789), ], [('n', int), ('s', '|S10'), ('f', float)]) view['RR'] = R R2 = view['RR'] r_dtype, r_shape = view.apply_sync(interactive(lambda : (RR.dtype, RR.shape))) self.assertEqual(r_dtype, R.dtype) self.assertEqual(r_shape, R.shape) self.assertEqual(R2.dtype, R.dtype) self.assertEqual(R2.shape, R.shape) assert_array_equal(R2, R)