def test_push_pull_recarray(self): """push/pull recarrays""" import numpy from numpy.testing import assert_array_equal view = self.client[-1] R = numpy.array( [ (1, 'hi', 0.0), (2**30, 'there', 2.5), (-99999, 'world', -12345.6789), ], [('n', int), ('s', '|S10'), ('f', float)], ) view['RR'] = R R2 = view['RR'] r_dtype, r_shape = view.apply_sync( interactive(lambda: (RR.dtype, RR.shape)) # noqa: F821 ) self.assertEqual(r_dtype, R.dtype) self.assertEqual(r_shape, R.shape) self.assertEqual(R2.dtype, R.dtype) self.assertEqual(R2.shape, R.shape) assert_array_equal(R2, R)
def line_pmap(self, function_name, args, kernel_name=None, set_variable=None): """ %pmap FUNCTION [ARGS1,ARGS2,...] - ("parallel map") call a FUNCTION on args This line magic will apply a function name to all of the arguments given one at a time using a dynamic load balancing scheduler. Currently, the args are provided as a Python expression (with no spaces). You must first setup a cluster using the %parallel magic. Examples: %pmap function-name-in-language range(10) %pmap function-name-in-language [1,2,3,4] %pmap run_experiment range(1,100,5) %pmap run_experiment ["test1","test2","test3"] %pmap f [(1,4,7),(2,3,5),(7,2,2)] The function name must be a function that is available on all nodes in the cluster. For example, you could: %%px (define myfunc (lambda (n) (+ n 1))) to define myfunc on all machines (use %%px -e to also define it in the running notebook or console). Then you can apply it to a list of arguments: %%pmap myfunc range(100) The load balancer will run myfunc on the next available node in the cluster. Note: not all languages may support running a function via this magic. """ if kernel_name is None: kernel_name = self.kernel_name # To make sure we can find `kernels`: try: from ipyparallel.util import interactive except ImportError: from IPython.parallel.util import interactive f = interactive(lambda arg, kname=kernel_name, fname=function_name: \ kernels[kname].do_function_direct(fname, arg)) results = self.view_load_balanced.map_async(f, eval(args)) if set_variable is None: self.retval = results else: self.kernel.set_variable(set_variable, results) self.retval = None
def test_push_pull_recarray(self): """push/pull recarrays""" import numpy from numpy.testing.utils import assert_array_equal view = self.client[-1] R = numpy.array( [(1, "hi", 0.0), (2 ** 30, "there", 2.5), (-99999, "world", -12345.6789)], [("n", int), ("s", "|S10"), ("f", float)], ) view["RR"] = R R2 = view["RR"] r_dtype, r_shape = view.apply_sync(interactive(lambda: (RR.dtype, RR.shape))) self.assertEqual(r_dtype, R.dtype) self.assertEqual(r_shape, R.shape) self.assertEqual(R2.dtype, R.dtype) self.assertEqual(R2.shape, R.shape) assert_array_equal(R2, R)
def test_push_pull_recarray(self): """push/pull recarrays""" import numpy from numpy.testing.utils import assert_array_equal view = self.client[-1] R = numpy.array([ (1, 'hi', 0.), (2**30, 'there', 2.5), (-99999, 'world', -12345.6789), ], [('n', int), ('s', '|S10'), ('f', float)]) view['RR'] = R R2 = view['RR'] r_dtype, r_shape = view.apply_sync(interactive(lambda : (RR.dtype, RR.shape))) self.assertEqual(r_dtype, R.dtype) self.assertEqual(r_shape, R.shape) self.assertEqual(R2.dtype, R.dtype) self.assertEqual(R2.shape, R.shape) assert_array_equal(R2, R)