def __init__(self): from ipyparallel import Client rc = Client() rc.block=True self.cpu = rc[:] print '{} cores ready'.format(len(self.cpu)) self.cpu.execute('import numpy as np') self.cpu.execute('from sklearn.neighbors import KDTree, BallTree')
def main(): parser = OptionParser() parser.set_defaults(n=100) parser.set_defaults(tmin=1e-3) parser.set_defaults(tmax=1) parser.set_defaults(profile='default') parser.add_option("-n", type='int', dest='n', help='the number of tasks to run') parser.add_option("-t", type='float', dest='tmin', help='the minimum task length in seconds') parser.add_option("-T", type='float', dest='tmax', help='the maximum task length in seconds') parser.add_option("-p", '--profile', type='str', dest='profile', help="the cluster profile [default: 'default']") (opts, args) = parser.parse_args() assert opts.tmax >= opts.tmin, "tmax must not be smaller than tmin" rc = Client() view = rc.load_balanced_view() print(view) rc.block=True nengines = len(rc.ids) with rc[:].sync_imports(): from IPython.utils.timing import time # the jobs should take a random time within a range times = [random.random()*(opts.tmax-opts.tmin)+opts.tmin for i in range(opts.n)] stime = sum(times) print("executing %i tasks, totalling %.1f secs on %i engines"%(opts.n, stime, nengines)) time.sleep(1) start = time.time() amr = view.map(time.sleep, times) amr.get() stop = time.time() ptime = stop-start scale = stime/ptime print("executed %.1f secs in %.1f secs"%(stime, ptime)) print("%.3fx parallel performance on %i engines"%(scale, nengines)) print("%.1f%% of theoretical max"%(100*scale/nengines))
of megabytes you might saturate the network interface of a single node and potentially its memory buffers if the messages are not consumed in a streamed manner. Note that the AllReduce scheme implemented with the spanning tree strategy impose the aggregation function to be commutative and distributive. It might not be the case if you implement the naive gather / reduce / broadcast strategy where you can reorder the partial data before performing the reduce. """ from __future__ import print_function from ipyparallel import Client, Reference # connect client and create views rc = Client() rc.block = True ids = rc.ids root_id = ids[0] root = rc[root_id] view = rc[:] # run bintree.py script defining bintree functions, etc. exec(compile(open('bintree.py').read(), 'bintree.py', 'exec')) # generate binary tree of parents btree = bintree(ids) print("setting up binary tree interconnect:") print_bintree(btree)
potentially its memory buffers if the messages are not consumed in a streamed manner. Note that the AllReduce scheme implemented with the spanning tree strategy impose the aggregation function to be commutative and distributive. It might not be the case if you implement the naive gather / reduce / broadcast strategy where you can reorder the partial data before performing the reduce. """ from __future__ import print_function from ipyparallel import Client, Reference # connect client and create views rc = Client() rc.block = True ids = rc.ids root_id = ids[0] root = rc[root_id] view = rc[:] # run bintree.py script defining bintree functions, etc. exec(compile(open("bintree.py").read(), "bintree.py", "exec")) # generate binary tree of parents btree = bintree(ids) print("setting up binary tree interconnect:") print_bintree(btree)
def __init__(self): from ipyparallel import Client rc = Client() rc.block=True self.cpu = rc[:]