def test2(obj): from pathos.pp import ParallelPythonPool p = ParallelPythonPool(2) x = [1, 2, 3] assert map(obj, x) == p.map(obj, x)
#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 1997-2014 California Institute of Technology. # License: 3-clause BSD. The full license text is available at: # - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/pathos/LICENSE from pathos.pp import stats from pathos.pp import ParallelPythonPool as Pool pool = Pool() def host(id): import socket import time time.sleep(1.0) return "Rank: %d -- %s" % (id, socket.gethostname()) print "Evaluate 10 items on 2 cpus" #FIXME: reset lport below pool.ncpus = 2 pool.servers = ('localhost:5653',) res5 = pool.map(host, range(10)) print pool print '\n'.join(res5) print stats() print '' # end of file
#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 1997-2014 California Institute of Technology. # License: 3-clause BSD. The full license text is available at: # - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/pathos/LICENSE from pathos.pp import stats from pathos.pp import ParallelPythonPool as Pool pool = Pool() def host(id): import socket import time time.sleep(1.0) return "Rank: %d -- %s" % (id, socket.gethostname()) print "Evaluate 10 items on 2 cpus" #FIXME: reset lport below pool.ncpus = 2 pool.servers = ('localhost:5653', ) res5 = pool.map(host, range(10)) print pool print '\n'.join(res5) print stats() print '' # end of file
basic = PS() print basic start = time.time() res = basic.map(busy_add, _x, _y, _d) print "time to queue:", time.time() - start start = time.time() _basic = list(res) print "time to results:", time.time() - start print "" from pathos.pp import ParallelPythonPool as PPP #from pathos.pp import stats pp_pool = PPP(4, servers=('localhost:5653','localhost:2414')) print pp_pool start = time.time() res = pp_pool.map(busy_add, _x, _y, _d) print "time to queue:", time.time() - start start = time.time() _pp_pool = list(res) print "time to results:", time.time() - start #print stats() assert _basic == _pp_pool print "" from pathos.multiprocessing import ProcessingPool as MPP mp_pool = MPP(4) print mp_pool start = time.time() res = mp_pool.map(busy_add, _x, _y, _d) print "time to queue:", time.time() - start
def test_ppmap(obj): from pathos.pp import ParallelPythonPool p = ParallelPythonPool(2) x = [1, 2, 3] assert map(obj, x) == p.map(obj, x)
if __name__ == '__main__': def add(x, y, z): """Add three values""" return x + y + z def busybeaver(x): """This can take a while""" for num in range(1000000): x = x + num return x # Immediate evaluation example import time start = time.time() results = pool.map(busybeaver, range(10)) print 'Time to queue the jobs:', time.time() - start start = time.time() # Casting the ppmap generator to a list forces each result to be # evaluated. When done immediately after the jobs are submitted, # our program twiddles its thumbs while the work is finished. print list(results) print 'Time to get the results:', time.time() - start # Delayed evaluation example start = time.time() results = pool.imap(busybeaver, range(10)) print 'Time to queue the jobs:', time.time() - start # In contrast with the above example, this time we're submitting a # batch of jobs then going off to do more work while they're # processing. Maybe "time.sleep" isn't the most exciting example,