def test_failures(): """ Tests whether scheduling jobs works on known failure cases. """ from pylada.jobfolder.jobfolder import JobFolder from pylada.process.pool import PoolProcess from pylada import default_comm from functional import Functional root = JobFolder() for n in xrange(8): job = root / "a{0}".format(n) job.functional = Functional("whatever", [n]) job.params['sleep'] = 1 comm = default_comm.copy() comm['n'] = 4 d = {'a1': 1, 'a0': 3, 'a3': 3, 'a2': 3, 'a5': 3, 'a4': 2, 'a7': 2, 'a6': 1} def processalloc_test1(job): return d[job.name[1:-1]] program = PoolProcess(root, processalloc=processalloc_test1, outdir="whatever") program._comm = comm for i in xrange(10000): jobs = program._getjobs() assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\ (jobs, [program._alloc[u] for u in jobs])
def test_getjobs(nprocs=8, njobs=20): """ Test scheduling. """ from pylada.jobfolder.jobfolder import JobFolder from pylada.process.pool import PoolProcess from pylada import default_comm from functional import Functional root = JobFolder() for n in xrange(njobs): job = root / "a{0}".format(n) job.functional = Functional("whatever", [n]) job.params['sleep'] = 1 comm = default_comm.copy() comm['n'] = nprocs def processalloc(job): """ returns a random number between 1 and 4 included. """ from random import randint return randint(1, comm['n']) for j in xrange(100): program = PoolProcess(root, processalloc=processalloc, outdir="whatever") program._comm = comm for i in xrange(1000): jobs = program._getjobs() assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\ (jobs, [program._alloc[u] for u in jobs])
def test_failures(d): """ Tests whether scheduling jobs works on known failure cases. """ from pylada.jobfolder.jobfolder import JobFolder from pylada.process.pool import PoolProcess from pylada import default_comm from functional import Functional root = JobFolder() for n in xrange(8): job = root / "a{0}".format(n) job.functional = Functional("whatever", [n]) job.params['sleep'] = 1 comm = default_comm.copy() comm['n'] = 4 def processalloc_test1(job): return d[job.name[1:-1]] program = PoolProcess(root, processalloc=processalloc_test1, outdir="whatever") program._comm = comm for i in xrange(10000): jobs = program._getjobs() assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\ (jobs, [program._alloc[u] for u in jobs])
def test_update_with_fail(): """ Tests JobFolderProcess with update. """ from tempfile import mkdtemp from os.path import join, abspath, dirname from shutil import rmtree from pylada.jobfolder.jobfolder import JobFolder from pylada.jobfolder import save from pylada.process.pool import PoolProcess from pylada.process import Fail from pylada import default_comm from pylada.process.tests.functional import Functional from pylada.process.tests.pifunctional import __file__ as executable executable = join(dirname(executable), "pifunctional.py") root = JobFolder() for n in xrange(3): job = root / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 root['1'].functional.order = 50 root['1'].functional.fail = 'midway' root['1'].sleep = None supp = JobFolder() for n in xrange(3, 6): job = supp / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 supp['5'].sleep = 0 supp['5'].functional.order = 34 supp['5'].functional.fail = 'end' comm = default_comm.copy() comm['n'] = 4 dir = mkdtemp() save(root, join(dir, 'dict.dict'), overwrite=True) try: program = PoolProcess(root, processalloc=processalloc, outdir=dir, keepalive=True) assert program.nbjobsleft > 0 # compute current jobs. program.start(comm) try: program.wait() except Fail: pass else: print program.errors raise Exception() assert hasattr(program, '_comm') # compute second set of updated jobs program.update(supp) try: program.wait() except Fail: pass else: raise Exception() finally: try: rmtree(dir) except: pass
def test_large(): """ Test speed of job-scheduling for largers, more numerous jobs. """ from random import random from pylada.jobfolder.jobfolder import JobFolder from pylada.process.pool import PoolProcess from pylada.process.dummy import DummyFunctional from pylada import default_comm root = JobFolder() for n in xrange(100): job = root / "a{0}".format(n) job.functional = DummyFunctional(chance=random()*0.5+0.15) comm = default_comm.copy() comm['n'] = 256 def processalloc_test1(job): """ returns a random number between 1 and 4 included. """ from random import randint return randint(1, 64) program = PoolProcess(root, processalloc=processalloc_test1, outdir="whatever") program.start(comm) while not program.poll(): # print 256 - program._comm['n'], len(program.process) continue def processalloc_test1(job): """ returns a random number between 1 and 4 included. """ from random import choice return choice([31, 37, 43]) program = PoolProcess(root, processalloc=processalloc_test1, outdir="whatever") program.start(comm) while not program.poll(): # print 256 - program._comm['n'], len(program.process) continue
def test_getjobs(comm, tmpdir, executable, nprocs, njobs): """ Test scheduling. """ root = jobfolders(executable, 0, 8) def processalloc(job): """ returns a random number between 1 and 4 included. """ from random import randint return randint(1, comm['n']) for j in range(100): program = PoolProcess(root, processalloc=processalloc, outdir=str(tmpdir)) program._comm = comm for i in range(1000): jobs = program._getjobs() assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\ (jobs, [program._alloc[u] for u in jobs])
def test_update(executable): """ Tests JobFolderProcess with update. """ from tempfile import mkdtemp from os.path import join from shutil import rmtree from pylada.jobfolder.jobfolder import JobFolder from pylada.jobfolder import save from pylada.process.pool import PoolProcess from pylada import default_comm from functional import Functional root = JobFolder() for n in xrange(3): job = root / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 supp = JobFolder() for n in xrange(3, 6): job = supp / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 comm = default_comm.copy() comm['n'] = 4 dir = mkdtemp() save(root, join(dir, 'dict.dict'), overwrite=True) try: program = PoolProcess(root, processalloc=processalloc, outdir=dir, keepalive=True) assert program.nbjobsleft > 0 # compute current jobs. program.start(comm) program.wait() assert hasattr(program, '_comm') # compute second set of updated jobs program.update(supp) program.wait() finally: try: rmtree(dir) except: pass
def test_failures(tmpdir, executable, comm): """Tests whether scheduling jobs works on known failure cases.""" from pylada import default_comm root = jobfolders(executable, 0, 8) def processalloc_test1(job): d = {'1': 1, '0': 3, '3': 3, '2': 3, '5': 3, '4': 2, '7': 2, '6': 1} return d[job.name[1:-1]] program = PoolProcess(root, processalloc=processalloc_test1, outdir=str(tmpdir)) program._comm = comm for i in range(10000): jobs = program._getjobs() assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\ (jobs, [program._alloc[u] for u in jobs])
def test_failures(tmpdir, executable, comm): """ Tests whether scheduling jobs works on known failure cases. """ from pylada import default_comm from pylada.process.tests.functional import Functional root = jobfolders(executable, 0, 8) def processalloc_test1(job): d = {'1': 1, '0': 3, '3': 3, '2': 3, '5': 3, '4': 2, '7': 2, '6': 1} return d[job.name[1:-1]] program = PoolProcess(root, processalloc=processalloc_test1, outdir=str(tmpdir)) program._comm = comm for i in range(10000): jobs = program._getjobs() assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\ (jobs, [program._alloc[u] for u in jobs])
def test_getjobs(comm, tmpdir, executable, nprocs, njobs): """Test scheduling.""" root = jobfolders(executable, 0, 8) def processalloc(job): """returns a random number between 1 and 4 included.""" from random import randint return randint(1, comm['n']) for j in range(100): program = PoolProcess(root, processalloc=processalloc, outdir=str(tmpdir)) program._comm = comm for i in range(1000): jobs = program._getjobs() assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\ (jobs, [program._alloc[u] for u in jobs])
def test_large(): """ Test speed of job-scheduling for largers, more numerous jobs. """ from random import random from pylada.jobfolder.jobfolder import JobFolder from pylada.process.pool import PoolProcess from pylada.process.dummy import DummyFunctional from pylada import default_comm root = JobFolder() for n in xrange(100): job = root / "a{0}".format(n) job.functional = DummyFunctional(chance=random() * 0.5 + 0.15) comm = default_comm.copy() comm['n'] = 256 def processalloc_test1(job): """ returns a random number between 1 and 4 included. """ from random import randint return randint(1, 64) program = PoolProcess(root, processalloc=processalloc_test1, outdir="whatever") program.start(comm) while not program.poll(): # print 256 - program._comm['n'], len(program.process) continue def processalloc_test1(job): """ returns a random number between 1 and 4 included. """ from random import choice return choice([31, 37, 43]) program = PoolProcess(root, processalloc=processalloc_test1, outdir="whatever") program.start(comm) while not program.poll(): # print 256 - program._comm['n'], len(program.process) continue
def test_fakeexec(): """ Tests Pool. Includes failure modes. """ from tempfile import mkdtemp from os.path import join, abspath, dirname from shutil import rmtree from numpy import all, arange, abs, array from pylada.jobfolder.jobfolder import JobFolder from pylada.jobfolder.massextract import MassExtract from pylada.jobfolder import save from pylada.process.pool import PoolProcess from pylada.process import Fail, NotStarted from pylada import default_comm from pylada.process.tests.functional import Functional from pylada.process.tests.pifunctional import __file__ as executable executable = join(dirname(executable), "pifunctional.py") root = JobFolder() for n in xrange(8): job = root / "a{0}".format(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 comm = default_comm.copy() comm['n'] = 4 dir = mkdtemp() save(root, join(dir, 'dict.dict'), overwrite=True) try: program = PoolProcess(root, processalloc=processalloc, outdir=dir) assert program.nbjobsleft > 0 # program not started. should fail. try: program.poll() except NotStarted: pass else: raise Exception() try: program.wait() except NotStarted: pass else: raise Exception() # now starting for real. program.start(comm) program.wait() assert program.nbjobsleft == 0 extract = MassExtract(join(dir, 'dict.dict')) assert all(extract.success.itervalues()) order = array(extract.order.values()).flatten() assert all(arange(8) - order == 0) pi = array(extract.pi.values()).flatten() expected = [0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907, 3.143293] assert all(abs(pi - array(expected)) < 1e-5) error = array(extract.error.values()).flatten() expected = [3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865, 0.00333321, 0.002314774, 0.001700664] assert all(abs(error - array(expected)) < 1e-5) assert all(n['n'] == comm['n'] for n in extract.comm) # restart assert program.poll() assert len(program.process) == 0 program.start(comm) assert len(program.process) == 0 assert program.poll() try: job = root / str(666) job.functional = Functional(executable, [50], fail='end') program = PoolProcess(root, nbpools=2, outdir=dir, processalloc=processalloc) assert program.nbjobsleft > 0 program.start(comm) program.wait() assert False # Should not reach this point except Fail: assert len(program.errors.keys()) == 1 assert '666' in program.errors else: raise Exception() job.functional.order = [50] program = PoolProcess(root, nbpools=2, outdir=dir, processalloc=processalloc) assert program.nbjobsleft > 0 program.start(comm) program.wait() assert program.nbjobsleft == 0 finally: try: rmtree(dir) except: pass
def test(executable): """ Tests JobFolderProcess. Includes failure modes. """ from tempfile import mkdtemp from os.path import join from shutil import rmtree from numpy import all, arange, abs, array from pylada.jobfolder.jobfolder import JobFolder from pylada.jobfolder.massextract import MassExtract from pylada.jobfolder import save from pylada.process.pool import PoolProcess from pylada.process import Fail, NotStarted from pylada import default_comm from functional import Functional root = JobFolder() for n in xrange(8): job = root / "a{0}".format(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 comm = default_comm.copy() comm['n'] = 4 dir = mkdtemp() save(root, join(dir, 'dict.dict'), overwrite=True) try: program = PoolProcess(root, processalloc=processalloc, outdir=dir) assert program.nbjobsleft > 0 # program not started. should fail. try: program.poll() except NotStarted: pass else: raise Exception() try: program.wait() except NotStarted: pass else: raise Exception() # now starting for real. program.start(comm) program.wait() assert program.nbjobsleft == 0 extract = MassExtract(join(dir, 'dict.dict')) assert all(extract.success.itervalues()) order = array(extract.order.values()).flatten() assert all(arange(8) - order == 0) pi = array(extract.pi.values()).flatten() assert all(abs(pi - array([0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907, 3.143293]))\ < 1e-5 ) error = array(extract.error.values()).flatten() assert all(abs(error - array([3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865, 0.00333321, 0.002314774, 0.001700664]))\ < 1e-5 ) assert all(n['n'] == comm['n'] for n in extract.comm) # restart assert program.poll() assert len(program.process) == 0 program.start(comm) assert len(program.process) == 0 assert program.poll() finally: try: rmtree(dir) except: pass return try: job = root / str(666) job.functional = Functional(executable, [666]) program = PoolProcess(root, nbpools=2, outdir=dir, processalloc=processalloc) assert program.nbjobsleft > 0 program.start(comm) program.wait() assert program.nbjobsleft == 0 except Fail: assert len(program.errors.keys()) == 1 assert '666' in program.errors else: raise Exception finally: try: rmtree(dir) except: pass try: job.functional.order = [667] program = PoolProcess(root, nbpools=2, outdir=dir, processalloc=processalloc) assert program.nbjobsleft > 0 program.start(comm) program.wait() assert program.nbjobsleft == 0 finally: try: rmtree(dir) except: pass
def pool_process(tmpdir, root, pal=processalloc, **kwargs): from pylada.process.pool import PoolProcess return PoolProcess(root, processalloc=pal, outdir=str(tmpdir), **kwargs)