def test_completes(program, comm, tmpdir, root, do_multiple_mpi_programs): from numpy import all, arange, abs, array from pylada.jobfolder.massextract import MassExtract from pylada.jobfolder import save program.start(comm) program.wait() assert program.nbjobsleft == 0 save(root, str(tmpdir.join('dict.dict')), overwrite=True) extract = MassExtract(str(tmpdir.join('dict.dict'))) assert all(extract.success.values()) order = array(list(extract.order.values())).flatten() assert all(arange(8) - order == 0) pi = array(list(extract.pi.values())).flatten() expected = [ 0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907, 3.143293 ] assert all(abs(pi - array(expected)) < 1e-5) error = array(list(extract.error.values())).flatten() expected = [ 3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865, 0.00333321, 0.002314774, 0.001700664 ] assert all(abs(error - array(expected)) < 1e-5) assert all(n['n'] == comm['n'] for n in extract.comm)
def test_update(tmpdir, executable, comm, Process, do_multiple_mpi_programs): from copy import deepcopy from pylada.jobfolder.massextract import MassExtract from pylada.jobfolder import save # creates and start computing first set of jobs root = jobfolders(executable, 0, 3) program = Process(tmpdir, deepcopy(root), keepalive=True) assert program.keepalive program.start(comm) assert hasattr(program, '_comm') # update jobfolder and start computing second set of jobs supp = jobfolders(executable, 3, 6) program.update(supp) for key in supp.keys(): assert key in program.jobfolder for key in root.keys(): assert key in program.jobfolder assert len(program.jobfolder) == len(root) + len(supp) # create mass extraction object and check succcess save(program.jobfolder, str(tmpdir.join('dict.dict')), overwrite=True) extract = MassExtract(str(tmpdir.join('dict.dict'))) # wait for job completion and check for success program.wait() assert len(extract.success.values()) == 6 assert all((u for u in extract.success.values()))
def test_update_with_fail(): """ Tests JobFolderProcess with update. """ from tempfile import mkdtemp from os.path import join, abspath, dirname from shutil import rmtree from pylada.jobfolder.jobfolder import JobFolder from pylada.jobfolder import save from pylada.process.pool import PoolProcess from pylada.process import Fail from pylada import default_comm from pylada.process.tests.functional import Functional from pylada.process.tests.pifunctional import __file__ as executable executable = join(dirname(executable), "pifunctional.py") root = JobFolder() for n in xrange(3): job = root / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 root['1'].functional.order = 50 root['1'].functional.fail = 'midway' root['1'].sleep = None supp = JobFolder() for n in xrange(3, 6): job = supp / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 supp['5'].sleep = 0 supp['5'].functional.order = 34 supp['5'].functional.fail = 'end' comm = default_comm.copy() comm['n'] = 4 dir = mkdtemp() save(root, join(dir, 'dict.dict'), overwrite=True) try: program = PoolProcess(root, processalloc=processalloc, outdir=dir, keepalive=True) assert program.nbjobsleft > 0 # compute current jobs. program.start(comm) try: program.wait() except Fail: pass else: print program.errors raise Exception() assert hasattr(program, '_comm') # compute second set of updated jobs program.update(supp) try: program.wait() except Fail: pass else: raise Exception() finally: try: rmtree(dir) except: pass
def test_update(executable): """ Tests JobFolderProcess with update. """ from tempfile import mkdtemp from os.path import join from shutil import rmtree from pylada.jobfolder.jobfolder import JobFolder from pylada.jobfolder import save from pylada.process.pool import PoolProcess from pylada import default_comm from functional import Functional root = JobFolder() for n in xrange(3): job = root / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 supp = JobFolder() for n in xrange(3, 6): job = supp / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 comm = default_comm.copy() comm['n'] = 4 dir = mkdtemp() save(root, join(dir, 'dict.dict'), overwrite=True) try: program = PoolProcess(root, processalloc=processalloc, outdir=dir, keepalive=True) assert program.nbjobsleft > 0 # compute current jobs. program.start(comm) program.wait() assert hasattr(program, '_comm') # compute second set of updated jobs program.update(supp) program.wait() finally: try: rmtree(dir) except: pass
def test(executable): """ Tests JobFolderProcess. Includes failure modes. """ from tempfile import mkdtemp from os.path import join from shutil import rmtree from numpy import all, arange, abs, array from pylada.jobfolder.jobfolder import JobFolder from pylada.jobfolder.massextract import MassExtract from pylada.jobfolder import save from pylada.process.jobfolder import JobFolderProcess from pylada.process import Fail, AlreadyStarted, NotStarted from pylada import default_comm from functional import Functional root = JobFolder() for n in xrange(8): job = root / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 comm = default_comm.copy() comm['n'] = 4 dir = mkdtemp() try: program = JobFolderProcess(root, nbpools=2, outdir=dir) assert program.nbjobsleft > 0 # program not started. should fail. try: program.poll() except NotStarted: pass else: raise Exception() try: program.wait() except NotStarted: pass else: raise Exception() # now starting for real. program.start(comm) assert len(program.process) == 2 # Should not be possible to start twice. try: program.start(comm) except AlreadyStarted: pass else: raise Exception() while not program.poll(): continue assert program.nbjobsleft == 0 save(root, join(dir, 'dict.dict'), overwrite=True) extract = MassExtract(join(dir, 'dict.dict')) assert all(extract.success.itervalues()) order = array(extract.order.values()).flatten() assert all(arange(8) - order == 0) pi = array(extract.pi.values()).flatten() assert all(abs(pi - array([0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907, 3.143293]))\ < 1e-5 ) error = array(extract.error.values()).flatten() assert all(abs(error - array([3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865, 0.00333321, 0.002314774, 0.001700664]))\ < 1e-5 ) assert all(n['n'] == comm['n'] for n in extract.comm) # restart assert program.poll() assert len(program.process) == 0 program.start(comm) assert len(program.process) == 0 assert program.poll() finally: try: rmtree(dir) except: pass try: job = root / str(666) job.functional = Functional(executable, [666]) program = JobFolderProcess(root, nbpools=2, outdir=dir) assert program.nbjobsleft > 0 program.start(comm) program.wait() assert program.nbjobsleft == 0 except Fail as r: assert len(program.errors.keys()) == 1 assert '666' in program.errors assert len(program._finished) == 8 else: raise Exception finally: try: rmtree(dir) except: pass try: job.functional.order = [667] program = JobFolderProcess(root, nbpools=2, outdir=dir) assert program.nbjobsleft > 0 program.start(comm) program.wait() assert program.nbjobsleft == 0 finally: try: rmtree(dir) except: pass
def test(executable): """ Tests JobFolderProcess. Includes failure modes. """ from tempfile import mkdtemp from os.path import join from shutil import rmtree from numpy import all, arange, abs, array from pylada.jobfolder.jobfolder import JobFolder from pylada.jobfolder.massextract import MassExtract from pylada.jobfolder import save from pylada.process.jobfolder import JobFolderProcess from pylada.process import Fail, AlreadyStarted, NotStarted from pylada import default_comm from functional import Functional root = JobFolder() for n in xrange(8): job = root / str(n) job.functional = Functional(executable, [n]) job.params["sleep"] = 1 comm = default_comm.copy() comm["n"] = 4 dir = mkdtemp() try: program = JobFolderProcess(root, nbpools=2, outdir=dir) assert program.nbjobsleft > 0 # program not started. should fail. try: program.poll() except NotStarted: pass else: raise Exception() try: program.wait() except NotStarted: pass else: raise Exception() # now starting for real. program.start(comm) assert len(program.process) == 2 # Should not be possible to start twice. try: program.start(comm) except AlreadyStarted: pass else: raise Exception() while not program.poll(): continue assert program.nbjobsleft == 0 save(root, join(dir, "dict.dict"), overwrite=True) extract = MassExtract(join(dir, "dict.dict")) assert all(extract.success.itervalues()) order = array(extract.order.values()).flatten() assert all(arange(8) - order == 0) pi = array(extract.pi.values()).flatten() assert all(abs(pi - array([0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907, 3.143293])) < 1e-5) error = array(extract.error.values()).flatten() assert all( abs( error - array( [3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865, 0.00333321, 0.002314774, 0.001700664] ) ) < 1e-5 ) assert all(n["n"] == comm["n"] for n in extract.comm) # restart assert program.poll() assert len(program.process) == 0 program.start(comm) assert len(program.process) == 0 assert program.poll() finally: try: rmtree(dir) except: pass try: job = root / str(666) job.functional = Functional(executable, [666]) program = JobFolderProcess(root, nbpools=2, outdir=dir) assert program.nbjobsleft > 0 program.start(comm) program.wait() assert program.nbjobsleft == 0 except Fail as r: assert len(program.errors.keys()) == 1 assert "666" in program.errors assert len(program._finished) == 8 else: raise Exception finally: try: rmtree(dir) except: pass try: job.functional.order = [667] program = JobFolderProcess(root, nbpools=2, outdir=dir) assert program.nbjobsleft > 0 program.start(comm) program.wait() assert program.nbjobsleft == 0 finally: try: rmtree(dir) except: pass
def test_fakeexec(): """ Tests Pool. Includes failure modes. """ from tempfile import mkdtemp from os.path import join, abspath, dirname from shutil import rmtree from numpy import all, arange, abs, array from pylada.jobfolder.jobfolder import JobFolder from pylada.jobfolder.massextract import MassExtract from pylada.jobfolder import save from pylada.process.pool import PoolProcess from pylada.process import Fail, NotStarted from pylada import default_comm from pylada.process.tests.functional import Functional from pylada.process.tests.pifunctional import __file__ as executable executable = join(dirname(executable), "pifunctional.py") root = JobFolder() for n in xrange(8): job = root / "a{0}".format(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 comm = default_comm.copy() comm['n'] = 4 dir = mkdtemp() save(root, join(dir, 'dict.dict'), overwrite=True) try: program = PoolProcess(root, processalloc=processalloc, outdir=dir) assert program.nbjobsleft > 0 # program not started. should fail. try: program.poll() except NotStarted: pass else: raise Exception() try: program.wait() except NotStarted: pass else: raise Exception() # now starting for real. program.start(comm) program.wait() assert program.nbjobsleft == 0 extract = MassExtract(join(dir, 'dict.dict')) assert all(extract.success.itervalues()) order = array(extract.order.values()).flatten() assert all(arange(8) - order == 0) pi = array(extract.pi.values()).flatten() expected = [0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907, 3.143293] assert all(abs(pi - array(expected)) < 1e-5) error = array(extract.error.values()).flatten() expected = [3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865, 0.00333321, 0.002314774, 0.001700664] assert all(abs(error - array(expected)) < 1e-5) assert all(n['n'] == comm['n'] for n in extract.comm) # restart assert program.poll() assert len(program.process) == 0 program.start(comm) assert len(program.process) == 0 assert program.poll() try: job = root / str(666) job.functional = Functional(executable, [50], fail='end') program = PoolProcess(root, nbpools=2, outdir=dir, processalloc=processalloc) assert program.nbjobsleft > 0 program.start(comm) program.wait() assert False # Should not reach this point except Fail: assert len(program.errors.keys()) == 1 assert '666' in program.errors else: raise Exception() job.functional.order = [50] program = PoolProcess(root, nbpools=2, outdir=dir, processalloc=processalloc) assert program.nbjobsleft > 0 program.start(comm) program.wait() assert program.nbjobsleft == 0 finally: try: rmtree(dir) except: pass