def test_update_with_fail(executable): """ Tests JobFolderProcess with update. """ from tempfile import mkdtemp from shutil import rmtree from pylada.jobfolder.jobfolder import JobFolder from pylada.process.jobfolder import JobFolderProcess from pylada.process import Fail from pylada import default_comm from functional import Functional root = JobFolder() for n in xrange(3): job = root / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 root['1'].functional.order = 666 root['1'].sleep = None supp = JobFolder() for n in xrange(3, 6): job = supp / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 supp['5'].sleep = 0 supp['5'].functional.order = 666 comm = default_comm.copy() comm['n'] = 4 dir = mkdtemp() try: program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True) # compute current jobs. program.start(comm) try: program.wait() except Fail: pass else: raise Exception() assert len(program.errors) == 1 # compute second set of updated jobs program.update(supp) try: program.wait() except Fail: pass else: raise Exception() assert len(program.errors) == 2 program.errors.clear() finally: try: rmtree(dir) except: pass
def test_update_with_fail(executable): """ Tests JobFolderProcess with update. """ from tempfile import mkdtemp from shutil import rmtree from pylada.jobfolder.jobfolder import JobFolder from pylada.process.jobfolder import JobFolderProcess from pylada.process import Fail from pylada import default_comm from functional import Functional root = JobFolder() for n in xrange(3): job = root / str(n) job.functional = Functional(executable, [n]) job.params["sleep"] = 1 root["1"].functional.order = 666 root["1"].sleep = None supp = JobFolder() for n in xrange(3, 6): job = supp / str(n) job.functional = Functional(executable, [n]) job.params["sleep"] = 1 supp["5"].sleep = 0 supp["5"].functional.order = 666 comm = default_comm.copy() comm["n"] = 4 dir = mkdtemp() try: program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True) # compute current jobs. program.start(comm) try: program.wait() except Fail: pass else: raise Exception() assert len(program.errors) == 1 # compute second set of updated jobs program.update(supp) try: program.wait() except Fail: pass else: raise Exception() assert len(program.errors) == 2 program.errors.clear() finally: try: rmtree(dir) except: pass
def main(): import re from sys import path as python_path from os import getcwd from os.path import exists from argparse import ArgumentParser from pylada import jobfolder from pylada.process.mpi import create_global_comm from pylada.process.jobfolder import JobFolderProcess import pylada # below would go additional imports. parser = ArgumentParser(prog="runasone", description=re.sub("\\s+", " ", __doc__[1:])) parser.add_argument("--pools", type=int, default=0, help="Number of pools") parser.add_argument("--ppath", dest="ppath", default=None, help="Directory to add to python path", metavar="Directory") parser.add_argument('--nbprocs', dest="nbprocs", default=pylada.default_comm['n'], type=int, help="Number of processors with which to launch job.") parser.add_argument('--ppn', dest="ppn", default=pylada.default_comm['ppn'], type=int, help="Number of processors with which to launch job.") parser.add_argument('--timeout', dest="timeout", default=300, type=int, help="Time to wait for job-dictionary to becom available " "before timing out (in seconds). A negative or null " "value implies forever. Defaults to 5mn.") parser.add_argument('pickle', metavar='FILE', type=str, help='Path to a job-folder.') try: options = parser.parse_args() except SystemExit: return # additional path to look into. if options.ppath is not None: python_path.append(options.ppath) if not exists(options.pickle): print("Could not find file {0}.".format(options.pickle)) return # Set up mpi processes. pylada.default_comm['ppn'] = options.ppn pylada.default_comm['n'] = options.nbprocs create_global_comm(options.nbprocs) timeout = None if options.timeout <= 0 else options.timeout jobfolder = jobfolder.load(options.pickle, timeout=timeout) process = JobFolderProcess(jobfolder, outdir=getcwd(), nbpools=options.pools) process.start(pylada.default_comm) process.wait(60)
def test_update_with_fail(): """ Tests JobFolderProcess with update and failure. """ from os.path import join, abspath, dirname from tempfile import mkdtemp from shutil import rmtree from pylada.jobfolder.jobfolder import JobFolder from pylada.process.jobfolder import JobFolderProcess from pylada.process import Fail from pylada import default_comm from pylada.process.tests.functional import Functional from pylada.process.tests.pifunctional import __file__ as executable executable = join(dirname(executable), "pifunctional.py") root = JobFolder() for n in xrange(3): job = root / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 root['1'].functional.order = 68 root['1'].functional.fail = 'end' root['1'].sleep = None supp = JobFolder() for n in xrange(3, 6): job = supp / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 supp['5'].sleep = 0 supp['5'].functional.order = 78 supp['5'].functional.fail = 'midway' comm = default_comm.copy() comm['n'] = 4 dir = mkdtemp() try: program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True) # compute current jobs. program.start(comm) try: program.wait() except Fail: pass else: raise Exception() assert len(program.errors) == 1 # compute second set of updated jobs program.update(supp) try: program.wait() except Fail: pass else: raise Exception() assert len(program.errors) == 2 program.errors.clear() finally: try: rmtree(dir) except: pass
def main(): import re from sys import path as python_path from os import getcwd from os.path import exists from argparse import ArgumentParser from pylada import jobfolder from pylada.process.mpi import create_global_comm from pylada.process.jobfolder import JobFolderProcess import pylada # below would go additional imports. parser = ArgumentParser(prog="runasone", description=re.sub("\\s+", " ", __doc__[1:])) parser.add_argument("--pools", type=int, default=0, help="Number of pools") parser.add_argument( "--ppath", dest="ppath", default=None, \ help="Directory to add to python path", metavar="Directory" ) parser.add_argument('--nbprocs', dest="nbprocs", default=pylada.default_comm['n'], type=int,\ help="Number of processors with which to launch job.") parser.add_argument('--ppn', dest="ppn", default=pylada.default_comm['ppn'], type=int,\ help="Number of processors with which to launch job.") parser.add_argument('--timeout', dest="timeout", default=300, type=int,\ help="Time to wait for job-dictionary to becom available " "before timing out (in seconds). A negative or null " "value implies forever. Defaults to 5mn.") parser.add_argument('pickle', metavar='FILE', type=str, help='Path to a job-folder.') try: options = parser.parse_args() except SystemExit: return # additional path to look into. if options.ppath is not None: python_path.append(options.ppath) if not exists(options.pickle): print "Could not find file {0}.".format(options.pickle) return # Set up mpi processes. pylada.default_comm['ppn'] = options.ppn pylada.default_comm['n'] = options.nbprocs create_global_comm(options.nbprocs) timeout = None if options.timeout <= 0 else options.timeout jobfolder = jobfolder.load(options.pickle, timeout=timeout) process = JobFolderProcess(jobfolder, outdir=getcwd(), nbpools=options.pools) process.start(pylada.default_comm) process.wait(60)
def test(executable): """ Tests JobFolderProcess. Includes failure modes. """ from tempfile import mkdtemp from os.path import join from shutil import rmtree from numpy import all, arange, abs, array from pylada.jobfolder.jobfolder import JobFolder from pylada.jobfolder.massextract import MassExtract from pylada.jobfolder import save from pylada.process.jobfolder import JobFolderProcess from pylada.process import Fail, AlreadyStarted, NotStarted from pylada import default_comm from functional import Functional root = JobFolder() for n in xrange(8): job = root / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 comm = default_comm.copy() comm['n'] = 4 dir = mkdtemp() try: program = JobFolderProcess(root, nbpools=2, outdir=dir) assert program.nbjobsleft > 0 # program not started. should fail. try: program.poll() except NotStarted: pass else: raise Exception() try: program.wait() except NotStarted: pass else: raise Exception() # now starting for real. program.start(comm) assert len(program.process) == 2 # Should not be possible to start twice. try: program.start(comm) except AlreadyStarted: pass else: raise Exception() while not program.poll(): continue assert program.nbjobsleft == 0 save(root, join(dir, 'dict.dict'), overwrite=True) extract = MassExtract(join(dir, 'dict.dict')) assert all(extract.success.itervalues()) order = array(extract.order.values()).flatten() assert all(arange(8) - order == 0) pi = array(extract.pi.values()).flatten() assert all(abs(pi - array([0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907, 3.143293]))\ < 1e-5 ) error = array(extract.error.values()).flatten() assert all(abs(error - array([3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865, 0.00333321, 0.002314774, 0.001700664]))\ < 1e-5 ) assert all(n['n'] == comm['n'] for n in extract.comm) # restart assert program.poll() assert len(program.process) == 0 program.start(comm) assert len(program.process) == 0 assert program.poll() finally: try: rmtree(dir) except: pass try: job = root / str(666) job.functional = Functional(executable, [666]) program = JobFolderProcess(root, nbpools=2, outdir=dir) assert program.nbjobsleft > 0 program.start(comm) program.wait() assert program.nbjobsleft == 0 except Fail as r: assert len(program.errors.keys()) == 1 assert '666' in program.errors assert len(program._finished) == 8 else: raise Exception finally: try: rmtree(dir) except: pass try: job.functional.order = [667] program = JobFolderProcess(root, nbpools=2, outdir=dir) assert program.nbjobsleft > 0 program.start(comm) program.wait() assert program.nbjobsleft == 0 finally: try: rmtree(dir) except: pass
def test_update(executable): """ Tests JobFolderProcess with update. """ from tempfile import mkdtemp from shutil import rmtree from pylada.jobfolder.jobfolder import JobFolder from pylada.process.jobfolder import JobFolderProcess from pylada import default_comm from functional import Functional root = JobFolder() for n in xrange(3): job = root / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 supp = JobFolder() for n in xrange(3, 6): job = supp / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 comm = default_comm.copy() comm['n'] = 4 dir = mkdtemp() try: program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True) assert program.keepalive # compute current jobs. program.start(comm) program.wait() assert hasattr(program, '_comm') # compute second set of updated jobs program.update(supp) program.wait() finally: try: rmtree(dir) except: pass # check with deleteold=True dir = mkdtemp() try: program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True) assert program.keepalive # compute current jobs. program.start(comm) program.wait() assert hasattr(program, '_comm') # compute second set of updated jobs program.update(supp, deleteold=True) assert hasattr(program, '_comm') program.wait() finally: try: rmtree(dir) except: pass
def test(executable): """ Tests JobFolderProcess. Includes failure modes. """ from tempfile import mkdtemp from os.path import join from shutil import rmtree from numpy import all, arange, abs, array from pylada.jobfolder.jobfolder import JobFolder from pylada.jobfolder.massextract import MassExtract from pylada.jobfolder import save from pylada.process.jobfolder import JobFolderProcess from pylada.process import Fail, AlreadyStarted, NotStarted from pylada import default_comm from functional import Functional root = JobFolder() for n in xrange(8): job = root / str(n) job.functional = Functional(executable, [n]) job.params["sleep"] = 1 comm = default_comm.copy() comm["n"] = 4 dir = mkdtemp() try: program = JobFolderProcess(root, nbpools=2, outdir=dir) assert program.nbjobsleft > 0 # program not started. should fail. try: program.poll() except NotStarted: pass else: raise Exception() try: program.wait() except NotStarted: pass else: raise Exception() # now starting for real. program.start(comm) assert len(program.process) == 2 # Should not be possible to start twice. try: program.start(comm) except AlreadyStarted: pass else: raise Exception() while not program.poll(): continue assert program.nbjobsleft == 0 save(root, join(dir, "dict.dict"), overwrite=True) extract = MassExtract(join(dir, "dict.dict")) assert all(extract.success.itervalues()) order = array(extract.order.values()).flatten() assert all(arange(8) - order == 0) pi = array(extract.pi.values()).flatten() assert all(abs(pi - array([0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907, 3.143293])) < 1e-5) error = array(extract.error.values()).flatten() assert all( abs( error - array( [3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865, 0.00333321, 0.002314774, 0.001700664] ) ) < 1e-5 ) assert all(n["n"] == comm["n"] for n in extract.comm) # restart assert program.poll() assert len(program.process) == 0 program.start(comm) assert len(program.process) == 0 assert program.poll() finally: try: rmtree(dir) except: pass try: job = root / str(666) job.functional = Functional(executable, [666]) program = JobFolderProcess(root, nbpools=2, outdir=dir) assert program.nbjobsleft > 0 program.start(comm) program.wait() assert program.nbjobsleft == 0 except Fail as r: assert len(program.errors.keys()) == 1 assert "666" in program.errors assert len(program._finished) == 8 else: raise Exception finally: try: rmtree(dir) except: pass try: job.functional.order = [667] program = JobFolderProcess(root, nbpools=2, outdir=dir) assert program.nbjobsleft > 0 program.start(comm) program.wait() assert program.nbjobsleft == 0 finally: try: rmtree(dir) except: pass
def test_update(executable): """ Tests JobFolderProcess with update. """ from tempfile import mkdtemp from shutil import rmtree from pylada.jobfolder.jobfolder import JobFolder from pylada.process.jobfolder import JobFolderProcess from pylada import default_comm from functional import Functional root = JobFolder() for n in xrange(3): job = root / str(n) job.functional = Functional(executable, [n]) job.params["sleep"] = 1 supp = JobFolder() for n in xrange(3, 6): job = supp / str(n) job.functional = Functional(executable, [n]) job.params["sleep"] = 1 comm = default_comm.copy() comm["n"] = 4 dir = mkdtemp() try: program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True) assert program.keepalive # compute current jobs. program.start(comm) program.wait() assert hasattr(program, "_comm") # compute second set of updated jobs program.update(supp) program.wait() finally: try: rmtree(dir) except: pass # check with deleteold=True dir = mkdtemp() try: program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True) assert program.keepalive # compute current jobs. program.start(comm) program.wait() assert hasattr(program, "_comm") # compute second set of updated jobs program.update(supp, deleteold=True) assert hasattr(program, "_comm") program.wait() finally: try: rmtree(dir) except: pass
def jobfolder_process(tmpdir, root, **kwargs): from pylada.process.jobfolder import JobFolderProcess return JobFolderProcess(root, nbpools=2, outdir=str(tmpdir), **kwargs)