def test_update(executable): """ Tests JobFolderProcess with update. """ from tempfile import mkdtemp from shutil import rmtree from pylada.jobfolder.jobfolder import JobFolder from pylada.process.jobfolder import JobFolderProcess from pylada import default_comm from functional import Functional root = JobFolder() for n in xrange(3): job = root / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 supp = JobFolder() for n in xrange(3, 6): job = supp / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 comm = default_comm.copy() comm['n'] = 4 dir = mkdtemp() try: program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True) assert program.keepalive # compute current jobs. program.start(comm) program.wait() assert hasattr(program, '_comm') # compute second set of updated jobs program.update(supp) program.wait() finally: try: rmtree(dir) except: pass # check with deleteold=True dir = mkdtemp() try: program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True) assert program.keepalive # compute current jobs. program.start(comm) program.wait() assert hasattr(program, '_comm') # compute second set of updated jobs program.update(supp, deleteold=True) assert hasattr(program, '_comm') program.wait() finally: try: rmtree(dir) except: pass
def test_update_with_fail(executable): """ Tests JobFolderProcess with update. """ from tempfile import mkdtemp from shutil import rmtree from pylada.jobfolder.jobfolder import JobFolder from pylada.process.jobfolder import JobFolderProcess from pylada.process import Fail from pylada import default_comm from functional import Functional root = JobFolder() for n in xrange(3): job = root / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 root['1'].functional.order = 666 root['1'].sleep = None supp = JobFolder() for n in xrange(3, 6): job = supp / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 supp['5'].sleep = 0 supp['5'].functional.order = 666 comm = default_comm.copy() comm['n'] = 4 dir = mkdtemp() try: program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True) # compute current jobs. program.start(comm) try: program.wait() except Fail: pass else: raise Exception() assert len(program.errors) == 1 # compute second set of updated jobs program.update(supp) try: program.wait() except Fail: pass else: raise Exception() assert len(program.errors) == 2 program.errors.clear() finally: try: rmtree(dir) except: pass
def test_update_with_fail(executable): """ Tests JobFolderProcess with update. """ from tempfile import mkdtemp from shutil import rmtree from pylada.jobfolder.jobfolder import JobFolder from pylada.process.jobfolder import JobFolderProcess from pylada.process import Fail from pylada import default_comm from functional import Functional root = JobFolder() for n in xrange(3): job = root / str(n) job.functional = Functional(executable, [n]) job.params["sleep"] = 1 root["1"].functional.order = 666 root["1"].sleep = None supp = JobFolder() for n in xrange(3, 6): job = supp / str(n) job.functional = Functional(executable, [n]) job.params["sleep"] = 1 supp["5"].sleep = 0 supp["5"].functional.order = 666 comm = default_comm.copy() comm["n"] = 4 dir = mkdtemp() try: program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True) # compute current jobs. program.start(comm) try: program.wait() except Fail: pass else: raise Exception() assert len(program.errors) == 1 # compute second set of updated jobs program.update(supp) try: program.wait() except Fail: pass else: raise Exception() assert len(program.errors) == 2 program.errors.clear() finally: try: rmtree(dir) except: pass
def test_update_with_fail(): """ Tests JobFolderProcess with update and failure. """ from os.path import join, abspath, dirname from tempfile import mkdtemp from shutil import rmtree from pylada.jobfolder.jobfolder import JobFolder from pylada.process.jobfolder import JobFolderProcess from pylada.process import Fail from pylada import default_comm from pylada.process.tests.functional import Functional from pylada.process.tests.pifunctional import __file__ as executable executable = join(dirname(executable), "pifunctional.py") root = JobFolder() for n in xrange(3): job = root / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 root['1'].functional.order = 68 root['1'].functional.fail = 'end' root['1'].sleep = None supp = JobFolder() for n in xrange(3, 6): job = supp / str(n) job.functional = Functional(executable, [n]) job.params['sleep'] = 1 supp['5'].sleep = 0 supp['5'].functional.order = 78 supp['5'].functional.fail = 'midway' comm = default_comm.copy() comm['n'] = 4 dir = mkdtemp() try: program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True) # compute current jobs. program.start(comm) try: program.wait() except Fail: pass else: raise Exception() assert len(program.errors) == 1 # compute second set of updated jobs program.update(supp) try: program.wait() except Fail: pass else: raise Exception() assert len(program.errors) == 2 program.errors.clear() finally: try: rmtree(dir) except: pass