def test_update_with_fail(executable):
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.jobfolder import JobFolderProcess
from pylada.process import Fail
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
root['1'].functional.order = 666
root['1'].sleep = None
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
supp['5'].sleep = 0
supp['5'].functional.order = 666
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
# compute current jobs.
program.start(comm)
try:
program.wait()
except Fail:
pass
else:
raise Exception()
assert len(program.errors) == 1
# compute second set of updated jobs
program.update(supp)
try:
program.wait()
except Fail:
pass
else:
raise Exception()
assert len(program.errors) == 2
program.errors.clear()
finally:
try:
rmtree(dir)
except:
pass
def test_update_with_fail(executable):
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.jobfolder import JobFolderProcess
from pylada.process import Fail
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params["sleep"] = 1
root["1"].functional.order = 666
root["1"].sleep = None
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params["sleep"] = 1
supp["5"].sleep = 0
supp["5"].functional.order = 666
comm = default_comm.copy()
comm["n"] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
# compute current jobs.
program.start(comm)
try:
program.wait()
except Fail:
pass
else:
raise Exception()
assert len(program.errors) == 1
# compute second set of updated jobs
program.update(supp)
try:
program.wait()
except Fail:
pass
else:
raise Exception()
assert len(program.errors) == 2
program.errors.clear()
finally:
try:
rmtree(dir)
except:
pass
def main():
import re
from sys import path as python_path
from os import getcwd
from os.path import exists
from argparse import ArgumentParser
from pylada import jobfolder
from pylada.process.mpi import create_global_comm
from pylada.process.jobfolder import JobFolderProcess
import pylada
# below would go additional imports.
parser = ArgumentParser(prog="runasone", description=re.sub("\\s+", " ", __doc__[1:]))
parser.add_argument("--pools", type=int, default=0, help="Number of pools")
parser.add_argument("--ppath", dest="ppath", default=None,
help="Directory to add to python path",
metavar="Directory")
parser.add_argument('--nbprocs', dest="nbprocs", default=pylada.default_comm['n'], type=int,
help="Number of processors with which to launch job.")
parser.add_argument('--ppn', dest="ppn", default=pylada.default_comm['ppn'], type=int,
help="Number of processors with which to launch job.")
parser.add_argument('--timeout', dest="timeout", default=300, type=int,
help="Time to wait for job-dictionary to becom available "
"before timing out (in seconds). A negative or null "
"value implies forever. Defaults to 5mn.")
parser.add_argument('pickle', metavar='FILE', type=str, help='Path to a job-folder.')
try:
options = parser.parse_args()
except SystemExit:
return
# additional path to look into.
if options.ppath is not None:
python_path.append(options.ppath)
if not exists(options.pickle):
print("Could not find file {0}.".format(options.pickle))
return
# Set up mpi processes.
pylada.default_comm['ppn'] = options.ppn
pylada.default_comm['n'] = options.nbprocs
create_global_comm(options.nbprocs)
timeout = None if options.timeout <= 0 else options.timeout
jobfolder = jobfolder.load(options.pickle, timeout=timeout)
process = JobFolderProcess(jobfolder, outdir=getcwd(), nbpools=options.pools)
process.start(pylada.default_comm)
process.wait(60)
def test_update_with_fail():
""" Tests JobFolderProcess with update and failure. """
from os.path import join, abspath, dirname
from tempfile import mkdtemp
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.jobfolder import JobFolderProcess
from pylada.process import Fail
from pylada import default_comm
from pylada.process.tests.functional import Functional
from pylada.process.tests.pifunctional import __file__ as executable
executable = join(dirname(executable), "pifunctional.py")
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
root['1'].functional.order = 68
root['1'].functional.fail = 'end'
root['1'].sleep = None
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
supp['5'].sleep = 0
supp['5'].functional.order = 78
supp['5'].functional.fail = 'midway'
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
# compute current jobs.
program.start(comm)
try: program.wait()
except Fail: pass
else: raise Exception()
assert len(program.errors) == 1
# compute second set of updated jobs
program.update(supp)
try: program.wait()
except Fail: pass
else: raise Exception()
assert len(program.errors) == 2
program.errors.clear()
finally:
try: rmtree(dir)
except: pass
def main():
import re
from sys import path as python_path
from os import getcwd
from os.path import exists
from argparse import ArgumentParser
from pylada import jobfolder
from pylada.process.mpi import create_global_comm
from pylada.process.jobfolder import JobFolderProcess
import pylada
# below would go additional imports.
parser = ArgumentParser(prog="runasone",
description=re.sub("\\s+", " ", __doc__[1:]))
parser.add_argument("--pools", type=int, default=0, help="Number of pools")
parser.add_argument( "--ppath", dest="ppath", default=None, \
help="Directory to add to python path",
metavar="Directory" )
parser.add_argument('--nbprocs', dest="nbprocs", default=pylada.default_comm['n'], type=int,\
help="Number of processors with which to launch job.")
parser.add_argument('--ppn', dest="ppn", default=pylada.default_comm['ppn'], type=int,\
help="Number of processors with which to launch job.")
parser.add_argument('--timeout', dest="timeout", default=300, type=int,\
help="Time to wait for job-dictionary to becom available "
"before timing out (in seconds). A negative or null "
"value implies forever. Defaults to 5mn.")
parser.add_argument('pickle',
metavar='FILE',
type=str,
help='Path to a job-folder.')
try:
options = parser.parse_args()
except SystemExit:
return
# additional path to look into.
if options.ppath is not None: python_path.append(options.ppath)
if not exists(options.pickle):
print "Could not find file {0}.".format(options.pickle)
return
# Set up mpi processes.
pylada.default_comm['ppn'] = options.ppn
pylada.default_comm['n'] = options.nbprocs
create_global_comm(options.nbprocs)
timeout = None if options.timeout <= 0 else options.timeout
jobfolder = jobfolder.load(options.pickle, timeout=timeout)
process = JobFolderProcess(jobfolder,
outdir=getcwd(),
nbpools=options.pools)
process.start(pylada.default_comm)
process.wait(60)
def test(executable):
""" Tests JobFolderProcess. Includes failure modes. """
from tempfile import mkdtemp
from os.path import join
from shutil import rmtree
from numpy import all, arange, abs, array
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder.massextract import MassExtract
from pylada.jobfolder import save
from pylada.process.jobfolder import JobFolderProcess
from pylada.process import Fail, AlreadyStarted, NotStarted
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(8):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
# program not started. should fail.
try:
program.poll()
except NotStarted:
pass
else:
raise Exception()
try:
program.wait()
except NotStarted:
pass
else:
raise Exception()
# now starting for real.
program.start(comm)
assert len(program.process) == 2
# Should not be possible to start twice.
try:
program.start(comm)
except AlreadyStarted:
pass
else:
raise Exception()
while not program.poll():
continue
assert program.nbjobsleft == 0
save(root, join(dir, 'dict.dict'), overwrite=True)
extract = MassExtract(join(dir, 'dict.dict'))
assert all(extract.success.itervalues())
order = array(extract.order.values()).flatten()
assert all(arange(8) - order == 0)
pi = array(extract.pi.values()).flatten()
assert all(abs(pi - array([0.0, 3.2, 3.162353, 3.150849,
3.146801, 3.144926, 3.143907, 3.143293]))\
< 1e-5 )
error = array(extract.error.values()).flatten()
assert all(abs(error - array([3.141593, 0.05840735, 0.02076029, 0.009256556,
0.005207865, 0.00333321, 0.002314774, 0.001700664]))\
< 1e-5 )
assert all(n['n'] == comm['n'] for n in extract.comm)
# restart
assert program.poll()
assert len(program.process) == 0
program.start(comm)
assert len(program.process) == 0
assert program.poll()
finally:
try:
rmtree(dir)
except:
pass
try:
job = root / str(666)
job.functional = Functional(executable, [666])
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
except Fail as r:
assert len(program.errors.keys()) == 1
assert '666' in program.errors
assert len(program._finished) == 8
else:
raise Exception
finally:
try:
rmtree(dir)
except:
pass
try:
job.functional.order = [667]
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
finally:
try:
rmtree(dir)
except:
pass
def test_update(executable):
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.jobfolder import JobFolderProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
assert program.keepalive
# compute current jobs.
program.start(comm)
program.wait()
assert hasattr(program, '_comm')
# compute second set of updated jobs
program.update(supp)
program.wait()
finally:
try:
rmtree(dir)
except:
pass
# check with deleteold=True
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
assert program.keepalive
# compute current jobs.
program.start(comm)
program.wait()
assert hasattr(program, '_comm')
# compute second set of updated jobs
program.update(supp, deleteold=True)
assert hasattr(program, '_comm')
program.wait()
finally:
try:
rmtree(dir)
except:
pass
def test(executable):
""" Tests JobFolderProcess. Includes failure modes. """
from tempfile import mkdtemp
from os.path import join
from shutil import rmtree
from numpy import all, arange, abs, array
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder.massextract import MassExtract
from pylada.jobfolder import save
from pylada.process.jobfolder import JobFolderProcess
from pylada.process import Fail, AlreadyStarted, NotStarted
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(8):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params["sleep"] = 1
comm = default_comm.copy()
comm["n"] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
# program not started. should fail.
try:
program.poll()
except NotStarted:
pass
else:
raise Exception()
try:
program.wait()
except NotStarted:
pass
else:
raise Exception()
# now starting for real.
program.start(comm)
assert len(program.process) == 2
# Should not be possible to start twice.
try:
program.start(comm)
except AlreadyStarted:
pass
else:
raise Exception()
while not program.poll():
continue
assert program.nbjobsleft == 0
save(root, join(dir, "dict.dict"), overwrite=True)
extract = MassExtract(join(dir, "dict.dict"))
assert all(extract.success.itervalues())
order = array(extract.order.values()).flatten()
assert all(arange(8) - order == 0)
pi = array(extract.pi.values()).flatten()
assert all(abs(pi - array([0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907, 3.143293])) < 1e-5)
error = array(extract.error.values()).flatten()
assert all(
abs(
error
- array(
[3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865, 0.00333321, 0.002314774, 0.001700664]
)
)
< 1e-5
)
assert all(n["n"] == comm["n"] for n in extract.comm)
# restart
assert program.poll()
assert len(program.process) == 0
program.start(comm)
assert len(program.process) == 0
assert program.poll()
finally:
try:
rmtree(dir)
except:
pass
try:
job = root / str(666)
job.functional = Functional(executable, [666])
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
except Fail as r:
assert len(program.errors.keys()) == 1
assert "666" in program.errors
assert len(program._finished) == 8
else:
raise Exception
finally:
try:
rmtree(dir)
except:
pass
try:
job.functional.order = [667]
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
finally:
try:
rmtree(dir)
except:
pass
def test_update(executable):
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.jobfolder import JobFolderProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params["sleep"] = 1
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params["sleep"] = 1
comm = default_comm.copy()
comm["n"] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
assert program.keepalive
# compute current jobs.
program.start(comm)
program.wait()
assert hasattr(program, "_comm")
# compute second set of updated jobs
program.update(supp)
program.wait()
finally:
try:
rmtree(dir)
except:
pass
# check with deleteold=True
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
assert program.keepalive
# compute current jobs.
program.start(comm)
program.wait()
assert hasattr(program, "_comm")
# compute second set of updated jobs
program.update(supp, deleteold=True)
assert hasattr(program, "_comm")
program.wait()
finally:
try:
rmtree(dir)
except:
pass
def jobfolder_process(tmpdir, root, **kwargs):
from pylada.process.jobfolder import JobFolderProcess
return JobFolderProcess(root, nbpools=2, outdir=str(tmpdir), **kwargs)
