def test_completes(program, comm, tmpdir, root, do_multiple_mpi_programs):
from numpy import all, arange, abs, array
from pylada.jobfolder.massextract import MassExtract
from pylada.jobfolder import save
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
save(root, str(tmpdir.join('dict.dict')), overwrite=True)
extract = MassExtract(str(tmpdir.join('dict.dict')))
assert all(extract.success.values())
order = array(list(extract.order.values())).flatten()
assert all(arange(8) - order == 0)
pi = array(list(extract.pi.values())).flatten()
expected = [
0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907, 3.143293
]
assert all(abs(pi - array(expected)) < 1e-5)
error = array(list(extract.error.values())).flatten()
expected = [
3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865, 0.00333321,
0.002314774, 0.001700664
]
assert all(abs(error - array(expected)) < 1e-5)
assert all(n['n'] == comm['n'] for n in extract.comm)
def test_completes(program, comm, tmpdir, root, do_multiple_mpi_programs):
from numpy import all, arange, abs, array
from pylada.jobfolder.massextract import MassExtract
from pylada.jobfolder import save
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
save(root, str(tmpdir.join('dict.dict')), overwrite=True)
extract = MassExtract(str(tmpdir.join('dict.dict')))
assert all(extract.success.values())
order = array(list(extract.order.values())).flatten()
assert all(arange(8) - order == 0)
pi = array(list(extract.pi.values())).flatten()
expected = [
0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907, 3.143293
]
assert all(abs(pi - array(expected)) < 1e-5)
error = array(list(extract.error.values())).flatten()
expected = [
3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865, 0.00333321,
0.002314774, 0.001700664
]
assert all(abs(error - array(expected)) < 1e-5)
assert all(n['n'] == comm['n'] for n in extract.comm)
def test_update(tmpdir, executable, comm, Process, do_multiple_mpi_programs):
from copy import deepcopy
from pylada.jobfolder.massextract import MassExtract
from pylada.jobfolder import save
# creates and start computing first set of jobs
root = jobfolders(executable, 0, 3)
program = Process(tmpdir, deepcopy(root), keepalive=True)
assert program.keepalive
program.start(comm)
assert hasattr(program, '_comm')
# update jobfolder and start computing second set of jobs
supp = jobfolders(executable, 3, 6)
program.update(supp)
for key in supp.keys():
assert key in program.jobfolder
for key in root.keys():
assert key in program.jobfolder
assert len(program.jobfolder) == len(root) + len(supp)
# create mass extraction object and check succcess
save(program.jobfolder, str(tmpdir.join('dict.dict')), overwrite=True)
extract = MassExtract(str(tmpdir.join('dict.dict')))
# wait for job completion and check for success
program.wait()
assert len(extract.success.values()) == 6
assert all((u for u in extract.success.values()))
def test_update(tmpdir, executable, comm, Process, do_multiple_mpi_programs):
from copy import deepcopy
from pylada.jobfolder.massextract import MassExtract
from pylada.jobfolder import save
# creates and start computing first set of jobs
root = jobfolders(executable, 0, 3)
program = Process(tmpdir, deepcopy(root), keepalive=True)
assert program.keepalive
program.start(comm)
assert hasattr(program, '_comm')
# update jobfolder and start computing second set of jobs
supp = jobfolders(executable, 3, 6)
program.update(supp)
for key in supp.keys():
assert key in program.jobfolder
for key in root.keys():
assert key in program.jobfolder
assert len(program.jobfolder) == len(root) + len(supp)
# create mass extraction object and check succcess
save(program.jobfolder, str(tmpdir.join('dict.dict')), overwrite=True)
extract = MassExtract(str(tmpdir.join('dict.dict')))
# wait for job completion and check for success
program.wait()
assert len(extract.success.values()) == 6
assert all((u for u in extract.success.values()))
def test_update_with_fail():
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from os.path import join, abspath, dirname
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder import save
from pylada.process.pool import PoolProcess
from pylada.process import Fail
from pylada import default_comm
from pylada.process.tests.functional import Functional
from pylada.process.tests.pifunctional import __file__ as executable
executable = join(dirname(executable), "pifunctional.py")
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
root['1'].functional.order = 50
root['1'].functional.fail = 'midway'
root['1'].sleep = None
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
supp['5'].sleep = 0
supp['5'].functional.order = 34
supp['5'].functional.fail = 'end'
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
save(root, join(dir, 'dict.dict'), overwrite=True)
try:
program = PoolProcess(root, processalloc=processalloc, outdir=dir, keepalive=True)
assert program.nbjobsleft > 0
# compute current jobs.
program.start(comm)
try: program.wait()
except Fail: pass
else:
print program.errors
raise Exception()
assert hasattr(program, '_comm')
# compute second set of updated jobs
program.update(supp)
try: program.wait()
except Fail: pass
else: raise Exception()
finally:
try: rmtree(dir)
except: pass
def test_update(executable):
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from os.path import join
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder import save
from pylada.process.pool import PoolProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
save(root, join(dir, 'dict.dict'), overwrite=True)
try:
program = PoolProcess(root,
processalloc=processalloc,
outdir=dir,
keepalive=True)
assert program.nbjobsleft > 0
# compute current jobs.
program.start(comm)
program.wait()
assert hasattr(program, '_comm')
# compute second set of updated jobs
program.update(supp)
program.wait()
finally:
try:
rmtree(dir)
except:
pass
def test_update(executable):
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from os.path import join
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder import save
from pylada.process.pool import PoolProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
save(root, join(dir, 'dict.dict'), overwrite=True)
try:
program = PoolProcess(root, processalloc=processalloc, outdir=dir, keepalive=True)
assert program.nbjobsleft > 0
# compute current jobs.
program.start(comm)
program.wait()
assert hasattr(program, '_comm')
# compute second set of updated jobs
program.update(supp)
program.wait()
finally:
try: rmtree(dir)
except: pass
def test(executable):
""" Tests JobFolderProcess. Includes failure modes. """
from tempfile import mkdtemp
from os.path import join
from shutil import rmtree
from numpy import all, arange, abs, array
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder.massextract import MassExtract
from pylada.jobfolder import save
from pylada.process.jobfolder import JobFolderProcess
from pylada.process import Fail, AlreadyStarted, NotStarted
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(8):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
# program not started. should fail.
try:
program.poll()
except NotStarted:
pass
else:
raise Exception()
try:
program.wait()
except NotStarted:
pass
else:
raise Exception()
# now starting for real.
program.start(comm)
assert len(program.process) == 2
# Should not be possible to start twice.
try:
program.start(comm)
except AlreadyStarted:
pass
else:
raise Exception()
while not program.poll():
continue
assert program.nbjobsleft == 0
save(root, join(dir, 'dict.dict'), overwrite=True)
extract = MassExtract(join(dir, 'dict.dict'))
assert all(extract.success.itervalues())
order = array(extract.order.values()).flatten()
assert all(arange(8) - order == 0)
pi = array(extract.pi.values()).flatten()
assert all(abs(pi - array([0.0, 3.2, 3.162353, 3.150849,
3.146801, 3.144926, 3.143907, 3.143293]))\
< 1e-5 )
error = array(extract.error.values()).flatten()
assert all(abs(error - array([3.141593, 0.05840735, 0.02076029, 0.009256556,
0.005207865, 0.00333321, 0.002314774, 0.001700664]))\
< 1e-5 )
assert all(n['n'] == comm['n'] for n in extract.comm)
# restart
assert program.poll()
assert len(program.process) == 0
program.start(comm)
assert len(program.process) == 0
assert program.poll()
finally:
try:
rmtree(dir)
except:
pass
try:
job = root / str(666)
job.functional = Functional(executable, [666])
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
except Fail as r:
assert len(program.errors.keys()) == 1
assert '666' in program.errors
assert len(program._finished) == 8
else:
raise Exception
finally:
try:
rmtree(dir)
except:
pass
try:
job.functional.order = [667]
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
finally:
try:
rmtree(dir)
except:
pass
def test(executable):
""" Tests JobFolderProcess. Includes failure modes. """
from tempfile import mkdtemp
from os.path import join
from shutil import rmtree
from numpy import all, arange, abs, array
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder.massextract import MassExtract
from pylada.jobfolder import save
from pylada.process.jobfolder import JobFolderProcess
from pylada.process import Fail, AlreadyStarted, NotStarted
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(8):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params["sleep"] = 1
comm = default_comm.copy()
comm["n"] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
# program not started. should fail.
try:
program.poll()
except NotStarted:
pass
else:
raise Exception()
try:
program.wait()
except NotStarted:
pass
else:
raise Exception()
# now starting for real.
program.start(comm)
assert len(program.process) == 2
# Should not be possible to start twice.
try:
program.start(comm)
except AlreadyStarted:
pass
else:
raise Exception()
while not program.poll():
continue
assert program.nbjobsleft == 0
save(root, join(dir, "dict.dict"), overwrite=True)
extract = MassExtract(join(dir, "dict.dict"))
assert all(extract.success.itervalues())
order = array(extract.order.values()).flatten()
assert all(arange(8) - order == 0)
pi = array(extract.pi.values()).flatten()
assert all(abs(pi - array([0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907, 3.143293])) < 1e-5)
error = array(extract.error.values()).flatten()
assert all(
abs(
error
- array(
[3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865, 0.00333321, 0.002314774, 0.001700664]
)
)
< 1e-5
)
assert all(n["n"] == comm["n"] for n in extract.comm)
# restart
assert program.poll()
assert len(program.process) == 0
program.start(comm)
assert len(program.process) == 0
assert program.poll()
finally:
try:
rmtree(dir)
except:
pass
try:
job = root / str(666)
job.functional = Functional(executable, [666])
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
except Fail as r:
assert len(program.errors.keys()) == 1
assert "666" in program.errors
assert len(program._finished) == 8
else:
raise Exception
finally:
try:
rmtree(dir)
except:
pass
try:
job.functional.order = [667]
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
finally:
try:
rmtree(dir)
except:
pass
def test_fakeexec():
""" Tests Pool. Includes failure modes. """
from tempfile import mkdtemp
from os.path import join, abspath, dirname
from shutil import rmtree
from numpy import all, arange, abs, array
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder.massextract import MassExtract
from pylada.jobfolder import save
from pylada.process.pool import PoolProcess
from pylada.process import Fail, NotStarted
from pylada import default_comm
from pylada.process.tests.functional import Functional
from pylada.process.tests.pifunctional import __file__ as executable
executable = join(dirname(executable), "pifunctional.py")
root = JobFolder()
for n in xrange(8):
job = root / "a{0}".format(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
save(root, join(dir, 'dict.dict'), overwrite=True)
try:
program = PoolProcess(root, processalloc=processalloc, outdir=dir)
assert program.nbjobsleft > 0
# program not started. should fail.
try: program.poll()
except NotStarted: pass
else: raise Exception()
try: program.wait()
except NotStarted: pass
else: raise Exception()
# now starting for real.
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
extract = MassExtract(join(dir, 'dict.dict'))
assert all(extract.success.itervalues())
order = array(extract.order.values()).flatten()
assert all(arange(8) - order == 0)
pi = array(extract.pi.values()).flatten()
expected = [0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907,
3.143293]
assert all(abs(pi - array(expected)) < 1e-5)
error = array(extract.error.values()).flatten()
expected = [3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865,
0.00333321, 0.002314774, 0.001700664]
assert all(abs(error - array(expected)) < 1e-5)
assert all(n['n'] == comm['n'] for n in extract.comm)
# restart
assert program.poll()
assert len(program.process) == 0
program.start(comm)
assert len(program.process) == 0
assert program.poll()
try:
job = root / str(666)
job.functional = Functional(executable, [50], fail='end')
program = PoolProcess(root, nbpools=2,
outdir=dir, processalloc=processalloc)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert False # Should not reach this point
except Fail:
assert len(program.errors.keys()) == 1
assert '666' in program.errors
else: raise Exception()
job.functional.order = [50]
program = PoolProcess(root, nbpools=2,
outdir=dir, processalloc=processalloc)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
finally:
try: rmtree(dir)
except: pass
