def test_completes(program, comm, tmpdir, root, do_multiple_mpi_programs):
from numpy import all, arange, abs, array
from pylada.jobfolder.massextract import MassExtract
from pylada.jobfolder import save
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
save(root, str(tmpdir.join('dict.dict')), overwrite=True)
extract = MassExtract(str(tmpdir.join('dict.dict')))
assert all(extract.success.values())
order = array(list(extract.order.values())).flatten()
assert all(arange(8) - order == 0)
pi = array(list(extract.pi.values())).flatten()
expected = [
0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907, 3.143293
]
assert all(abs(pi - array(expected)) < 1e-5)
error = array(list(extract.error.values())).flatten()
expected = [
3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865, 0.00333321,
0.002314774, 0.001700664
]
assert all(abs(error - array(expected)) < 1e-5)
assert all(n['n'] == comm['n'] for n in extract.comm)
def test_update(tmpdir, executable, comm, Process, do_multiple_mpi_programs):
from copy import deepcopy
from pylada.jobfolder.massextract import MassExtract
from pylada.jobfolder import save
# creates and start computing first set of jobs
root = jobfolders(executable, 0, 3)
program = Process(tmpdir, deepcopy(root), keepalive=True)
assert program.keepalive
program.start(comm)
assert hasattr(program, '_comm')
# update jobfolder and start computing second set of jobs
supp = jobfolders(executable, 3, 6)
program.update(supp)
for key in supp.keys():
assert key in program.jobfolder
for key in root.keys():
assert key in program.jobfolder
assert len(program.jobfolder) == len(root) + len(supp)
# create mass extraction object and check succcess
save(program.jobfolder, str(tmpdir.join('dict.dict')), overwrite=True)
extract = MassExtract(str(tmpdir.join('dict.dict')))
# wait for job completion and check for success
program.wait()
assert len(extract.success.values()) == 6
assert all((u for u in extract.success.values()))
def test(executable):
""" Tests JobFolderProcess. Includes failure modes. """
from tempfile import mkdtemp
from os.path import join
from shutil import rmtree
from numpy import all, arange, abs, array
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder.massextract import MassExtract
from pylada.jobfolder import save
from pylada.process.jobfolder import JobFolderProcess
from pylada.process import Fail, AlreadyStarted, NotStarted
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(8):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
# program not started. should fail.
try:
program.poll()
except NotStarted:
pass
else:
raise Exception()
try:
program.wait()
except NotStarted:
pass
else:
raise Exception()
# now starting for real.
program.start(comm)
assert len(program.process) == 2
# Should not be possible to start twice.
try:
program.start(comm)
except AlreadyStarted:
pass
else:
raise Exception()
while not program.poll():
continue
assert program.nbjobsleft == 0
save(root, join(dir, 'dict.dict'), overwrite=True)
extract = MassExtract(join(dir, 'dict.dict'))
assert all(extract.success.itervalues())
order = array(extract.order.values()).flatten()
assert all(arange(8) - order == 0)
pi = array(extract.pi.values()).flatten()
assert all(abs(pi - array([0.0, 3.2, 3.162353, 3.150849,
3.146801, 3.144926, 3.143907, 3.143293]))\
< 1e-5 )
error = array(extract.error.values()).flatten()
assert all(abs(error - array([3.141593, 0.05840735, 0.02076029, 0.009256556,
0.005207865, 0.00333321, 0.002314774, 0.001700664]))\
< 1e-5 )
assert all(n['n'] == comm['n'] for n in extract.comm)
# restart
assert program.poll()
assert len(program.process) == 0
program.start(comm)
assert len(program.process) == 0
assert program.poll()
finally:
try:
rmtree(dir)
except:
pass
try:
job = root / str(666)
job.functional = Functional(executable, [666])
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
except Fail as r:
assert len(program.errors.keys()) == 1
assert '666' in program.errors
assert len(program._finished) == 8
else:
raise Exception
finally:
try:
rmtree(dir)
except:
pass
try:
job.functional.order = [667]
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
finally:
try:
rmtree(dir)
except:
pass