def test_getjobs(nprocs=8, njobs=20):
""" Test scheduling. """
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.pool import PoolProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(njobs):
job = root / "a{0}".format(n)
job.functional = Functional("whatever", [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = nprocs
def processalloc(job):
""" returns a random number between 1 and 4 included. """
from random import randint
return randint(1, comm['n'])
for j in xrange(100):
program = PoolProcess(root, processalloc=processalloc, outdir="whatever")
program._comm = comm
for i in xrange(1000):
jobs = program._getjobs()
assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
(jobs, [program._alloc[u] for u in jobs])
def test_update(executable):
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.jobfolder import JobFolderProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
assert program.keepalive
# compute current jobs.
program.start(comm)
program.wait()
assert hasattr(program, '_comm')
# compute second set of updated jobs
program.update(supp)
program.wait()
finally:
try: rmtree(dir)
except: pass
# check with deleteold=True
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
assert program.keepalive
# compute current jobs.
program.start(comm)
program.wait()
assert hasattr(program, '_comm')
# compute second set of updated jobs
program.update(supp, deleteold=True)
assert hasattr(program, '_comm')
program.wait()
finally:
try: rmtree(dir)
except: pass
def test_getjobs(nprocs=8, njobs=20):
""" Test scheduling. """
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.pool import PoolProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(njobs):
job = root / "a{0}".format(n)
job.functional = Functional("whatever", [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = nprocs
def processalloc(job):
""" returns a random number between 1 and 4 included. """
from random import randint
return randint(1, comm['n'])
for j in xrange(100):
program = PoolProcess(root,
processalloc=processalloc,
outdir="whatever")
program._comm = comm
for i in xrange(1000):
jobs = program._getjobs()
assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
(jobs, [program._alloc[u] for u in jobs])
def test_failures(d):
""" Tests whether scheduling jobs works on known failure cases. """
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.pool import PoolProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(8):
job = root / "a{0}".format(n)
job.functional = Functional("whatever", [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
def processalloc_test1(job):
return d[job.name[1:-1]]
program = PoolProcess(root,
processalloc=processalloc_test1,
outdir="whatever")
program._comm = comm
for i in xrange(10000):
jobs = program._getjobs()
assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
(jobs, [program._alloc[u] for u in jobs])
def test_large():
""" Test speed of job-scheduling for largers, more numerous jobs. """
from random import random
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.pool import PoolProcess
from pylada.process.dummy import DummyFunctional
from pylada import default_comm
root = JobFolder()
for n in xrange(100):
job = root / "a{0}".format(n)
job.functional = DummyFunctional(chance=random()*0.5+0.15)
comm = default_comm.copy()
comm['n'] = 256
def processalloc_test1(job):
""" returns a random number between 1 and 4 included. """
from random import randint
return randint(1, 64)
program = PoolProcess(root, processalloc=processalloc_test1, outdir="whatever")
program.start(comm)
while not program.poll():
# print 256 - program._comm['n'], len(program.process)
continue
def processalloc_test1(job):
""" returns a random number between 1 and 4 included. """
from random import choice
return choice([31, 37, 43])
program = PoolProcess(root, processalloc=processalloc_test1, outdir="whatever")
program.start(comm)
while not program.poll():
# print 256 - program._comm['n'], len(program.process)
continue
def test_failures():
""" Tests whether scheduling jobs works on known failure cases. """
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.pool import PoolProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(8):
job = root / "a{0}".format(n)
job.functional = Functional("whatever", [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
d = {'a1': 1, 'a0': 3, 'a3': 3, 'a2': 3, 'a5': 3, 'a4': 2, 'a7': 2, 'a6': 1}
def processalloc_test1(job): return d[job.name[1:-1]]
program = PoolProcess(root, processalloc=processalloc_test1,
outdir="whatever")
program._comm = comm
for i in xrange(10000):
jobs = program._getjobs()
assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
(jobs, [program._alloc[u] for u in jobs])
def test_update_with_fail(executable):
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.jobfolder import JobFolderProcess
from pylada.process import Fail
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params["sleep"] = 1
root["1"].functional.order = 666
root["1"].sleep = None
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params["sleep"] = 1
supp["5"].sleep = 0
supp["5"].functional.order = 666
comm = default_comm.copy()
comm["n"] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
# compute current jobs.
program.start(comm)
try:
program.wait()
except Fail:
pass
else:
raise Exception()
assert len(program.errors) == 1
# compute second set of updated jobs
program.update(supp)
try:
program.wait()
except Fail:
pass
else:
raise Exception()
assert len(program.errors) == 2
program.errors.clear()
finally:
try:
rmtree(dir)
except:
pass
def test_update_with_fail(executable):
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.jobfolder import JobFolderProcess
from pylada.process import Fail
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
root['1'].functional.order = 666
root['1'].sleep = None
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
supp['5'].sleep = 0
supp['5'].functional.order = 666
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
# compute current jobs.
program.start(comm)
try:
program.wait()
except Fail:
pass
else:
raise Exception()
assert len(program.errors) == 1
# compute second set of updated jobs
program.update(supp)
try:
program.wait()
except Fail:
pass
else:
raise Exception()
assert len(program.errors) == 2
program.errors.clear()
finally:
try:
rmtree(dir)
except:
pass
def test_update_with_fail():
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from os.path import join, abspath, dirname
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder import save
from pylada.process.pool import PoolProcess
from pylada.process import Fail
from pylada import default_comm
from pylada.process.tests.functional import Functional
from pylada.process.tests.pifunctional import __file__ as executable
executable = join(dirname(executable), "pifunctional.py")
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
root['1'].functional.order = 50
root['1'].functional.fail = 'midway'
root['1'].sleep = None
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
supp['5'].sleep = 0
supp['5'].functional.order = 34
supp['5'].functional.fail = 'end'
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
save(root, join(dir, 'dict.dict'), overwrite=True)
try:
program = PoolProcess(root, processalloc=processalloc, outdir=dir, keepalive=True)
assert program.nbjobsleft > 0
# compute current jobs.
program.start(comm)
try: program.wait()
except Fail: pass
else:
print program.errors
raise Exception()
assert hasattr(program, '_comm')
# compute second set of updated jobs
program.update(supp)
try: program.wait()
except Fail: pass
else: raise Exception()
finally:
try: rmtree(dir)
except: pass
def test_update_with_fail():
""" Tests JobFolderProcess with update and failure. """
from os.path import join, abspath, dirname
from tempfile import mkdtemp
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.jobfolder import JobFolderProcess
from pylada.process import Fail
from pylada import default_comm
from pylada.process.tests.functional import Functional
from pylada.process.tests.pifunctional import __file__ as executable
executable = join(dirname(executable), "pifunctional.py")
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
root['1'].functional.order = 68
root['1'].functional.fail = 'end'
root['1'].sleep = None
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
supp['5'].sleep = 0
supp['5'].functional.order = 78
supp['5'].functional.fail = 'midway'
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
# compute current jobs.
program.start(comm)
try: program.wait()
except Fail: pass
else: raise Exception()
assert len(program.errors) == 1
# compute second set of updated jobs
program.update(supp)
try: program.wait()
except Fail: pass
else: raise Exception()
assert len(program.errors) == 2
program.errors.clear()
finally:
try: rmtree(dir)
except: pass
def __init__(self, *args, **kwargs):
from pylada import default_comm
keywords = default_comm.copy()
keywords['n'] = 0
keywords.update(kwargs)
super(Communicator, self).__init__(*args)
super(Communicator, self).update(keywords)
self._nodefile = None
""" Path to temporary node file.
None if no temporary node file yet.
"""
self.machines = {}
""" Holds map of machines this communicator can access. """
self.parent = None
""" Reference to parent communicator from which machines are acquired.
def test_update(executable):
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from os.path import join
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder import save
from pylada.process.pool import PoolProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
save(root, join(dir, 'dict.dict'), overwrite=True)
try:
program = PoolProcess(root,
processalloc=processalloc,
outdir=dir,
keepalive=True)
assert program.nbjobsleft > 0
# compute current jobs.
program.start(comm)
program.wait()
assert hasattr(program, '_comm')
# compute second set of updated jobs
program.update(supp)
program.wait()
finally:
try:
rmtree(dir)
except:
pass
def test_update(executable):
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from os.path import join
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder import save
from pylada.process.pool import PoolProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
save(root, join(dir, 'dict.dict'), overwrite=True)
try:
program = PoolProcess(root, processalloc=processalloc, outdir=dir, keepalive=True)
assert program.nbjobsleft > 0
# compute current jobs.
program.start(comm)
program.wait()
assert hasattr(program, '_comm')
# compute second set of updated jobs
program.update(supp)
program.wait()
finally:
try: rmtree(dir)
except: pass
def test_large():
""" Test speed of job-scheduling for largers, more numerous jobs. """
from random import random
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.pool import PoolProcess
from pylada.process.dummy import DummyFunctional
from pylada import default_comm
root = JobFolder()
for n in xrange(100):
job = root / "a{0}".format(n)
job.functional = DummyFunctional(chance=random() * 0.5 + 0.15)
comm = default_comm.copy()
comm['n'] = 256
def processalloc_test1(job):
""" returns a random number between 1 and 4 included. """
from random import randint
return randint(1, 64)
program = PoolProcess(root,
processalloc=processalloc_test1,
outdir="whatever")
program.start(comm)
while not program.poll():
# print 256 - program._comm['n'], len(program.process)
continue
def processalloc_test1(job):
""" returns a random number between 1 and 4 included. """
from random import choice
return choice([31, 37, 43])
program = PoolProcess(root,
processalloc=processalloc_test1,
outdir="whatever")
program.start(comm)
while not program.poll():
# print 256 - program._comm['n'], len(program.process)
continue
def test(executable):
""" Tests JobFolderProcess. Includes failure modes. """
from tempfile import mkdtemp
from os.path import join
from shutil import rmtree
from numpy import all, arange, abs, array
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder.massextract import MassExtract
from pylada.jobfolder import save
from pylada.process.jobfolder import JobFolderProcess
from pylada.process import Fail, AlreadyStarted, NotStarted
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(8):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
# program not started. should fail.
try:
program.poll()
except NotStarted:
pass
else:
raise Exception()
try:
program.wait()
except NotStarted:
pass
else:
raise Exception()
# now starting for real.
program.start(comm)
assert len(program.process) == 2
# Should not be possible to start twice.
try:
program.start(comm)
except AlreadyStarted:
pass
else:
raise Exception()
while not program.poll():
continue
assert program.nbjobsleft == 0
save(root, join(dir, 'dict.dict'), overwrite=True)
extract = MassExtract(join(dir, 'dict.dict'))
assert all(extract.success.itervalues())
order = array(extract.order.values()).flatten()
assert all(arange(8) - order == 0)
pi = array(extract.pi.values()).flatten()
assert all(abs(pi - array([0.0, 3.2, 3.162353, 3.150849,
3.146801, 3.144926, 3.143907, 3.143293]))\
< 1e-5 )
error = array(extract.error.values()).flatten()
assert all(abs(error - array([3.141593, 0.05840735, 0.02076029, 0.009256556,
0.005207865, 0.00333321, 0.002314774, 0.001700664]))\
< 1e-5 )
assert all(n['n'] == comm['n'] for n in extract.comm)
# restart
assert program.poll()
assert len(program.process) == 0
program.start(comm)
assert len(program.process) == 0
assert program.poll()
finally:
try:
rmtree(dir)
except:
pass
try:
job = root / str(666)
job.functional = Functional(executable, [666])
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
except Fail as r:
assert len(program.errors.keys()) == 1
assert '666' in program.errors
assert len(program._finished) == 8
else:
raise Exception
finally:
try:
rmtree(dir)
except:
pass
try:
job.functional.order = [667]
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
finally:
try:
rmtree(dir)
except:
pass
def test_update(executable):
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.jobfolder import JobFolderProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
assert program.keepalive
# compute current jobs.
program.start(comm)
program.wait()
assert hasattr(program, '_comm')
# compute second set of updated jobs
program.update(supp)
program.wait()
finally:
try:
rmtree(dir)
except:
pass
# check with deleteold=True
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
assert program.keepalive
# compute current jobs.
program.start(comm)
program.wait()
assert hasattr(program, '_comm')
# compute second set of updated jobs
program.update(supp, deleteold=True)
assert hasattr(program, '_comm')
program.wait()
finally:
try:
rmtree(dir)
except:
pass
def test(executable):
""" Tests IteratorProcess. Includes failure modes. """
from tempfile import mkdtemp
from shutil import rmtree
from numpy import all, arange, abs, array
from pylada.process.iterator import IteratorProcess
from pylada.process import Fail, NotStarted
from pylada import default_comm
from functional import Functional
comm = default_comm.copy()
dir = mkdtemp()
try:
functional = Functional(executable, range(8))
program = IteratorProcess(functional, outdir=dir)
# program not started. should fail.
try:
program.poll()
except NotStarted:
pass
else:
raise Exception()
try:
program.wait()
except NotStarted:
pass
else:
raise Exception()
# now starting for real.
program.start(comm)
while not program.poll():
continue
extract = functional.Extract(dir)
assert extract.success
assert all(arange(8) - extract.order == 0)
assert all(abs(extract.pi - array([0.0, 3.2, 3.162353, 3.150849,
3.146801, 3.144926, 3.143907, 3.143293]))\
< 1e-5 )
assert all(abs(extract.error - array([3.141593, 0.05840735, 0.02076029, 0.009256556,
0.005207865, 0.00333321, 0.002314774, 0.001700664]))\
< 1e-5 )
assert all(n['n'] == comm['n'] for n in extract.comm)
# restart
assert program.poll()
program.start(comm)
assert program.process is None
assert program.poll()
# true restart
program = IteratorProcess(functional, outdir=dir)
program.start(comm)
assert program.process is None
assert program.poll()
extract = functional.Extract(dir)
assert extract.success
assert all(arange(8) - extract.order == 0)
assert all(abs(extract.pi - array([0.0, 3.2, 3.162353, 3.150849,
3.146801, 3.144926, 3.143907, 3.143293]))\
< 1e-5 )
assert all(abs(extract.error - array([3.141593, 0.05840735, 0.02076029, 0.009256556,
0.005207865, 0.00333321, 0.002314774, 0.001700664]))\
< 1e-5 )
finally:
try:
rmtree(dir)
except:
pass
try:
functional = Functional(executable, [666])
program = IteratorProcess(functional, outdir=dir)
program.start(comm)
program.wait()
except Fail:
pass
else:
raise Exception
finally:
try:
rmtree(dir)
except:
pass
try:
functional = Functional(executable, [667])
program = IteratorProcess(functional, outdir=dir)
program.start(comm)
program.wait()
finally:
try:
rmtree(dir)
except:
pass
def comm():
from pylada import default_comm
result = default_comm.copy()
result['n'] = 4
return result
def test(executable):
""" Tests JobFolderProcess. Includes failure modes. """
from tempfile import mkdtemp
from os.path import join
from shutil import rmtree
from numpy import all, arange, abs, array
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder.massextract import MassExtract
from pylada.jobfolder import save
from pylada.process.jobfolder import JobFolderProcess
from pylada.process import Fail, AlreadyStarted, NotStarted
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(8):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params["sleep"] = 1
comm = default_comm.copy()
comm["n"] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
# program not started. should fail.
try:
program.poll()
except NotStarted:
pass
else:
raise Exception()
try:
program.wait()
except NotStarted:
pass
else:
raise Exception()
# now starting for real.
program.start(comm)
assert len(program.process) == 2
# Should not be possible to start twice.
try:
program.start(comm)
except AlreadyStarted:
pass
else:
raise Exception()
while not program.poll():
continue
assert program.nbjobsleft == 0
save(root, join(dir, "dict.dict"), overwrite=True)
extract = MassExtract(join(dir, "dict.dict"))
assert all(extract.success.itervalues())
order = array(extract.order.values()).flatten()
assert all(arange(8) - order == 0)
pi = array(extract.pi.values()).flatten()
assert all(abs(pi - array([0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907, 3.143293])) < 1e-5)
error = array(extract.error.values()).flatten()
assert all(
abs(
error
- array(
[3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865, 0.00333321, 0.002314774, 0.001700664]
)
)
< 1e-5
)
assert all(n["n"] == comm["n"] for n in extract.comm)
# restart
assert program.poll()
assert len(program.process) == 0
program.start(comm)
assert len(program.process) == 0
assert program.poll()
finally:
try:
rmtree(dir)
except:
pass
try:
job = root / str(666)
job.functional = Functional(executable, [666])
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
except Fail as r:
assert len(program.errors.keys()) == 1
assert "666" in program.errors
assert len(program._finished) == 8
else:
raise Exception
finally:
try:
rmtree(dir)
except:
pass
try:
job.functional.order = [667]
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
finally:
try:
rmtree(dir)
except:
pass
def test(executable):
""" Tests IteratorProcess. Includes failure modes. """
from tempfile import mkdtemp
from shutil import rmtree
from numpy import all, arange, abs, array
from pylada.process.iterator import IteratorProcess
from pylada.process import Fail, NotStarted
from pylada import default_comm
from functional import Functional
comm = default_comm.copy()
dir = mkdtemp()
try:
functional = Functional(executable, range(8))
program = IteratorProcess(functional, outdir=dir)
# program not started. should fail.
try: program.poll()
except NotStarted: pass
else: raise Exception()
try: program.wait()
except NotStarted: pass
else: raise Exception()
# now starting for real.
program.start(comm)
while not program.poll(): continue
extract = functional.Extract(dir)
assert extract.success
assert all(arange(8) - extract.order == 0)
assert all(abs(extract.pi - array([0.0, 3.2, 3.162353, 3.150849,
3.146801, 3.144926, 3.143907, 3.143293]))\
< 1e-5 )
assert all(abs(extract.error - array([3.141593, 0.05840735, 0.02076029, 0.009256556,
0.005207865, 0.00333321, 0.002314774, 0.001700664]))\
< 1e-5 )
assert all(n['n'] == comm['n'] for n in extract.comm)
# restart
assert program.poll()
program.start(comm)
assert program.process is None
assert program.poll()
# true restart
program = IteratorProcess(functional, outdir=dir)
program.start(comm)
assert program.process is None
assert program.poll()
extract = functional.Extract(dir)
assert extract.success
assert all(arange(8) - extract.order == 0)
assert all(abs(extract.pi - array([0.0, 3.2, 3.162353, 3.150849,
3.146801, 3.144926, 3.143907, 3.143293]))\
< 1e-5 )
assert all(abs(extract.error - array([3.141593, 0.05840735, 0.02076029, 0.009256556,
0.005207865, 0.00333321, 0.002314774, 0.001700664]))\
< 1e-5 )
finally:
try: rmtree(dir)
except: pass
try:
functional = Functional(executable, [666])
program = IteratorProcess(functional, outdir=dir)
program.start(comm)
program.wait()
except Fail: pass
else: raise Exception
finally:
try: rmtree(dir)
except: pass
try:
functional = Functional(executable, [667])
program = IteratorProcess(functional, outdir=dir)
program.start(comm)
program.wait()
finally:
try: rmtree(dir)
except: pass
def test_call():
""" Tests CallProcess. Includes failure modes. """
from tempfile import mkdtemp
from os.path import join, abspath, dirname
from shutil import rmtree
from numpy import all, arange, abs, array
from pylada.process.call import CallProcess
from pylada.process import Fail, NotStarted
from pylada import default_comm
from pylada.process.tests.functional import Functional
from pylada.process.tests.pifunctional import __file__ as executable
executable = join(dirname(executable), "pifunctional.py")
comm = default_comm.copy()
dir = mkdtemp()
try:
functional = Functional(executable, range(8))
program = CallProcess(functional, outdir=dir, dompi=False )
# program not started. should fail.
try: program.poll()
except NotStarted: pass
else: raise Exception()
try: program.wait()
except NotStarted: pass
else: raise Exception()
# now starting for real.
assert program.start(comm) == False
assert program.process is not None
while not program.poll(): continue
extract = functional.Extract(dir)
assert extract.success
assert all(arange(8) - extract.order == 0)
expected = [0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907,
3.143293]
assert all(abs(extract.pi - array(expected)) < 1e-5)
expected = [3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865,
0.00333321, 0.002314774, 0.001700664]
assert all(abs(extract.error -array(expected)) < 1e-5)
assert all(n['n'] == comm['n'] for n in extract.comm)
# restart
assert program.start(comm)
assert program.process is None
# true restart
program = CallProcess(functional, outdir=dir, dompi=False )
assert program.start(comm)
assert program.process is None
assert program.poll()
extract = functional.Extract(dir)
assert extract.success
assert all(arange(8) - extract.order == 0)
expected = [0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907,
3.143293]
assert all(abs(extract.pi - array(expected)) < 1e-5)
expected = [3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865,
0.00333321, 0.002314774, 0.001700664]
assert all(abs(extract.error - array(expected)) < 1e-5)
finally:
try: rmtree(dir)
except: pass
try:
functional = Functional(executable, [50], fail='midway')
program = CallProcess(functional, outdir=dir, stderr=join(dir, 'error'),
dompi=False)
program.start(comm)
program.wait()
except Fail: pass
else: raise Exception
finally:
try: rmtree(dir)
except: pass
try:
functional = Functional(executable, [50])
program = CallProcess(functional, outdir=dir, dompi=False )
program.start(comm)
program.wait()
finally:
try: rmtree(dir)
except: pass
def test_fakeexec():
""" Tests Pool. Includes failure modes. """
from tempfile import mkdtemp
from os.path import join, abspath, dirname
from shutil import rmtree
from numpy import all, arange, abs, array
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder.massextract import MassExtract
from pylada.jobfolder import save
from pylada.process.pool import PoolProcess
from pylada.process import Fail, NotStarted
from pylada import default_comm
from pylada.process.tests.functional import Functional
from pylada.process.tests.pifunctional import __file__ as executable
executable = join(dirname(executable), "pifunctional.py")
root = JobFolder()
for n in xrange(8):
job = root / "a{0}".format(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
save(root, join(dir, 'dict.dict'), overwrite=True)
try:
program = PoolProcess(root, processalloc=processalloc, outdir=dir)
assert program.nbjobsleft > 0
# program not started. should fail.
try: program.poll()
except NotStarted: pass
else: raise Exception()
try: program.wait()
except NotStarted: pass
else: raise Exception()
# now starting for real.
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
extract = MassExtract(join(dir, 'dict.dict'))
assert all(extract.success.itervalues())
order = array(extract.order.values()).flatten()
assert all(arange(8) - order == 0)
pi = array(extract.pi.values()).flatten()
expected = [0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907,
3.143293]
assert all(abs(pi - array(expected)) < 1e-5)
error = array(extract.error.values()).flatten()
expected = [3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865,
0.00333321, 0.002314774, 0.001700664]
assert all(abs(error - array(expected)) < 1e-5)
assert all(n['n'] == comm['n'] for n in extract.comm)
# restart
assert program.poll()
assert len(program.process) == 0
program.start(comm)
assert len(program.process) == 0
assert program.poll()
try:
job = root / str(666)
job.functional = Functional(executable, [50], fail='end')
program = PoolProcess(root, nbpools=2,
outdir=dir, processalloc=processalloc)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert False # Should not reach this point
except Fail:
assert len(program.errors.keys()) == 1
assert '666' in program.errors
else: raise Exception()
job.functional.order = [50]
program = PoolProcess(root, nbpools=2,
outdir=dir, processalloc=processalloc)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
finally:
try: rmtree(dir)
except: pass
def comm():
from pylada import default_comm
result = default_comm.copy()
result['n'] = 4
return result