def test_failures():
""" Tests whether scheduling jobs works on known failure cases. """
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.pool import PoolProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(8):
job = root / "a{0}".format(n)
job.functional = Functional("whatever", [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
d = {'a1': 1, 'a0': 3, 'a3': 3, 'a2': 3, 'a5': 3, 'a4': 2, 'a7': 2, 'a6': 1}
def processalloc_test1(job): return d[job.name[1:-1]]
program = PoolProcess(root, processalloc=processalloc_test1,
outdir="whatever")
program._comm = comm
for i in xrange(10000):
jobs = program._getjobs()
assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
(jobs, [program._alloc[u] for u in jobs])
def test_getjobs(nprocs=8, njobs=20):
""" Test scheduling. """
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.pool import PoolProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(njobs):
job = root / "a{0}".format(n)
job.functional = Functional("whatever", [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = nprocs
def processalloc(job):
""" returns a random number between 1 and 4 included. """
from random import randint
return randint(1, comm['n'])
for j in xrange(100):
program = PoolProcess(root, processalloc=processalloc, outdir="whatever")
program._comm = comm
for i in xrange(1000):
jobs = program._getjobs()
assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
(jobs, [program._alloc[u] for u in jobs])
def test_getjobs(nprocs=8, njobs=20):
""" Test scheduling. """
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.pool import PoolProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(njobs):
job = root / "a{0}".format(n)
job.functional = Functional("whatever", [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = nprocs
def processalloc(job):
""" returns a random number between 1 and 4 included. """
from random import randint
return randint(1, comm['n'])
for j in xrange(100):
program = PoolProcess(root,
processalloc=processalloc,
outdir="whatever")
program._comm = comm
for i in xrange(1000):
jobs = program._getjobs()
assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
(jobs, [program._alloc[u] for u in jobs])
def test_failures(d):
""" Tests whether scheduling jobs works on known failure cases. """
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.pool import PoolProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(8):
job = root / "a{0}".format(n)
job.functional = Functional("whatever", [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
def processalloc_test1(job):
return d[job.name[1:-1]]
program = PoolProcess(root,
processalloc=processalloc_test1,
outdir="whatever")
program._comm = comm
for i in xrange(10000):
jobs = program._getjobs()
assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
(jobs, [program._alloc[u] for u in jobs])
def test_update_with_fail():
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from os.path import join, abspath, dirname
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder import save
from pylada.process.pool import PoolProcess
from pylada.process import Fail
from pylada import default_comm
from pylada.process.tests.functional import Functional
from pylada.process.tests.pifunctional import __file__ as executable
executable = join(dirname(executable), "pifunctional.py")
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
root['1'].functional.order = 50
root['1'].functional.fail = 'midway'
root['1'].sleep = None
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
supp['5'].sleep = 0
supp['5'].functional.order = 34
supp['5'].functional.fail = 'end'
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
save(root, join(dir, 'dict.dict'), overwrite=True)
try:
program = PoolProcess(root, processalloc=processalloc, outdir=dir, keepalive=True)
assert program.nbjobsleft > 0
# compute current jobs.
program.start(comm)
try: program.wait()
except Fail: pass
else:
print program.errors
raise Exception()
assert hasattr(program, '_comm')
# compute second set of updated jobs
program.update(supp)
try: program.wait()
except Fail: pass
else: raise Exception()
finally:
try: rmtree(dir)
except: pass
def test_large():
""" Test speed of job-scheduling for largers, more numerous jobs. """
from random import random
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.pool import PoolProcess
from pylada.process.dummy import DummyFunctional
from pylada import default_comm
root = JobFolder()
for n in xrange(100):
job = root / "a{0}".format(n)
job.functional = DummyFunctional(chance=random()*0.5+0.15)
comm = default_comm.copy()
comm['n'] = 256
def processalloc_test1(job):
""" returns a random number between 1 and 4 included. """
from random import randint
return randint(1, 64)
program = PoolProcess(root, processalloc=processalloc_test1, outdir="whatever")
program.start(comm)
while not program.poll():
# print 256 - program._comm['n'], len(program.process)
continue
def processalloc_test1(job):
""" returns a random number between 1 and 4 included. """
from random import choice
return choice([31, 37, 43])
program = PoolProcess(root, processalloc=processalloc_test1, outdir="whatever")
program.start(comm)
while not program.poll():
# print 256 - program._comm['n'], len(program.process)
continue
def test_getjobs(comm, tmpdir, executable, nprocs, njobs):
""" Test scheduling. """
root = jobfolders(executable, 0, 8)
def processalloc(job):
""" returns a random number between 1 and 4 included. """
from random import randint
return randint(1, comm['n'])
for j in range(100):
program = PoolProcess(root, processalloc=processalloc, outdir=str(tmpdir))
program._comm = comm
for i in range(1000):
jobs = program._getjobs()
assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
(jobs, [program._alloc[u] for u in jobs])
def test_update(executable):
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from os.path import join
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder import save
from pylada.process.pool import PoolProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
save(root, join(dir, 'dict.dict'), overwrite=True)
try:
program = PoolProcess(root,
processalloc=processalloc,
outdir=dir,
keepalive=True)
assert program.nbjobsleft > 0
# compute current jobs.
program.start(comm)
program.wait()
assert hasattr(program, '_comm')
# compute second set of updated jobs
program.update(supp)
program.wait()
finally:
try:
rmtree(dir)
except:
pass
def test_failures(tmpdir, executable, comm):
"""Tests whether scheduling jobs works on known failure cases."""
from pylada import default_comm
root = jobfolders(executable, 0, 8)
def processalloc_test1(job):
d = {'1': 1, '0': 3, '3': 3, '2': 3, '5': 3, '4': 2, '7': 2, '6': 1}
return d[job.name[1:-1]]
program = PoolProcess(root,
processalloc=processalloc_test1,
outdir=str(tmpdir))
program._comm = comm
for i in range(10000):
jobs = program._getjobs()
assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
(jobs, [program._alloc[u] for u in jobs])
def test_failures(tmpdir, executable, comm):
""" Tests whether scheduling jobs works on known failure cases. """
from pylada import default_comm
from pylada.process.tests.functional import Functional
root = jobfolders(executable, 0, 8)
def processalloc_test1(job):
d = {'1': 1, '0': 3, '3': 3, '2': 3, '5': 3, '4': 2, '7': 2, '6': 1}
return d[job.name[1:-1]]
program = PoolProcess(root, processalloc=processalloc_test1,
outdir=str(tmpdir))
program._comm = comm
for i in range(10000):
jobs = program._getjobs()
assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
(jobs, [program._alloc[u] for u in jobs])
def test_getjobs(comm, tmpdir, executable, nprocs, njobs):
"""Test scheduling."""
root = jobfolders(executable, 0, 8)
def processalloc(job):
"""returns a random number between 1 and 4 included."""
from random import randint
return randint(1, comm['n'])
for j in range(100):
program = PoolProcess(root,
processalloc=processalloc,
outdir=str(tmpdir))
program._comm = comm
for i in range(1000):
jobs = program._getjobs()
assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
(jobs, [program._alloc[u] for u in jobs])
def test_large():
""" Test speed of job-scheduling for largers, more numerous jobs. """
from random import random
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.pool import PoolProcess
from pylada.process.dummy import DummyFunctional
from pylada import default_comm
root = JobFolder()
for n in xrange(100):
job = root / "a{0}".format(n)
job.functional = DummyFunctional(chance=random() * 0.5 + 0.15)
comm = default_comm.copy()
comm['n'] = 256
def processalloc_test1(job):
""" returns a random number between 1 and 4 included. """
from random import randint
return randint(1, 64)
program = PoolProcess(root,
processalloc=processalloc_test1,
outdir="whatever")
program.start(comm)
while not program.poll():
# print 256 - program._comm['n'], len(program.process)
continue
def processalloc_test1(job):
""" returns a random number between 1 and 4 included. """
from random import choice
return choice([31, 37, 43])
program = PoolProcess(root,
processalloc=processalloc_test1,
outdir="whatever")
program.start(comm)
while not program.poll():
# print 256 - program._comm['n'], len(program.process)
continue
def test_update(executable):
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from os.path import join
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder import save
from pylada.process.pool import PoolProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
save(root, join(dir, 'dict.dict'), overwrite=True)
try:
program = PoolProcess(root, processalloc=processalloc, outdir=dir, keepalive=True)
assert program.nbjobsleft > 0
# compute current jobs.
program.start(comm)
program.wait()
assert hasattr(program, '_comm')
# compute second set of updated jobs
program.update(supp)
program.wait()
finally:
try: rmtree(dir)
except: pass
def test_fakeexec():
""" Tests Pool. Includes failure modes. """
from tempfile import mkdtemp
from os.path import join, abspath, dirname
from shutil import rmtree
from numpy import all, arange, abs, array
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder.massextract import MassExtract
from pylada.jobfolder import save
from pylada.process.pool import PoolProcess
from pylada.process import Fail, NotStarted
from pylada import default_comm
from pylada.process.tests.functional import Functional
from pylada.process.tests.pifunctional import __file__ as executable
executable = join(dirname(executable), "pifunctional.py")
root = JobFolder()
for n in xrange(8):
job = root / "a{0}".format(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
save(root, join(dir, 'dict.dict'), overwrite=True)
try:
program = PoolProcess(root, processalloc=processalloc, outdir=dir)
assert program.nbjobsleft > 0
# program not started. should fail.
try: program.poll()
except NotStarted: pass
else: raise Exception()
try: program.wait()
except NotStarted: pass
else: raise Exception()
# now starting for real.
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
extract = MassExtract(join(dir, 'dict.dict'))
assert all(extract.success.itervalues())
order = array(extract.order.values()).flatten()
assert all(arange(8) - order == 0)
pi = array(extract.pi.values()).flatten()
expected = [0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907,
3.143293]
assert all(abs(pi - array(expected)) < 1e-5)
error = array(extract.error.values()).flatten()
expected = [3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865,
0.00333321, 0.002314774, 0.001700664]
assert all(abs(error - array(expected)) < 1e-5)
assert all(n['n'] == comm['n'] for n in extract.comm)
# restart
assert program.poll()
assert len(program.process) == 0
program.start(comm)
assert len(program.process) == 0
assert program.poll()
try:
job = root / str(666)
job.functional = Functional(executable, [50], fail='end')
program = PoolProcess(root, nbpools=2,
outdir=dir, processalloc=processalloc)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert False # Should not reach this point
except Fail:
assert len(program.errors.keys()) == 1
assert '666' in program.errors
else: raise Exception()
job.functional.order = [50]
program = PoolProcess(root, nbpools=2,
outdir=dir, processalloc=processalloc)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
finally:
try: rmtree(dir)
except: pass
def test(executable):
""" Tests JobFolderProcess. Includes failure modes. """
from tempfile import mkdtemp
from os.path import join
from shutil import rmtree
from numpy import all, arange, abs, array
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder.massextract import MassExtract
from pylada.jobfolder import save
from pylada.process.pool import PoolProcess
from pylada.process import Fail, NotStarted
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(8):
job = root / "a{0}".format(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
save(root, join(dir, 'dict.dict'), overwrite=True)
try:
program = PoolProcess(root, processalloc=processalloc, outdir=dir)
assert program.nbjobsleft > 0
# program not started. should fail.
try:
program.poll()
except NotStarted:
pass
else:
raise Exception()
try:
program.wait()
except NotStarted:
pass
else:
raise Exception()
# now starting for real.
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
extract = MassExtract(join(dir, 'dict.dict'))
assert all(extract.success.itervalues())
order = array(extract.order.values()).flatten()
assert all(arange(8) - order == 0)
pi = array(extract.pi.values()).flatten()
assert all(abs(pi - array([0.0, 3.2, 3.162353, 3.150849,
3.146801, 3.144926, 3.143907, 3.143293]))\
< 1e-5 )
error = array(extract.error.values()).flatten()
assert all(abs(error - array([3.141593, 0.05840735, 0.02076029, 0.009256556,
0.005207865, 0.00333321, 0.002314774, 0.001700664]))\
< 1e-5 )
assert all(n['n'] == comm['n'] for n in extract.comm)
# restart
assert program.poll()
assert len(program.process) == 0
program.start(comm)
assert len(program.process) == 0
assert program.poll()
finally:
try:
rmtree(dir)
except:
pass
return
try:
job = root / str(666)
job.functional = Functional(executable, [666])
program = PoolProcess(root,
nbpools=2,
outdir=dir,
processalloc=processalloc)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
except Fail:
assert len(program.errors.keys()) == 1
assert '666' in program.errors
else:
raise Exception
finally:
try:
rmtree(dir)
except:
pass
try:
job.functional.order = [667]
program = PoolProcess(root,
nbpools=2,
outdir=dir,
processalloc=processalloc)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
finally:
try:
rmtree(dir)
except:
pass
def pool_process(tmpdir, root, pal=processalloc, **kwargs):
from pylada.process.pool import PoolProcess
return PoolProcess(root, processalloc=pal, outdir=str(tmpdir), **kwargs)