def test_update_with_fail(executable):
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.jobfolder import JobFolderProcess
from pylada.process import Fail
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
root['1'].functional.order = 666
root['1'].sleep = None
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
supp['5'].sleep = 0
supp['5'].functional.order = 666
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
# compute current jobs.
program.start(comm)
try:
program.wait()
except Fail:
pass
else:
raise Exception()
assert len(program.errors) == 1
# compute second set of updated jobs
program.update(supp)
try:
program.wait()
except Fail:
pass
else:
raise Exception()
assert len(program.errors) == 2
program.errors.clear()
finally:
try:
rmtree(dir)
except:
pass
def test_getjobs(nprocs=8, njobs=20):
""" Test scheduling. """
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.pool import PoolProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(njobs):
job = root / "a{0}".format(n)
job.functional = Functional("whatever", [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = nprocs
def processalloc(job):
""" returns a random number between 1 and 4 included. """
from random import randint
return randint(1, comm['n'])
for j in xrange(100):
program = PoolProcess(root,
processalloc=processalloc,
outdir="whatever")
program._comm = comm
for i in xrange(1000):
jobs = program._getjobs()
assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
(jobs, [program._alloc[u] for u in jobs])
def test_failures(d):
""" Tests whether scheduling jobs works on known failure cases. """
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.pool import PoolProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(8):
job = root / "a{0}".format(n)
job.functional = Functional("whatever", [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
def processalloc_test1(job):
return d[job.name[1:-1]]
program = PoolProcess(root,
processalloc=processalloc_test1,
outdir="whatever")
program._comm = comm
for i in xrange(10000):
jobs = program._getjobs()
assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
(jobs, [program._alloc[u] for u in jobs])
def test_update(executable):
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from os.path import join
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder import save
from pylada.process.pool import PoolProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
save(root, join(dir, 'dict.dict'), overwrite=True)
try:
program = PoolProcess(root,
processalloc=processalloc,
outdir=dir,
keepalive=True)
assert program.nbjobsleft > 0
# compute current jobs.
program.start(comm)
program.wait()
assert hasattr(program, '_comm')
# compute second set of updated jobs
program.update(supp)
program.wait()
finally:
try:
rmtree(dir)
except:
pass
def test(nbprocs, ppn, executable):
from os import getcwd
print 'IN DIRECTORY', getcwd()
from pylada.process.mpi import create_global_comm
from pylada.process.iterator import IteratorProcess
from functional import Functional
import pylada
print 'CREATING GLOBAL COMM'
pylada.default_comm['ppn'] = ppn
pylada.default_comm['n'] = nbprocs
create_global_comm(nbprocs)
print 'CREATING FUNCTIONALS AND PROCESSES'
lfunc = Functional(executable, range(ppn*10, ppn*10+8))
long = IteratorProcess(lfunc, outdir='long')
sfunc = Functional(executable, [10])
short0 = IteratorProcess(sfunc, outdir='short0')
short1 = IteratorProcess(sfunc, outdir='short1')
print 'CREATING COMMUNICATORS'
long_comm = pylada.default_comm.lend(3*(nbprocs//4))
assert len(long_comm.machines) == 2
short_comm0 = pylada.default_comm.lend(pylada.default_comm['n']//2)
assert len(short_comm0.machines) == 1
short_comm1 = pylada.default_comm.lend('all')
assert len(short_comm1.machines) == 1
print 'STARTING LONG PROCESS'
long.start(long_comm)
assert not long.poll()
print 'STARTING SHORT PROCESSES'
short0.start(short_comm0)
short1.start(short_comm1)
print 'TESTING PROCESS OVERLAP'
assert not long.poll()
print 'TESTED PROCESS OVERLAP'
short0.wait()
print 'FIRST SHORT PROCESS FINISHED'
assert not long.poll()
print 'TESTED PROCESS OVERLAP'
short1.wait()
print 'SECOND SHORT PROCESS FINISHED'
assert not long.poll()
print 'TESTED PROCESS OVERLAP'
long.wait()
print 'LONG PROCESS FINISHED'
assert lfunc.Extract('long').success
assert sfunc.Extract('short0').success
assert sfunc.Extract('short1').success
print 'END'
def test(executable):
""" Tests JobFolderProcess. Includes failure modes. """
from tempfile import mkdtemp
from os.path import join
from shutil import rmtree
from numpy import all, arange, abs, array
from pylada.jobfolder.jobfolder import JobFolder
from pylada.jobfolder.massextract import MassExtract
from pylada.jobfolder import save
from pylada.process.jobfolder import JobFolderProcess
from pylada.process import Fail, AlreadyStarted, NotStarted
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(8):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
# program not started. should fail.
try:
program.poll()
except NotStarted:
pass
else:
raise Exception()
try:
program.wait()
except NotStarted:
pass
else:
raise Exception()
# now starting for real.
program.start(comm)
assert len(program.process) == 2
# Should not be possible to start twice.
try:
program.start(comm)
except AlreadyStarted:
pass
else:
raise Exception()
while not program.poll():
continue
assert program.nbjobsleft == 0
save(root, join(dir, 'dict.dict'), overwrite=True)
extract = MassExtract(join(dir, 'dict.dict'))
assert all(extract.success.itervalues())
order = array(extract.order.values()).flatten()
assert all(arange(8) - order == 0)
pi = array(extract.pi.values()).flatten()
assert all(abs(pi - array([0.0, 3.2, 3.162353, 3.150849,
3.146801, 3.144926, 3.143907, 3.143293]))\
< 1e-5 )
error = array(extract.error.values()).flatten()
assert all(abs(error - array([3.141593, 0.05840735, 0.02076029, 0.009256556,
0.005207865, 0.00333321, 0.002314774, 0.001700664]))\
< 1e-5 )
assert all(n['n'] == comm['n'] for n in extract.comm)
# restart
assert program.poll()
assert len(program.process) == 0
program.start(comm)
assert len(program.process) == 0
assert program.poll()
finally:
try:
rmtree(dir)
except:
pass
try:
job = root / str(666)
job.functional = Functional(executable, [666])
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
except Fail as r:
assert len(program.errors.keys()) == 1
assert '666' in program.errors
assert len(program._finished) == 8
else:
raise Exception
finally:
try:
rmtree(dir)
except:
pass
try:
job.functional.order = [667]
program = JobFolderProcess(root, nbpools=2, outdir=dir)
assert program.nbjobsleft > 0
program.start(comm)
program.wait()
assert program.nbjobsleft == 0
finally:
try:
rmtree(dir)
except:
pass
def test_update(executable):
""" Tests JobFolderProcess with update. """
from tempfile import mkdtemp
from shutil import rmtree
from pylada.jobfolder.jobfolder import JobFolder
from pylada.process.jobfolder import JobFolderProcess
from pylada import default_comm
from functional import Functional
root = JobFolder()
for n in xrange(3):
job = root / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
supp = JobFolder()
for n in xrange(3, 6):
job = supp / str(n)
job.functional = Functional(executable, [n])
job.params['sleep'] = 1
comm = default_comm.copy()
comm['n'] = 4
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
assert program.keepalive
# compute current jobs.
program.start(comm)
program.wait()
assert hasattr(program, '_comm')
# compute second set of updated jobs
program.update(supp)
program.wait()
finally:
try:
rmtree(dir)
except:
pass
# check with deleteold=True
dir = mkdtemp()
try:
program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
assert program.keepalive
# compute current jobs.
program.start(comm)
program.wait()
assert hasattr(program, '_comm')
# compute second set of updated jobs
program.update(supp, deleteold=True)
assert hasattr(program, '_comm')
program.wait()
finally:
try:
rmtree(dir)
except:
pass
def main():
root = tk.Tk()
app = Application()
main = app.Main(root, 540, 380, 'String calculator', False)
input_ = app.FieldEntry(root, 65, 35, 300)
output = app.FieldLabel(root, 70, 70, 'Result: ')
f = Functional()
#/////////////////////////create buttons////////////////////////////////////////////////
#/////////////////////////////digits///////////////////////////////////////////////////////
btns = app.Buttons(root)
commands = lambda text: (input_.add_text(text), btns.turn_on_buttons(btn_del, btn_clr, btn_calc))
btn_7 = btns.create_button(text_button='7', x=50, y=100, width_button=1, cms=commands, params=('7',))
btn_8 = btns.create_button(text_button='8', x=90, y=100, width_button=1, cms=commands, params=('8',))
btn_9 = btns.create_button(text_button='9', x=130, y=100, width_button=1, cms=commands, params=('9',))
btn_4 = btns.create_button(text_button='4', x=50, y=135, width_button=1, cms=commands, params=('4',))
btn_5 = btns.create_button(text_button='5', x=90, y=135, width_button=1, cms=commands, params=('5',))
btn_6 = btns.create_button(text_button='6', x=130, y=135, width_button=1, cms=commands, params=('6',))
btn_1 =btns.create_button(text_button='1', x=50, y=170, width_button=1, cms=commands, params=('1',))
btn_2 = btns.create_button(text_button='2', x=90, y=170, width_button=1, cms=commands, params=('2',))
btn_3 = btns.create_button(text_button='3', x=130, y=170, width_button=1, cms=commands, params=('3',))
btn_0 = btns.create_button(text_button='0', x=50, y=205, width_button=1, cms=commands, params=('0',))
btn_i = btns.create_button(text_button='i', x=50, y=240, width_button=1, cms=commands, params=('i',))
btn_pi = btns.create_button(text_button='pi', x=90, y=240, width_button=1, cms=commands, params=('pi',))
btn_e = btns.create_button(text_button='e', x=130, y=240, width_button=1, cms=commands, params=('e',))
#////////////////////////////signs/////////////////////////////////////////////////////////
btn_plus = btns.create_button(text_button='+', x=210, y=100, width_button=1, cms=commands, params=('+',))
btn_minus = btns.create_button(text_button='-', x=250, y=100, width_button=1, cms=commands, params=('-',))
btn_mul = btns.create_button(text_button='*', x=210, y=135, width_button=1, cms=commands, params=('*',))
btn_div = btns.create_button(text_button='/', x=250, y=135, width_button=1, cms=commands, params=('/',))
btn_bracket_left = btns.create_button(text_button='(', x=210, y=170, width_button=1, cms=commands, params=('(',))
btn_bracket_right = btns.create_button(text_button=')', x=250, y=170, width_button=1, cms=commands, params=(')',))
btn_sqrt = btns.create_button(text_button='\u221A', x=210, y=205, width_button=1, cms=commands, params=('sqrt(',))
btn_pow = btns.create_button(text_button='^', x=250, y=205, width_button=1, cms=commands, params=('^',))
btn_comma = btns.create_button(text_button=',', x=90, y=205, width_button=1, cms=commands, params=(',',))
btn_fac = btns.create_button(text_button='!', x=210, y=240, width_button=1, cms=commands, params=('!',))
#///////////////////////////////functions/////////////////////////////////////////////////////
btn_log2 = btns.create_button(text_button='log2(x)', x=320, y=100, cms=commands, params=('log2(',))
btn_log10 = btns.create_button(text_button='log10(x)', x=320, y=140, cms=commands, params=('log10(',))
btn_ln = btns.create_button(text_button='ln(x)', x=320, y=180, cms=commands, params=('ln(',))
btn_exp = btns.create_button(text_button='exp(x)', x=390, y=100, cms=commands, params=('exp(',))
btn_sin = btns.create_button(text_button='sin(x)', x=390, y=140, cms=commands, params=('sin(',))
btn_cos = btns.create_button(text_button='cos(x)', x=390, y=180, cms=commands, params=('cos(',))
btn_tan = btns.create_button(text_button='tan(x)', x=390, y=220, cms=commands, params=('tan(',))
#////////////////////////////////////////////////others////////////////////////////////////////////////////////////////////
commands_calc = lambda : (f.calculate(input_.get_text(), output.set_text), btns.turn_on_buttons(btn_ans, btn_clr, btn_del))
btn_calc = btns.create_button(text_button='Calculate', x=375, y=55, width_button=7, enabled=tk.DISABLED, cms=commands_calc)
commands_del = lambda : input_.step_back() if len(input_.get_text()) > 1 else (input_.step_back(), btns.turn_off_buttons(btn_del, btn_clr, btn_calc))
btn_del = btns.create_button(text_button='Delete', x=375, y=5, width_button=7, enabled=tk.DISABLED, cms=commands_del)
commands_clr = lambda : (input_.clear_input(), btns.turn_off_buttons(btn_del, btn_clr, btn_calc))
btn_clr = btns.create_button(text_button='Clear', x=375, y=30, width_button=7, enabled=tk.DISABLED, cms=commands_clr)
commands_ans = lambda : (input_.add_text(f.ans), btns.turn_on_buttons(btn_del))
btn_ans = btns.create_button(text_button='Ans', x=130, y=205, width_button=1, enabled=tk.DISABLED, cms=commands_ans)
#////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
input_command = lambda : btns.turn_on_buttons(btn_clr, btn_del, btn_calc)
input_.bind_commands(input_command)
main_commands = commands_calc
main.bind_commands(main_commands, 'Return')
signs = ['+', '-', '*', '/', '^', '(', ')', '!', ',', '.', 'e', 'x', 'p',
'l', 'o', 'n', 'g', 's', 'q', 'r', 't', 'i', 'c', 'e', 'a']
signs.extend([str(i) for i in range(10)])
input_.set_legal_symbols(*signs)
root.mainloop()
def test(executable):
""" Tests IteratorProcess. Includes failure modes. """
from tempfile import mkdtemp
from shutil import rmtree
from numpy import all, arange, abs, array
from pylada.process.iterator import IteratorProcess
from pylada.process import Fail, NotStarted
from pylada import default_comm
from functional import Functional
comm = default_comm.copy()
dir = mkdtemp()
try:
functional = Functional(executable, range(8))
program = IteratorProcess(functional, outdir=dir)
# program not started. should fail.
try:
program.poll()
except NotStarted:
pass
else:
raise Exception()
try:
program.wait()
except NotStarted:
pass
else:
raise Exception()
# now starting for real.
program.start(comm)
while not program.poll():
continue
extract = functional.Extract(dir)
assert extract.success
assert all(arange(8) - extract.order == 0)
assert all(abs(extract.pi - array([0.0, 3.2, 3.162353, 3.150849,
3.146801, 3.144926, 3.143907, 3.143293]))\
< 1e-5 )
assert all(abs(extract.error - array([3.141593, 0.05840735, 0.02076029, 0.009256556,
0.005207865, 0.00333321, 0.002314774, 0.001700664]))\
< 1e-5 )
assert all(n['n'] == comm['n'] for n in extract.comm)
# restart
assert program.poll()
program.start(comm)
assert program.process is None
assert program.poll()
# true restart
program = IteratorProcess(functional, outdir=dir)
program.start(comm)
assert program.process is None
assert program.poll()
extract = functional.Extract(dir)
assert extract.success
assert all(arange(8) - extract.order == 0)
assert all(abs(extract.pi - array([0.0, 3.2, 3.162353, 3.150849,
3.146801, 3.144926, 3.143907, 3.143293]))\
< 1e-5 )
assert all(abs(extract.error - array([3.141593, 0.05840735, 0.02076029, 0.009256556,
0.005207865, 0.00333321, 0.002314774, 0.001700664]))\
< 1e-5 )
finally:
try:
rmtree(dir)
except:
pass
try:
functional = Functional(executable, [666])
program = IteratorProcess(functional, outdir=dir)
program.start(comm)
program.wait()
except Fail:
pass
else:
raise Exception
finally:
try:
rmtree(dir)
except:
pass
try:
functional = Functional(executable, [667])
program = IteratorProcess(functional, outdir=dir)
program.start(comm)
program.wait()
finally:
try:
rmtree(dir)
except:
pass
#!/usr/bin/python from functional import Functional,side_effect,lift_n_resolve_fs,curry from maybe import Maybe from data import DATA_Num, DATA_Str, DATA_List from my_funcs import * DATA_1 = Maybe(DATA_Num) print '--------traditional--------' run_f1 = lift_n_resolve_fs(*[sq,rt,sq,cube]) run_f2 = lift_n_resolve_fs(*[sq,rt,sq,rt_m]) print run_f1 (DATA_1) print run_f2 (DATA_1) print '---------------------------' print '--------functional---------' print Functional()(sq)(DATA_1) print Functional() >> sq >> cube >> sq >> DATA_1 print Functional() >> curry(lambda x,y:x+y, 5) >> sq >> DATA_1 print Functional(False)(rt_m)(rt_m)(rt_m)(rt_m)(Maybe(-4)) print Functional(False)(rt_m)(DATA_1) print Functional().unlift_all()(rt_m)(DATA_1) print '---------------------------' print '--------functional-break---' print Functional()(cube)(rt)(cube)(DATA_1) print '---------------------------' print '--------functional-generic-' print Functional()(cube)(lambda x:x*x)(lambda x:x-1)(DATA_1) print '---------------------------'