def dataProvider_testBuildProps(self):
testdir = self.testdir
# 0
pBuildProps = Proc()
pBuildProps.ppldir = testdir
yield pBuildProps, {}, {}, ProcTreeProcExists, 'There are two processes with id\(pBuildProps\) and tag\(notag\)'
# 1
pBuildProps1 = Proc(id = 'pBuildProps')
pBuildProps1.ppldir = testdir
yield pBuildProps1, {}, {}, ProcTreeProcExists, 'There are two processes with id\(pBuildProps\) and tag\(notag\)'
# 2-9
pBuildProps2 = Proc()
pBuildProps2.ppldir = testdir
yield pBuildProps2, {}, {'template': TemplatePyPPL}
yield pBuildProps2, {'template': ''}, {'template': TemplatePyPPL}
if helpers.moduleInstalled('jinja2'):
yield pBuildProps2, {'template': TemplateJinja2}, {'template': TemplateJinja2}
yield pBuildProps2, {'template': 'jinja2'}, {'template': TemplateJinja2}
yield pBuildProps2, {'rc': ' 0, 1, '}, {'rc': [0,1]}
yield pBuildProps2, {'rc': 2}, {'rc': [2]}
yield pBuildProps2, {'rc': [0, 1]}, {'rc': [0, 1]}
yield pBuildProps2, {'workdir': path.join(testdir, 'pBuildProps2')}, {'workdir': path.join(testdir, 'pBuildProps2')}
# 10
pBuildProps3 = Proc()
pBuildProps3.ppldir = testdir
yield pBuildProps3, {}, {'workdir': path.join(testdir, 'PyPPL.pBuildProps3.notag.%s' % pBuildProps3._suffix())}, None, None, [lambda p: path.isdir(p.workdir)]
# 11
pBuildProps4 = Proc()
pBuildProps4.ppldir = testdir
yield pBuildProps4, {'resume': 'skip+'}, {}, ProcAttributeError, 'Cannot skip process, as workdir not exists'
# 12
pBuildProps5 = Proc()
pBuildProps5.ppldir = testdir
# exdir
yield pBuildProps5, {'exdir': path.relpath(path.join(testdir, 'exports'))}, {'exdir': path.abspath(path.join(testdir, 'exports'))}, None, None, [lambda p: path.isdir(p.exdir)]
# echo
# 13-20
yield pBuildProps5, {'echo': True}, {'echo': {'jobs': [0], 'type': {'stderr': None, 'stdout': None}}}
yield pBuildProps5, {'echo': False}, {'echo': {'jobs': [], 'type': {'stderr': None, 'stdout': None}}}
yield pBuildProps5, {'echo': 'stderr'}, {'echo': {'jobs': [0], 'type': {'stderr': None}}}
yield pBuildProps5, {'echo': 'stdout'}, {'echo': {'jobs': [0], 'type': {'stdout': None}}}
yield pBuildProps5, {'echo': {'type': 'all'}}, {'echo': {'jobs': [0], 'type': {'stderr': None, 'stdout': None}}}
yield pBuildProps5, {'echo': {'jobs': ' 0, 1 ', 'type': 'all'}}, {'echo': {'jobs': [0, 1], 'type': {'stderr': None, 'stdout': None}}}
yield pBuildProps5, {'echo': {'jobs': range(2), 'type': 'all'}}, {'echo': {'jobs': [0, 1], 'type': {'stderr': None, 'stdout': None}}}
yield pBuildProps5, {'echo': {'type': 'stderr'}}, {'echo': {'jobs': [0], 'type': {'stderr': None}}}
yield pBuildProps5, {'echo': {'type': {'all': r'^a'}}}, {'echo': {'jobs': [0], 'type': {'stderr': r'^a', 'stdout': r'^a'}}}
# expect
# 21
yield pBuildProps5, {'expect': 'expect template'}, {}, None, None, [lambda p: p.expect.source == 'expect template']
# expart
# 22
yield pBuildProps5, {'expart': 'a,b,c,d,"e,f"'}, {}, None, None, [
(lambda p: p.expart[i].source == v) for i,v in enumerate(['a', 'b', 'c', 'd', '"e,f"'])
]
def dataProvider_testLoadFile(self):
yield self.testdir, False, []
jsonfile = path.join(self.testdir, 'testLoadFile.json')
helpers.writeFile(
jsonfile, '\n'.join([
'{',
' "a": "2",',
' "a.desc": "Option a",',
' "a.type": "int",',
' "a.required": true',
'}',
]))
p1 = Parameter('a', 2)
p1.desc = "Option a"
p1.required = True
yield jsonfile, True, [p1]
if helpers.moduleInstalled('yaml'):
yamlfile = path.join(self.testdir, 'testLoadFile.yaml')
helpers.writeFile(
yamlfile, '\n'.join([
'a: 2',
'a.desc: Option a',
'a.type: int',
'a.required: false',
'a.show: true',
'',
]))
p2 = Parameter('a', 2)
p2.desc = "Option a"
p2.required = False
p2.show = True
yield yamlfile, False, [p2]
conffile = path.join(self.testdir, 'testLoadFile.conf')
helpers.writeFile(
conffile, '\n'.join([
'[PARAM1]',
'a = 2',
'a.desc = Option a',
'a.type = int',
'a.required = f',
'[PARAM2]',
'a.type = str',
'b:',
' 1',
' 2',
'b.type = list',
]))
p3 = Parameter('a', '2')
p3.desc = "Option a"
p3.required = False
p4 = Parameter('b', ['1', '2'])
yield conffile, True, [p3, p4]
def testNoYaml(self, testdir):
PyPPL.DEFAULT_CFGFILES = []
ymlfile = path.join(testdir, 'config.yaml')
helpers.writeFile(ymlfile, ['default:', ' forks: 10'])
import sys
if helpers.moduleInstalled('yaml'):
import yaml
del sys.modules['yaml']
paths = []
paths.extend(sys.path)
del sys.path[:]
#while sys.path:
# paths.append(sys.path.pop(0))
with helpers.log2str(levels='all') as (out, err):
pp = PyPPL(config={'_log': {'file': True}}, cfgfile=ymlfile)
#for p in paths: sys.path.append(p)
sys.path = paths
self.assertDictEqual(pp.config, {})
logfiles = glob(path.splitext(sys.argv[0])[0] + "*.pyppl.log")
self.assertTrue(logfiles)
for logfile in logfiles:
remove(logfile)
import testly, sys, helpers
if helpers.moduleInstalled('graphviz'):
from os import path, makedirs
from shutil import rmtree
from tempfile import gettempdir
from pyppl import Proc
from pyppl.flowchart import Flowchart
from graphviz import Digraph
class TestFlowchart(testly.TestCase):
def setUpMeta(self):
self.testdir = path.join(gettempdir(), 'PyPPL_unittest',
'TestFlowchart')
if path.exists(self.testdir):
rmtree(self.testdir)
makedirs(self.testdir)
def dataProvider_testInit(self):
dotfile = path.join(self.testdir, 'test.dot')
svgfile = path.join(self.testdir, 'test.svg')
yield dotfile, svgfile
yield 'x', 'y'
yield 'x', svgfile
yield dotfile, 'y'
def testInit(self, fcfile, dotfile):
fc = Flowchart(fcfile, dotfile)
self.assertEqual(fc.fcfile, fcfile)
self.assertEqual(fc.dotfile, dotfile)
self.assertIsInstance(fc, Flowchart)
def dataProvider_testSaveSettings(self):
pSaveSettings = Proc()
pSaveSettings.ppldir = self.testdir
yield pSaveSettings, [
# '[brings]',
'[channel]',
'value: []',
'[depends]',
'procs: []',
'[echo]',
'jobs: []',
'type: {"stderr": null, "stdout": null}',
'[expart]',
'value_0: TemplateLiquid < >',
'[expect]',
'value: TemplateLiquid < >',
'[input]',
'[output]',
'[procvars]',
'args: {}',
'proc: {',
'[rc]',
'value: [0]',
'[runner]',
'value: local',
'[script]',
'value:',
' "TemplateLiquid < #!/usr/bin/env bash >"',
'[sets]',
'value: [\'ppldir\']',
'[size]',
'value: 0',
'[suffix]',
'value: ',
'[template]',
'name: TemplateLiquid',
'[workdir]',
'value: ',
]
pSaveSettings1 = Proc()
pSaveSettings1.ppldir = self.testdir
infile1 = path.join(self.testdir, 'pSaveSettings1-in1.txt')
infile2 = path.join(self.testdir, 'pSaveSettings1-in2.txt')
brfile1 = path.join(self.testdir, 'pSaveSettings1-in1.br')
brfile2 = path.join(self.testdir, 'pSaveSettings1-in2.br')
helpers.writeFile(infile1)
helpers.writeFile(infile2)
helpers.writeFile(brfile1)
helpers.writeFile(brfile2)
pSaveSettings1.input = {
'a': 1,
'b:file': [infile1, infile2],
'c:files': [[infile1, infile2]]
}
#pSaveSettings1.brings = {'b': '{{fn(i.b)}}.br'}
pSaveSettings1.output = 'out:file:{{fn(i.b)}}-{{i.a}}.out'
pSaveSettings1.echo = {'jobs': [0, 1]}
pSaveSettings1.expart = '*-1.out'
pSaveSettings1.expect = 'grep 1 {{o.out}}'
pSaveSettings1.args.a = 'a'
pSaveSettings1.rc = '0,1'
pSaveSettings1.script = 'echo {{i.a}} > {{o.out}}'
pSaveSettings1.template = 'jinja2'
if helpers.moduleInstalled('jinja2'):
yield pSaveSettings1, [
#'[brings]',
#'b: [\'TemplateJinja2 < {{fn(i.b)}}.br >\']',
'[channel]',
'value: []',
'[depends]',
'procs: []',
'[echo]',
'jobs: [0, 1]',
'type: {"stderr": null, "stdout": null}',
'[expart]',
'value_0: TemplateJinja2 < *-1.out >',
'[expect]',
'value: TemplateJinja2 < grep 1 {{o.out}} >',
'[input]',
'a.type: var',
'a.data#0',
' 1',
'a.data#1',
' 1',
'b.type: file',
'b.data#0',
'pSaveSettings1-in1.txt',
'b.data#1',
'pSaveSettings1-in2.txt',
'[output]',
'out.type: file',
'out.data: TemplateJinja2 < {{fn(i.b)}}-{{i.a}}.out >',
'[procvars]',
'args: {"a": "a"}',
'proc: {',
'[rc]',
'value: [0, 1]',
'[runner]',
'value: local',
'[script]',
'value:',
' "TemplateJinja2 <<<"',
' "\\t#!/usr/bin/env bash"',
' "\\techo {{i.a}} > {{o.out}}"',
' ">>>"',
'[sets]',
'value: [\'ppldir\', \'input\', \'output\', \'echo\', \'expart\', \'expect\', \'rc\', \'script\', \'template\']',
'[size]',
'value: 2',
'[suffix]',
'value: ',
'[template]',
'name: TemplateJinja2',
'[workdir]',
'value: ',
]
import helpers, testly, json, sys
import copy as pycopy
from os import path, makedirs
from shutil import rmtree
from tempfile import gettempdir
from collections import OrderedDict
from multiprocessing import cpu_count
from pyppl import Proc, Box, Aggr, utils, ProcTree, Channel, Job, logger
from pyppl.exception import ProcTagError, ProcAttributeError, ProcTreeProcExists, ProcInputError, ProcOutputError, ProcScriptError, ProcRunCmdError
from pyppl.template import TemplateLiquid
if helpers.moduleInstalled('jinja2'):
from pyppl.template import TemplateJinja2
from pyppl.runners import RunnerLocal
__folder__ = path.realpath(path.dirname(__file__))
class TestProc(testly.TestCase):
def setUpMeta(self):
self.testdir = path.join(gettempdir(), 'PyPPL_unittest', 'TestProc')
if path.exists(self.testdir):
rmtree(self.testdir)
makedirs(self.testdir)
def dataProvider_testInit(self):
yield 'notag', 'No description', None, {
# 'brings': {},
'channel': [],
'depends': [],
'echo': {},
def dataProvider_testInit(self):
yield {
'_log': {
'file': False
}
}, None, {}, {
'theme': 'default'
}, [], ['PYPPL', 'TIPS']
yield {
'default': {
'forks': 8
},
'_log': {
'file': False
}
}, None, {
'default': {
'forks': 8
}
}, {
'theme': 'default'
}, [], ['PYPPL', 'TIPS']
# default conf files
if helpers.moduleInstalled('yaml'):
ymlfile = path.join(self.testdir, 'config.yaml')
helpers.writeFile(ymlfile, ['default:', ' forks: 10'])
j1file = path.join(self.testdir, 'config1.json')
helpers.writeFile(j1file, '{"default": {"forks": 8}}')
j2file = path.join(self.testdir, 'config2.json')
helpers.writeFile(j2file, '{"default": {"forks": 6}}')
logfile = path.join(self.testdir, 'init.log')
yield {
'_flowchart': {
'theme': 'dark'
},
'_log': {
'file': False
}
}, None, {
'default': {
'forks': 8
}
}, {
'theme': 'dark'
}, [j1file]
yield {
'_log': {
'file': False
}
}, None, {
'default': {
'forks': 6
}
}, {
'theme': 'default'
}, [j1file, j2file]
yield {
'_log': {
'file': False
}
}, None, {
'default': {
'forks': 8
}
}, {
'theme': 'default'
}, [j2file, j1file]
yield {
'_log': {
'file': False
}
}, j1file, {
'default': {
'forks': 8
}
}, {
'theme': 'default'
}, [j2file]
yield {
'default': {
'forks': 4
},
'_log': {
'file': False
}
}, j1file, {
'default': {
'forks': 4
}
}, {
'theme': 'default'
}, [j2file]
if helpers.moduleInstalled('yaml'):
yield {
'_log': {
'file': False
}
}, ymlfile, {
'default': {
'forks': 10
}
}, {
'theme': 'default'
}, [j2file, j1file]
yield {
'default': {
'forks': 4
},
'_log': {
'file': False
}
}, ymlfile, {
'default': {
'forks': 4
}
}, {
'theme': 'default'
}, [j2file, j1file]
yield {
'default': {
'forks': 4
},
'_log': {
'file': False
}
}, j1file, {
'default': {
'forks': 4
}
}, {
'theme': 'default'
}, [j2file, ymlfile]
class TestPyPPL(testly.TestCase):
def setUpMeta(self):
self.testdir = path.join(gettempdir(), 'PyPPL_unittest', 'TestPyPPL')
if path.exists(self.testdir):
rmtree(self.testdir)
makedirs(self.testdir)
def dataProvider_testInit(self):
yield {
'_log': {
'file': False
}
}, None, {}, {
'theme': 'default'
}, [], ['PYPPL', 'TIPS']
yield {
'default': {
'forks': 8
},
'_log': {
'file': False
}
}, None, {
'default': {
'forks': 8
}
}, {
'theme': 'default'
}, [], ['PYPPL', 'TIPS']
# default conf files
if helpers.moduleInstalled('yaml'):
ymlfile = path.join(self.testdir, 'config.yaml')
helpers.writeFile(ymlfile, ['default:', ' forks: 10'])
j1file = path.join(self.testdir, 'config1.json')
helpers.writeFile(j1file, '{"default": {"forks": 8}}')
j2file = path.join(self.testdir, 'config2.json')
helpers.writeFile(j2file, '{"default": {"forks": 6}}')
logfile = path.join(self.testdir, 'init.log')
yield {
'_flowchart': {
'theme': 'dark'
},
'_log': {
'file': False
}
}, None, {
'default': {
'forks': 8
}
}, {
'theme': 'dark'
}, [j1file]
yield {
'_log': {
'file': False
}
}, None, {
'default': {
'forks': 6
}
}, {
'theme': 'default'
}, [j1file, j2file]
yield {
'_log': {
'file': False
}
}, None, {
'default': {
'forks': 8
}
}, {
'theme': 'default'
}, [j2file, j1file]
yield {
'_log': {
'file': False
}
}, j1file, {
'default': {
'forks': 8
}
}, {
'theme': 'default'
}, [j2file]
yield {
'default': {
'forks': 4
},
'_log': {
'file': False
}
}, j1file, {
'default': {
'forks': 4
}
}, {
'theme': 'default'
}, [j2file]
if helpers.moduleInstalled('yaml'):
yield {
'_log': {
'file': False
}
}, ymlfile, {
'default': {
'forks': 10
}
}, {
'theme': 'default'
}, [j2file, j1file]
yield {
'default': {
'forks': 4
},
'_log': {
'file': False
}
}, ymlfile, {
'default': {
'forks': 4
}
}, {
'theme': 'default'
}, [j2file, j1file]
yield {
'default': {
'forks': 4
},
'_log': {
'file': False
}
}, j1file, {
'default': {
'forks': 4
}
}, {
'theme': 'default'
}, [j2file, ymlfile]
def testInit(self,
config,
cfgfile,
outconf,
outfcconf,
dftconffiles=[],
errs=[]):
PyPPL.DEFAULT_CFGFILES = dftconffiles
with helpers.log2str(levels='all') as (out, err):
pp = PyPPL(config, cfgfile)
stderr = err.getvalue()
self.assertIsInstance(pp, PyPPL)
self.assertIsInstance(pp.tree, ProcTree)
self.assertDictEqual(pp.config, outconf)
self.assertDictEqual(pp.fcconfig, outfcconf)
for err in errs:
self.assertIn(err, stderr)
stderr = stderr[(stderr.find(err) + len(err)):]
def dataProvider_testNoYaml(self):
yield self.testdir,
def testNoYaml(self, testdir):
PyPPL.DEFAULT_CFGFILES = []
ymlfile = path.join(testdir, 'config.yaml')
helpers.writeFile(ymlfile, ['default:', ' forks: 10'])
import sys
if helpers.moduleInstalled('yaml'):
import yaml
del sys.modules['yaml']
paths = []
paths.extend(sys.path)
del sys.path[:]
#while sys.path:
# paths.append(sys.path.pop(0))
with helpers.log2str(levels='all') as (out, err):
pp = PyPPL(config={'_log': {'file': True}}, cfgfile=ymlfile)
#for p in paths: sys.path.append(p)
sys.path = paths
self.assertDictEqual(pp.config, {})
logfiles = glob(path.splitext(sys.argv[0])[0] + "*.pyppl.log")
self.assertTrue(logfiles)
for logfile in logfiles:
remove(logfile)
def dataProvider_testRegisterProc(self):
pRegisterProc = Proc()
pRegisterProc1 = Proc()
yield pRegisterProc,
yield pRegisterProc1,
def testRegisterProc(self, p):
PyPPL._registerProc(p)
node = ProcTree.NODES[p]
self.assertIs(node.proc, p)
def dataProvider_testAny2Procs(self):
pAny2Procs1 = Proc()
pAny2Procs2 = Proc()
pAny2Procs3 = Proc()
pAny2Procs4 = Proc()
pAny2Procs51 = Proc(tag='51', id='pAny2Procs5')
pAny2Procs52 = Proc(tag='52', id='pAny2Procs5')
pAny2Procs6 = Proc()
pAny2Procs7 = Proc()
aAggr = Aggr(pAny2Procs6, pAny2Procs7)
aAggr.starts = [aAggr.pAny2Procs6, aAggr.pAny2Procs7]
yield [pAny2Procs1], [pAny2Procs1]
yield ['abc'], [], PyPPLProcFindError, 'Failed to find process'
yield [aAggr], [aAggr.pAny2Procs6, aAggr.pAny2Procs7]
yield ['pAny2Procs5'], [pAny2Procs51, pAny2Procs52]
yield ['pAny2Procs5.51'], [pAny2Procs51]
yield ['pAny2Procs1.notag'], [pAny2Procs1]
yield ['pAny2Procs5', aAggr, [pAny2Procs2, 'pAny2Procs1.notag']], [
pAny2Procs51, pAny2Procs52, pAny2Procs1, pAny2Procs2,
aAggr.pAny2Procs6, aAggr.pAny2Procs7
]
def testAny2Procs(self, args, procs, exception=None, msg=None):
if exception:
self.assertRaisesRegex(exception, msg, PyPPL._any2procs, *args)
else:
self.assertCountEqual(PyPPL._any2procs(*args), procs)
def dataProvider_testStart(self):
'''
/ p3 --- \
p1 -- p2 \ / p8
\ \ p4 \ p7
p10 p6 / \ p9
p5 /
'''
with helpers.log2str():
pp = PyPPL({'_log': {'file': None}})
pStart1 = Proc()
pStart2 = Proc()
pStart3 = Proc()
pStart4 = Proc()
pStart5 = Proc()
pStart6 = Proc()
pStart7 = Proc()
pStart8 = Proc()
pStart9 = Proc()
pStart10 = Proc()
pStart9.depends = pStart7
pStart8.depends = pStart7
pStart7.depends = pStart3, pStart6
pStart3.depends = pStart2
pStart6.depends = pStart4, pStart5
pStart3.depends = pStart2
pStart4.depends = pStart2
pStart2.depends = pStart1
pStart10.depends = pStart1
yield pp, pStart1, [pStart1]
yield pp, [pStart1, pStart5], [pStart1, pStart5]
yield pp, [pStart1, pStart5, pStart2], [pStart1, pStart5], [
'WARNING', 'Start process pStart2 ignored, depending on [pStart1]'
]
def testStart(self, pp, starts, outstarts, errs=[]):
with helpers.log2str(levels='all') as (out, err):
self.assertIs(pp.start(starts), pp)
for outstart in outstarts:
self.assertTrue(ProcTree.NODES[outstart].start)
stderr = err.getvalue()
for err in errs:
self.assertIn(err, stderr)
stderr = stderr[(stderr.find(err) + len(err)):]
def dataProvider_test_resume(self):
'''
/ p3 --- \
p1 -- p2 \ / p8
\ \ p4 \ p7
p10 p6 / \ p9
p5 /
'''
with helpers.log2str():
pp = PyPPL({'_log': {'_file': None}})
p_resume1 = Proc()
p_resume2 = Proc()
p_resume3 = Proc()
p_resume4 = Proc()
p_resume5 = Proc()
p_resume6 = Proc()
p_resume7 = Proc()
p_resume8 = Proc()
p_resume9 = Proc()
p_resume10 = Proc()
p_resume9.depends = p_resume7
p_resume8.depends = p_resume7
p_resume7.depends = p_resume3, p_resume6
p_resume3.depends = p_resume2
p_resume6.depends = p_resume4, p_resume5
p_resume3.depends = p_resume2
p_resume4.depends = p_resume2
p_resume2.depends = p_resume1
p_resume10.depends = p_resume1
yield pp, [p_resume1, p_resume5], [p_resume2, p_resume6], True, [
p_resume1, p_resume5
], PyPPLProcRelationError, 'One of the routes cannot be achived from resumed processes: \'p_resume10 <- \[p_resume1\]\''
yield pp, [p_resume1, p_resume5], [p_resume1,
p_resume6], True, [p_resume5]
yield pp, [p_resume1,
p_resume5], [p_resume1, p_resume3, p_resume6
], False, [p_resume5, p_resume2, p_resume4]
def test_resume(self,
pp,
starts,
resumes,
plus,
skips,
exception=None,
msg=None):
pp.tree = ProcTree()
for node in ProcTree.NODES.values():
node.proc.resume = ''
pp.start(starts)
helpers.log2sys()
skip = 'skip+' if plus else 'skip'
resume = 'resume+' if plus else 'resume'
if exception:
self.assertRaisesRegex(exception,
msg,
pp._resume,
*resumes,
plus=plus)
else:
pp._resume(*resumes, plus=plus)
for r in resumes:
self.assertEqual(r.resume, resume)
for s in skips:
self.assertEqual(s.resume, skip)
def dataProvider_testResume1(self):
with helpers.log2str():
pp = PyPPL({'_log': {'file': None}})
pResume11 = Proc()
pResume12 = Proc()
pResume13 = Proc()
pResume13.depends = pResume12
pResume12.depends = pResume11
yield pp, pResume11, []
yield pp, pResume11, [pResume12]
def testResume1(self, pp, start, procs):
pp.tree = ProcTree()
pp.start(start).resume(procs)
if not procs:
for node in ProcTree.NODES.values():
self.assertEqual(node.proc.resume, '')
else:
for node in ProcTree.NODES.values():
self.assertIn(node.proc.resume, ['', 'skip', 'resume'])
def dataProvider_testResume2(self):
with helpers.log2str():
pp = PyPPL({'_log': {'file': None}})
pResume21 = Proc()
pResume22 = Proc()
pResume23 = Proc()
pResume23.depends = pResume22
pResume22.depends = pResume21
yield pp, pResume21, []
yield pp, pResume21, [pResume22]
def testResume2(self, pp, start, procs):
pp.tree = ProcTree()
pp.start(start).resume2(procs)
if not procs:
for proc in ProcTree.NODES.keys():
if not proc.id.startswith('pResume2'): continue
self.assertEqual(proc.resume, '')
else:
for proc in ProcTree.NODES.keys():
if not proc.id.startswith('pResume2'): continue
self.assertIn(proc.resume, ['', 'skip+', 'resume+'])
'''
def dataProvider_testGetProfile(self):
yield {'log': {'file': None}, 'proc': {'id': 'a'}}, 'proc', {}, PyPPLConfigError, 'Cannot set a universal id for all process in configuration: \'a\''
yield {'log': {'file': None}, 'proc': {}, 'sge': {}}, 'sge', {'runner': 'sge'}
yield {'log': {'file': None}, 'proc': {}, 'haha': {'tag': 'new'}}, 'haha', {'runner': 'local', 'tag': 'new'}, None, None, [
'WARNING',
'No runner specified in profile \'haha\', will use local runner.'
]
def testGetProfile(self, inconf, profile, config, exception = None, msg = None, errs = []):
PyPPL.DEFAULT_CFGFILES = []
with helpers.log2str():
pp = PyPPL(inconf)
if exception:
self.assertRaisesRegex(exception, msg, pp._getProfile, profile)
else:
with helpers.log2str(levels = 'all') as (out, err):
c = pp._getProfile(profile)
stderr = err.getvalue()
self.assertEqual(c, config)
for err in errs:
self.assertIn(err, stderr)
'''
def dataProvider_testShowAllRoutes(self):
'''
/ p3 --- \
p1 -- p2 \ / p8
\ \ p4 \ p7
p10 p6 / \ p9
p5 /
'''
pShowAllRoutes1 = Proc()
pShowAllRoutes2 = Proc()
pShowAllRoutes3 = Proc()
pShowAllRoutes4 = Proc()
pShowAllRoutes5 = Proc()
pShowAllRoutes6 = Proc()
pShowAllRoutes7 = Proc()
pShowAllRoutes8 = Proc()
pShowAllRoutes9 = Proc()
pShowAllRoutes10 = Proc()
pShowAllRoutes3.depends = pShowAllRoutes2
pShowAllRoutes6.depends = pShowAllRoutes4, pShowAllRoutes5
pShowAllRoutes3.depends = pShowAllRoutes2
pShowAllRoutes4.depends = pShowAllRoutes2
pShowAllRoutes2.depends = pShowAllRoutes1
pShowAllRoutes10.depends = pShowAllRoutes1
aAggr = Aggr(pShowAllRoutes7, pShowAllRoutes8, pShowAllRoutes9)
aAggr.starts = [aAggr.pShowAllRoutes7]
aAggr.pShowAllRoutes8.depends = aAggr.pShowAllRoutes7
aAggr.pShowAllRoutes9.depends = aAggr.pShowAllRoutes7
aAggr.depends2 = [pShowAllRoutes3, pShowAllRoutes6]
yield [pShowAllRoutes1, pShowAllRoutes5], [
'DEBUG', '* pShowAllRoutes1 -> pShowAllRoutes10',
'pShowAllRoutes1 -> pShowAllRoutes2 -> pShowAllRoutes3 -> [aAggr]',
'* pShowAllRoutes1 -> pShowAllRoutes2 -> pShowAllRoutes4 -> pShowAllRoutes6 -> [aAggr]',
'* pShowAllRoutes5 -> pShowAllRoutes6 -> [aAggr]'
]
def testShowAllRoutes(self, start, errs):
with helpers.log2str():
pp = PyPPL({'_log': {'file': None}})
pp.start(start)
with helpers.log2str(levels='all') as (out, err):
pp.showAllRoutes()
stderr = err.getvalue()
for err in errs:
self.assertIn(err, stderr)
stderr = stderr[(stderr.find(err) + len(err)):]
def dataProvider_testFlowchart(self):
'''
/ p3 --- \
p1 -- p2 \ / p8
\ \ p4 \ p7
p10 p6 / \ p9
p5 /
'''
pFlowchart1 = Proc()
pFlowchart2 = Proc()
pFlowchart3 = Proc()
pFlowchart4 = Proc()
pFlowchart5 = Proc()
pFlowchart6 = Proc()
pFlowchart7 = Proc()
pFlowchart8 = Proc()
pFlowchart9 = Proc()
pFlowchart10 = Proc()
pFlowchart3.depends = pFlowchart2
pFlowchart6.depends = pFlowchart4, pFlowchart5
pFlowchart3.depends = pFlowchart2
pFlowchart4.depends = pFlowchart2
pFlowchart2.depends = pFlowchart1
pFlowchart10.depends = pFlowchart1
aAggr = Aggr(pFlowchart7, pFlowchart8, pFlowchart9)
aAggr.starts = [aAggr.pFlowchart7]
aAggr.pFlowchart8.depends = aAggr.pFlowchart7
aAggr.pFlowchart9.depends = aAggr.pFlowchart7
aAggr.depends2 = [pFlowchart3, pFlowchart6]
dotfile = path.join(self.testdir, 'test.dot')
fcfile = path.join(self.testdir, 'test.svg')
yield [pFlowchart1, pFlowchart5], fcfile, dotfile, [
'DEBUG',
'* pFlowchart1 -> pFlowchart10',
'pFlowchart1 -> pFlowchart2 -> pFlowchart3 -> [aAggr]',
'* pFlowchart1 -> pFlowchart2 -> pFlowchart4 -> pFlowchart6 -> [aAggr]',
'* pFlowchart5 -> pFlowchart6 -> [aAggr]',
'INFO',
'Flowchart file saved to: %s' % fcfile,
'DOT file saved to: %s' % dotfile,
]
yield [pFlowchart1, pFlowchart5], fcfile, None, [
'DEBUG',
'* pFlowchart1 -> pFlowchart10',
'pFlowchart1 -> pFlowchart2 -> pFlowchart3 -> [aAggr]',
'* pFlowchart1 -> pFlowchart2 -> pFlowchart4 -> pFlowchart6 -> [aAggr]',
'* pFlowchart5 -> pFlowchart6 -> [aAggr]',
'INFO',
'Flowchart file saved to: %s' % fcfile,
'DOT file saved to: %s' % dotfile,
]
@unittest.skipIf(not helpers.moduleInstalled('graphviz'),
'graphviz not installed.')
def testFlowchart(self, start, fcfile, dotfile, errs=[]):
with helpers.log2str():
pp = PyPPL({'_log': {'file': None}})
pp.start(start)
with helpers.log2str(levels='all') as (out, err):
pp.flowchart(fcfile=fcfile, dotfile=dotfile)
self.assertTrue(path.isfile(fcfile))
self.assertTrue(
path.isfile(dotfile or path.splitext(fcfile)[0] + '.dot'))
stderr = err.getvalue()
for err in errs:
self.assertIn(err, stderr)
stderr = stderr[(stderr.find(err) + len(err)):]
def dataProvider_testRun(self):
'''
/ p3 --- \
p1 -- p2 \ / p8
\ \ p4 \ p7
p10 p6 / \ p9
p5 /
'''
pRun0 = Proc()
pRun1 = pRun0.copy()
pRun2 = Proc()
pRun3 = Proc()
pRun4 = Proc()
pRun5 = Proc()
pRun6 = Proc()
pRun7 = Proc()
pRun8 = Proc()
pRun9 = Proc()
pRun10 = Proc()
pRun3.depends = pRun2
pRun6.depends = pRun4, pRun5
pRun3.depends = pRun2
pRun4.depends = pRun2
pRun2.depends = pRun1
pRun10.depends = pRun1
aAggr = Aggr(pRun7, pRun8, pRun9)
aAggr.starts = [aAggr.pRun7]
aAggr.pRun8.depends = aAggr.pRun7
aAggr.pRun9.depends = aAggr.pRun7
aAggr.depends2 = [pRun3, pRun6]
yield [pRun1, pRun5], 'profile', 'sge', [
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun1 (pRun0): No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun2: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun3: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun4: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun5: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun6: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun10: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun7.5gPF@aAggr: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun8.5gPF@aAggr: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
'pRun9.5gPF@aAggr: No description.',
'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~',
]
# pRun21 -> pRun22
# pRun23 -/
# \-> pRun24
pRun21 = Proc()
pRun23 = Proc()
pRun22 = Proc()
pRun22.depends = pRun21, pRun23
yield [pRun21], 'profile', 'local', [
"pRun22 won't run as path can't be reached: pRun22 <- pRun23"
]
def testRun(self, start, profile, runner, errs=[]):
with helpers.log2str():
pp = PyPPL({
'_log': {
'file': None
},
'profile': {
'ppldir': self.testdir,
'runner': runner
}
})
import sys
pp.start(start)
argv = sys.argv
sys.argv = [sys.argv[0]]
helpers.log2sys()
with helpers.log2str(levels='all') as (out, err):
pp.run(profile)
sys.argv = argv
for s in start:
self.assertEqual(s.runner, runner)
stderr = err.getvalue()
for err in errs:
self.assertIn(err, stderr)
stderr = stderr[(stderr.find(err) + len(err)):]
def dataProvider_testRegisterRunner(self):
yield RunnerLocal, 'local'
yield RunnerSsh, 'ssh'
yield RunnerDry, 'dry'
yield RunnerSge, 'sge'
yield RunnerSlurm, 'slurm'
class xxx(object):
pass
yield xxx, 'xxx'
class RunnerYyy(object):
pass
yield RunnerYyy, 'yyy'
def testRegisterRunner(self, runner, name):
if name in PyPPL.RUNNERS:
del PyPPL.RUNNERS[name]
PyPPL.registerRunner(runner)
self.assertIs(PyPPL.RUNNERS[name], runner)
class TestTemplateJinja2(testly.TestCase):
def dataProvider_testInit(self):
yield '', {}
yield '{{a}}', {'a': 1}
@unittest.skipIf(not helpers.moduleInstalled('jinja2'),
'Jinja2 not installed.')
def testInit(self, source, envs):
import jinja2
tpl = TemplateJinja2(source, **envs)
helpers.assertTextEqual(self, tpl.source, source)
self.assertDictContains(Template.DEFAULT_ENVS, tpl.envs)
self.assertDictContains(envs, tpl.envs)
self.assertIsInstance(tpl.engine, jinja2.Template)
def dataProvider_testStr(self):
yield '',
yield 'a',
yield '{{a}}',
yield '{{a}}\n{{b}}',
@unittest.skipIf(not helpers.moduleInstalled('jinja2'),
'Jinja2 not installed.')
def testStr(self, source):
tpl = TemplateJinja2(source)
lines = source.splitlines()
if len(lines) <= 1:
self.assertEqual(str(tpl),
'TemplateJinja2 < %s >' % ''.join(lines))
else:
helpers.assertTextEqual(
self, str(tpl),
'\n'.join(['TemplateJinja2 <<<'] +
['\t' + line
for line in tpl.source.splitlines()] + ['>>>']))
def dataProvider_testRender(self):
# 0
yield '{{name}}', {'name': 'John'}, 'John'
yield '{{names[0]}}', {'names': ['John', 'Tom']}, 'John'
yield '{{concate(v1, v2)}}', {
'v1': 'hello',
'v2': 'world',
'concate': lambda x, y: x + y
}, 'helloworld'
yield '{{R(v23)}}', {'v23': '"FALSE"'}, '\'"FALSE"\''
yield '{{R(v3)}}', {'v3': 'false'}, "FALSE"
# 5
yield '{{realpath(v4)}}', {'v4': __file__}, path.realpath(__file__)
#yield ('{{readlink(v5)}}', {'v5': path.join(path.dirname(path.realpath(path.abspath(__file__))), 'helpers.py')yield , path.relpath(path.jo(path.dirname(path.dirname(path.abspath(__file__))), 'bin', 'helpers.py'))),
yield '{{dirname(v6)}}', {'v6': '/a/b/c'}, '/a/b'
yield '{{basename(v7)}}{{bn(v7)}}', {'v7': '/a/b/c.txt'}, 'c.txtc.txt'
yield '{{basename(v8)}}{{bn(v8)}}', {
'v8': '/a/b/c[1].txt'
}, 'c.txtc.txt'
yield '{{basename(v9, v9b)}}{{bn(v9, v9b)}}', {
'v9': '/a/b/c.txt',
'v9b': True
}, 'c.txtc.txt'
# 10
yield '{{basename(v10, v10b)}}{{bn(v10, v10b)}} {{ext(v10)}}', {
'v10': '/a/b/c[1].txt',
'v10b': True
}, 'c[1].txtc[1].txt .txt'
yield '{{filename(v11)}}{{fn(v11)}} {{prefix(v11)}}', {
'v11': '/a/b/a.txt'
}, 'aa /a/b/a'
yield '{{filename(v12)}}{{fn(v12)}}', {'v12': '/a/b/b[1].txt'}, 'bb'
yield '{{filename(v13, v13b)}}{{fn(v13, v13b)}}', {
'v13': '/a/b/d.txt',
'v13b': True
}, 'dd'
yield '{{filename(v14, v14b)}}{{fn(v14, v14b)}}', {
'v14': '/a/b/c[1].txt',
'v14b': True
}, 'c[1]c[1]'
# 15
yield '{{R(var1)}}', {'var1': 'NULL'}, 'NULL'
yield '{{R(var2)}}', {'var2': 'abc'}, "'abc'"
yield '{% for var in varlist %}{{R(var)}}{% endfor %}', {
'varlist': ['abc', 'True', 1, False]
}, "'abc'TRUE1FALSE"
yield '{% if bool(var3) %}1{% else %}0{% endif %}', {
'var3': 'abc',
'bool': bool
}, '1'
yield '{% for k,v in data.items() %}{{k}}:{{v}}{% endfor %}', {
'data': {
'a': 1,
'b': 2
}
}, 'a:1b:2'
# 20
yield '{{quote(a)}}', {'a': ''}, '""'
yield '{{R(x)}}', {
'x': OrderedDict([(u'key1', 'val1'), ('key2', u'val2')])
}, "list(key1='val1',key2='val2')"
yield '{{asquote(b)}}', {'b': [1, 2]}, '"1" "2"'
yield '{{acquote(c)}}', {'c': [1, 2]}, '"1", "2"'
yield '{{squote(d)}}', {'d': '1'}, "'1'"
# 25
yield '{{json(e["f"])}}', {'e': {'f': [1, 2]}}, '[1, 2]'
yield '{{os.path.join(g,h)}}', {
'g': 'a',
'h': 'b',
'os': __import__('os')
}, 'a/b'
yield """
#!/usr/bin/env python
{% if x %}
{% for y in ylist %}
{{y}}
{% endfor %}
{% endif %}
""", {
'x': True,
'ylist': [1, 2, 3, 4, 5]
}, """
#!/usr/bin/env python\n\t\t\n\t\t
1\n\t\t
2\n\t\t
3\n\t\t
4\n\t\t
5\n\t\t
\n\t\t"""
yield '{{read(a)}}', {'a': __file__}, helpers.readFile(__file__)
file2read = path.join(path.dirname(__file__), 'helpers.py')
yield '{{"\\n".join(readlines(a))}}', {
'a': file2read
}, helpers.readFile(
file2read,
lambda x: '\n'.join(str(y) for y in x.splitlines() if y))
@unittest.skipIf(not helpers.moduleInstalled('jinja2'),
'Jinja2 not installed.')
def testRender(self, s, e, out):
t = TemplateJinja2(s)
helpers.assertTextEqual(self, t.render(e), out)
