def test_proctree_check():
p4 = Proc()
p5 = Proc(id='p4')
ProcTree.register(p4)
ProcTree.register(p5)
with pytest.raises(ProcTreeProcExists):
ProcTree.check(p4)
def test_appendix(proc, reportfile):
proc.report = '''
# Title
Some results
## Appendix
### Full detail
Some table
'''
p2 = Proc()
p2.depends = proc
p2.report = '''
# Title2
Some other results
## Appendix
### Full detail2
Some other table
'''
PyPPL().start(proc).run().report(outfile=reportfile)
assertInfile(
reportfile,
'<h2 id="title">Title</h2>',
'<p>Some results</p>',
'<h2 id="title2">Title2</h2>',
'<p>Some other results</p>',
'<h2 id="appendix">Appendix</h2>',
'<h3 id="full-detail">Full detail</h3>',
'<p>Some table</p>',
'<h3 id="full-detail2">Full detail2</h3>',
'<p>Some other table</p>',
)
def dataProvider_testSetattr(self):
pSetattra = Proc()
pSetattra.args.a = Box()
pSetattrb = Proc()
pSetattrb.args.a = Box()
pSetattrc = Proc()
pSetattrc.args.a = Box(b=1)
pSetattrd = Proc()
pSetattre = Proc()
aggr = Aggr(pSetattra, pSetattrb, pSetattrc, pSetattrd, pSetattre)
aggr.delegate('forks', 'pSetattrb')
aggr.delegate('tag', 'ends')
proxy = _Proxy(aggr, check = True)
yield proxy, 'forks', 10, [1, 10, 1, 1, 1]
yield proxy, 'tag', 't', [aggr.pSetattra.tag, aggr.pSetattra.tag, aggr.pSetattra.tag, aggr.pSetattra.tag, 't']
# depends and input
aggr.starts = 'pSetattra, pSetattrb'
yield proxy, 'depends', ['pSetattrc', 'pSetattrd'], [[aggr.pSetattrc], [aggr.pSetattrd], [aggr.pSetattrb], [aggr.pSetattrc], [aggr.pSetattrd]]
yield proxy, 'depends2', ['pSetattrc', 'pSetattrd'], [[aggr.pSetattrc, aggr.pSetattrd], [aggr.pSetattrc, aggr.pSetattrd], [aggr.pSetattrb], [aggr.pSetattrc], [aggr.pSetattrd]]
aggr1 = Aggr(pSetattra, pSetattrb, pSetattrc)
aggr1.delegate('args.a.b', 'pSetattra, pSetattrb')
aggr1.delegate('args.a', 'pSetattrb')
proxy1 = _Proxy(aggr1, prefix = ['args', 'a'], check = True)
yield proxy1, 'b', 3, [3, 3, 1]
def dataProvider_testInit(self):
yield TemplatePyPPLSyntaxError('', ''), TemplatePyPPLSyntaxError
yield TemplatePyPPLRenderError('', ''), TemplatePyPPLRenderError
yield LoggerThemeError('', ''), LoggerThemeError
yield ParameterNameError(''), ParameterNameError
yield ParameterTypeError(''), ParameterTypeError
yield ParametersParseError('', ''), ParametersParseError
yield ParametersLoadError('', ''), ParametersLoadError
p1 = Proc()
p2 = Proc()
yield ProcTreeProcExists(ProcTree.NODES[p1], ProcTree.NODES[p2]), ProcTreeProcExists
yield ProcTreeParseError('', ''), ProcTreeParseError
yield JobInputParseError('', ''), JobInputParseError
yield JobOutputParseError('', ''), JobOutputParseError
yield RunnerSshError(''), RunnerSshError
yield ProcTagError(''), ProcTagError
yield ProcAttributeError('', ''), ProcAttributeError
yield ProcInputError('', ''), ProcInputError
yield ProcOutputError('', ''), ProcOutputError
yield ProcScriptError('', ''), ProcScriptError
yield ProcRunCmdError('', ''), ProcRunCmdError
yield PyPPLProcFindError('', ''), PyPPLProcFindError
yield PyPPLProcRelationError('', ''), PyPPLProcRelationError
yield PyPPLConfigError('', ''), PyPPLConfigError
yield AggrAttributeError('', ''), AggrAttributeError
yield AggrCopyError('', ''), AggrCopyError
yield AggrKeyError('', ''), AggrKeyError
def test_run_defaultcfg(tmp_path):
os.environ['PYPPL2_f100_forks'] = '100'
config._load('PYPPL2.osenv')
pF100 = Proc()
pF100.input = {'a': [0]}
PyPPL().start(pF100).run('f100')
assert pF100.forks == 100
def dataProvider_testCheck(self):
proc_testCheck1 = Proc()
proc_testCheck2 = Proc()
proc_testCheck3 = Proc(id='proc_testCheck1')
yield proc_testCheck1, False
yield proc_testCheck2, False
yield proc_testCheck3, True
def dataProvider_testSetAttr(self):
pSetAttr = Proc()
pSetAttrDepends = Proc()
pSetAttrAggr = Proc()
aSetAttr = Aggr(pSetAttrAggr)
aSetAttr.ends = [aSetAttr.pSetAttrAggr]
yield pSetAttr, '__nosuchattr__', None, None, ProcAttributeError, 'Cannot set attribute for process'
yield pSetAttr, 'profile', 'sge', 'local', None, None, ['WARNING', 'Attribute "profile" is deprecated']
yield pSetAttr, 'envs', {'a': 1}
yield pSetAttr, 'depends', pSetAttr, None, ProcAttributeError, 'Process depends on itself'
yield pSetAttr, 'depends', 1, None, ProcAttributeError, "Process dependents should be 'Proc/Aggr', not: 'int'"
#5
yield pSetAttr, 'depends', pSetAttrDepends, [pSetAttrDepends]
yield pSetAttr, 'depends', aSetAttr, [aSetAttr.pSetAttrAggr]
yield pSetAttr, 'depends', (aSetAttr, pSetAttrDepends), [aSetAttr.pSetAttrAggr, pSetAttrDepends]
yield pSetAttr, 'script', 'file:' + path.abspath(__file__)
yield pSetAttr, 'script', 'file:' + path.relpath(__file__, __folder__), 'file:' + path.abspath(__file__)
#10
yield pSetAttr, 'args', {'a':1}, Box({'a':1})
yield pSetAttr, 'envs', {'a':1}, Box({'a':1})
yield pSetAttr, 'input', 'inkey1:var, inkey2:file'
yield pSetAttr, 'input', [('inval1', 'inval2')], {'inkey1:var, inkey2:file': [('inval1', 'inval2')]}
yield pSetAttr, 'input', {'inkey1:var, inkey2:file': [('inval1', 'inval2')]}
yield pSetAttr, 'input', [('inval3', 'inval4')], {'inkey1:var, inkey2:file': [('inval3', 'inval4')]}
yield pSetAttr, 'input', ['inkey3:var', 'inkey4:file'], {'inkey1:var, inkey2:file': ['inkey3:var', 'inkey4:file']}
yield pSetAttr, 'input', {'inkey1:var': 'inval1', 'inkey2:file': 'inval2'}
yield pSetAttr, 'input', [('inval3', 'inval4')], {'inkey1:var, inkey2:file': [('inval3', 'inval4')]}
def dataProvider_testSetattr(self):
pSetattra = Proc()
pSetattra.args.a = Box()
pSetattrb = Proc()
pSetattrb.args.a = Box()
pSetattrc = Proc()
pSetattrc.args.a = Box(b=1)
pSetattrd = Proc()
pSetattre = Proc()
aggr = Aggr(pSetattra, pSetattrb, pSetattrc, pSetattrd, pSetattre)
aggr.delegate('forks', 'pSetattrb')
aggr.delegate('tag', 'ends')
yield aggr, 'forks', 10, [1, 10, 1, 1, 1]
yield aggr, 'tag', 't', [
aggr.pSetattra.tag, aggr.pSetattra.tag, aggr.pSetattra.tag,
aggr.pSetattra.tag, 't'
]
# depends and input
aggr.starts = 'pSetattra, pSetattrb'
yield aggr, 'depends', ['pSetattrc', 'pSetattrd'], [[aggr.pSetattrc],
[aggr.pSetattrd],
[aggr.pSetattrb],
[aggr.pSetattrc],
[aggr.pSetattrd]]
yield aggr, 'depends2', ['pSetattrc', 'pSetattrd'
], [[aggr.pSetattrc, aggr.pSetattrd],
[aggr.pSetattrc, aggr.pSetattrd],
[aggr.pSetattrb], [aggr.pSetattrc],
[aggr.pSetattrd]]
aggr1 = Aggr(pSetattra, pSetattrb, pSetattrc)
aggr1.delegate('errntry', 'pSetattra, pSetattrb')
aggr1.delegate('args.a', 'pSetattrb')
yield aggr1, 'errntry', 8, [8, 8, 3]
def dataProvider_testBuildOutput(self):
pBuildOutput = Proc()
pBuildOutput.ppldir = self.testdir
pBuildOutput.output = ''
yield pBuildOutput, '', {}
yield pBuildOutput, {}, {}, ProcOutputError, 'Process output should be str/list/OrderedDict, not: \'dict\''
yield pBuildOutput, 'a', {}, ProcOutputError, 'One of <key>:<type>:<value> missed for process output in: \'a\''
yield pBuildOutput, [
'a:b:c:d'
], {}, ProcOutputError, 'Too many parts for process output in: \'a:b:c:d\''
yield pBuildOutput, OrderedDict(
[('a:b:c', 'd')]
), {}, ProcOutputError, 'Too many parts for process output key in: \'a:b:c\''
yield pBuildOutput, 'a:b:c', {}, ProcOutputError, 'Unknown output type: \'b\''
yield pBuildOutput, 'a:c, b:file:d, e:dir:f', OrderedDict([
('a', ('var', 'c')),
('b', ('file', 'd')),
('e', ('dir', 'f')),
])
yield pBuildOutput, 'a:c, b:file:d, e:dir:f, g:stdout:h, i:stderr:j', OrderedDict(
[
('a', ('var', 'c')),
('b', ('file', 'd')),
('e', ('dir', 'f')),
('g', ('stdout', 'h')),
('i', ('stderr', 'j')),
])
def dataProvider_testInit(self):
pInita = Proc()
pInitb = Proc()
pInitc = Proc()
aggr = Aggr(pInita, pInitb, pInitc)
yield aggr, [pInita, pInitb, pInitc], None, True
yield aggr, [pInita, pInitb, pInitc], 'args', False
def testInit(self, tag, desc, id, props, config, exception=None):
self.maxDiff = None
if exception:
self.assertRaises(exception, Proc, tag=tag, desc=desc, id=id)
else:
p = Proc(tag=tag, desc=desc, id=id)
self.assertDictEqual(p.props, props)
self.assertDictEqual(p.config, config)
config2 = config.copy()
del config2['tag']
del config2['desc']
del config2['id']
p2 = Proc(tag, desc, id=config['id'], **config2)
props['sets'] = list(
sorted([
'runner', 'echo', 'depends', 'expect', 'callfront',
'script', 'cache', 'nthread', 'beforeCmd', 'template',
'rc', 'input', 'forks', 'acache', 'workdir', 'resume',
'exhow', 'args', 'exow', 'dirsig', 'ppldir', 'errhow',
'lang', 'tplenvs', 'exdir', 'expart', 'afterCmd',
'callback', 'aggr', 'output', 'errntry'
]))
p2.props['sets'] = list(sorted(p2.sets))
self.assertDictEqual(p2.props, props)
self.assertDictEqual(p2.config, config)
def test_checkProc():
py = Proc()
pz = Proc(id='py')
PyPPL._registerProc(py)
PyPPL._registerProc(pz)
with pytest.raises(ProcTreeProcExists):
PyPPL._checkProc(pz)
def test_proctree_getnexttorun(set2):
# p15 -> p16 -> p17 -> 19
# p14 _/ \_ p18_/ \_ p20
# hide
p1 = Proc()
#ProcTree.register(p1)
p2 = Proc()
p2.depends = p1
pt = ProcTree()
pt.init()
pt.setStarts([set2.p14, set2.p15])
pt.NODES[set2.p14].ran = True
assert pt.getNextToRun() is set2.p15
pt.NODES[set2.p15].ran = True
assert pt.getNextToRun() is set2.p16
pt.NODES[set2.p16].ran = True
assert pt.getNextToRun() is set2.p18
pt.NODES[set2.p18].ran = True
assert pt.getNextToRun() is set2.p17
pt.NODES[set2.p17].ran = True
assert pt.getNextToRun() is set2.p20
pt.NODES[set2.p20].ran = True
assert pt.getNextToRun() is set2.p19
pt.NODES[set2.p19].ran = True
assert pt.getNextToRun() is None
def test_jobfail(caplog):
p = Proc(id='pPluginJobFail')
p.input = {'a': ['1']}
p.script = 'exit 1'
with pytest.raises(SystemExit):
PyPPL().start(p).run()
assert any('Job 0 failed' in msg for _, _, msg in caplog.record_tuples)
def dataProvider_testSameIdTag(self):
proc1 = Proc()
pn1 = ProcNode(proc1)
yield pn1, proc1, True
proc2 = Proc()
yield pn1, proc2, False
def test_proctree_getnextstr():
p8 = Proc()
p9 = Proc()
p9.depends = p8
ProcTree.register(p8)
ProcTree().init()
assert ProcTree.getNextStr(p8) == '[p9]'
assert ProcTree.getNextStr(p9) == 'END'
def dataProvider_testGetattr(self):
pGetattra = Proc()
pGetattrb = Proc()
pGetattrc = Proc()
aggr = Aggr(pGetattra, pGetattrb, pGetattrc)
proxy = _Proxy(aggr)
yield proxy, ''
yield proxy, 'args'
def dataProvider_testRepr(self):
pRepr = Proc()
yield pRepr, '<Proc(pRepr) @ %s>' % hex(id(pRepr))
pRepr1 = Proc(tag='atag')
yield pRepr1, '<Proc(pRepr1.atag) @ %s>' % hex(id(pRepr1))
pRepr2 = Proc(tag='aggr')
pRepr2.aggr = 'aggr'
yield pRepr2, '<Proc(pRepr2.aggr@aggr) @ %s>' % hex(id(pRepr2))
def dataProvider_testDelEnd(self):
p1 = Proc()
p2 = Proc()
p3 = Proc()
aggr = Aggr(p1, p2, p3, depends=False)
aggr.ends = 'p1, p2'
yield aggr, [aggr.p1], [aggr.p2]
yield aggr, ['p1, p2'], []
def test_proctree_getprevstr():
p6 = Proc()
p7 = Proc()
p7.depends = p6
ProcTree.register(p6)
ProcTree().init()
assert ProcTree.getPrevStr(p6) == 'START'
assert ProcTree.getPrevStr(p7) == '[p6]'
def test_setgetattr():
pSetAttr = Proc()
assert pSetAttr.ptest == 0
pSetAttr.ptest = 1
assert pSetAttr.ptest == 100
assert pSetAttr.pempty == 0
pSetAttr.pempty = 1
assert pSetAttr.pempty == 1
def test_prerun(caplog):
sys.argv = [sys.argv[0]]
with pytest.raises(plugin.PyPPLFuncWrongPositionError):
PyPPL().start(Proc(id='pPreRun1')).run().preRun()
assert not any('PYPPL PRERUN' in msg for _, _, msg in caplog.record_tuples)
PyPPL().start(Proc(id='pPreRun2')).preRun().run()
assert any('PYPPL PRERUN' in msg for _, _, msg in caplog.record_tuples)
def dataProvider_testAddEnd(self):
p1 = Proc()
p2 = Proc()
p3 = Proc()
aggr = Aggr(p1, p2, p3, depends=False)
yield aggr, [aggr.p1], [aggr.p1]
yield aggr, [aggr.p2], [aggr.p1, aggr.p2]
yield aggr, ['p1', 'p2'], [aggr.p1, aggr.p2]
yield aggr, ['p1, p2'], [aggr.p1, aggr.p2]
def dataProvider_testSetattr(self):
pGetAttr1 = Proc()
pGetAttr2 = Proc()
pGetAttr3 = Proc()
aggr1 = Aggr(pGetAttr1, pGetAttr2, pGetAttr3)
aggr2 = Aggr(pGetAttr1, pGetAttr2, pGetAttr3)
yield aggr1, 'forks', 10, [10] * 3
aggr2.delegate('forks', 'pGetAttr2')
yield aggr2, 'forks', 10, [1, 10, 1]
def dataProvider_testDepends2(self):
pDepends21 = Proc()
pDepends22 = Proc()
pDepends23 = Proc()
pDepends24 = Proc()
pDepends25 = Proc()
aggr = Aggr(pDepends21, pDepends22, pDepends23)
aggr.starts = [pDepends21, pDepends22]
yield aggr, [pDepends24, pDepends25]
def dataProvider_testName(self):
pName = Proc()
pName.tag = 'tag'
pName.aggr = 'aggr'
yield pName, True, 'pName.tag@aggr'
yield pName, False, 'pName.tag'
pName1 = Proc()
yield pName1, True, 'pName1'
yield pName1, False, 'pName1'
def dataProvider_testInit(self):
proc_testInit1 = Proc()
proc_testInit2 = Proc()
proc_testInit3 = Proc()
proc_testInit4 = Proc()
proc_testInit2.depends = proc_testInit1
proc_testInit3.depends = proc_testInit2
proc_testInit4.depends = proc_testInit2
yield [proc_testInit1, proc_testInit2, proc_testInit3, proc_testInit4],
yield [proc_testInit1, proc_testInit3],
def test_proctree_getpaths(set1):
# hide
# p15 -> p16 -> p17 -> 19
# \_ p18_/ \_ p20
pt = ProcTree()
pt.init()
assert pt.getPaths(set1.p15) == []
assert pt.getPaths(set1.p19) == [[set1.p17, set1.p16, set1.p15],
[set1.p17, set1.p18, set1.p16, set1.p15]]
assert pt.getPaths(set1.p20) == [[set1.p17, set1.p16, set1.p15],
[set1.p17, set1.p18, set1.p16, set1.p15]]
# assert pt.getPaths(set1.p19, check_hide = True) == [
# [set1.p17, set1.p15], [set1.p17, set1.p18, set1.p15]]
# assert pt.getPaths(set1.p20, check_hide = True) == [
# [set1.p17, set1.p15], [set1.p17, set1.p18, set1.p15]]
# circulic dependence
p21 = Proc()
p22 = Proc()
p23 = Proc()
# p21 -> p22 -> p23 -> p21
p21.depends = p23
p23.depends = p22
p22.depends = p21
pt = ProcTree()
pt.init()
with pytest.raises(ProcTreeParseError):
pt.getPaths(p23)
def dataProvider_testReset(self):
proc_testReset1 = Proc()
proc_testReset2 = Proc()
proc_testReset3 = Proc()
proc_testReset4 = Proc()
proc_testReset2.depends = proc_testReset1
proc_testReset3.depends = proc_testReset2
proc_testReset4.depends = proc_testReset2
yield [
proc_testReset1, proc_testReset2, proc_testReset3, proc_testReset4
],
def test_run_extrafile(tmp_path):
cfile = tmp_path / 'test_run_extrafile.ini'
cfile.write_text('''
[default]
forks = 2
[f10]
forks = 10
''')
pCfile = Proc()
pCfile.input = {'a': [0]}
PyPPL(cfgfile=cfile).start(pCfile).run('f10')
assert pCfile.forks == 10