def dataProvider_testBuildProps(self): testdir = self.testdir # 0 pBuildProps = Proc() pBuildProps.ppldir = testdir yield pBuildProps, {}, {}, ProcTreeProcExists, 'There are two processes with id\(pBuildProps\) and tag\(notag\)' # 1 pBuildProps1 = Proc(id = 'pBuildProps') pBuildProps1.ppldir = testdir yield pBuildProps1, {}, {}, ProcTreeProcExists, 'There are two processes with id\(pBuildProps\) and tag\(notag\)' # 2-9 pBuildProps2 = Proc() pBuildProps2.ppldir = testdir yield pBuildProps2, {}, {'template': TemplatePyPPL} yield pBuildProps2, {'template': ''}, {'template': TemplatePyPPL} if helpers.moduleInstalled('jinja2'): yield pBuildProps2, {'template': TemplateJinja2}, {'template': TemplateJinja2} yield pBuildProps2, {'template': 'jinja2'}, {'template': TemplateJinja2} yield pBuildProps2, {'rc': ' 0, 1, '}, {'rc': [0,1]} yield pBuildProps2, {'rc': 2}, {'rc': [2]} yield pBuildProps2, {'rc': [0, 1]}, {'rc': [0, 1]} yield pBuildProps2, {'workdir': path.join(testdir, 'pBuildProps2')}, {'workdir': path.join(testdir, 'pBuildProps2')} # 10 pBuildProps3 = Proc() pBuildProps3.ppldir = testdir yield pBuildProps3, {}, {'workdir': path.join(testdir, 'PyPPL.pBuildProps3.notag.%s' % pBuildProps3._suffix())}, None, None, [lambda p: path.isdir(p.workdir)] # 11 pBuildProps4 = Proc() pBuildProps4.ppldir = testdir yield pBuildProps4, {'resume': 'skip+'}, {}, ProcAttributeError, 'Cannot skip process, as workdir not exists' # 12 pBuildProps5 = Proc() pBuildProps5.ppldir = testdir # exdir yield pBuildProps5, {'exdir': path.relpath(path.join(testdir, 'exports'))}, {'exdir': path.abspath(path.join(testdir, 'exports'))}, None, None, [lambda p: path.isdir(p.exdir)] # echo # 13-20 yield pBuildProps5, {'echo': True}, {'echo': {'jobs': [0], 'type': {'stderr': None, 'stdout': None}}} yield pBuildProps5, {'echo': False}, {'echo': {'jobs': [], 'type': {'stderr': None, 'stdout': None}}} yield pBuildProps5, {'echo': 'stderr'}, {'echo': {'jobs': [0], 'type': {'stderr': None}}} yield pBuildProps5, {'echo': 'stdout'}, {'echo': {'jobs': [0], 'type': {'stdout': None}}} yield pBuildProps5, {'echo': {'type': 'all'}}, {'echo': {'jobs': [0], 'type': {'stderr': None, 'stdout': None}}} yield pBuildProps5, {'echo': {'jobs': ' 0, 1 ', 'type': 'all'}}, {'echo': {'jobs': [0, 1], 'type': {'stderr': None, 'stdout': None}}} yield pBuildProps5, {'echo': {'jobs': range(2), 'type': 'all'}}, {'echo': {'jobs': [0, 1], 'type': {'stderr': None, 'stdout': None}}} yield pBuildProps5, {'echo': {'type': 'stderr'}}, {'echo': {'jobs': [0], 'type': {'stderr': None}}} yield pBuildProps5, {'echo': {'type': {'all': r'^a'}}}, {'echo': {'jobs': [0], 'type': {'stderr': r'^a', 'stdout': r'^a'}}} # expect # 21 yield pBuildProps5, {'expect': 'expect template'}, {}, None, None, [lambda p: p.expect.source == 'expect template'] # expart # 22 yield pBuildProps5, {'expart': 'a,b,c,d,"e,f"'}, {}, None, None, [ (lambda p: p.expart[i].source == v) for i,v in enumerate(['a', 'b', 'c', 'd', '"e,f"']) ]
def dataProvider_testLoadFile(self): yield self.testdir, False, [] jsonfile = path.join(self.testdir, 'testLoadFile.json') helpers.writeFile( jsonfile, '\n'.join([ '{', ' "a": "2",', ' "a.desc": "Option a",', ' "a.type": "int",', ' "a.required": true', '}', ])) p1 = Parameter('a', 2) p1.desc = "Option a" p1.required = True yield jsonfile, True, [p1] if helpers.moduleInstalled('yaml'): yamlfile = path.join(self.testdir, 'testLoadFile.yaml') helpers.writeFile( yamlfile, '\n'.join([ 'a: 2', 'a.desc: Option a', 'a.type: int', 'a.required: false', 'a.show: true', '', ])) p2 = Parameter('a', 2) p2.desc = "Option a" p2.required = False p2.show = True yield yamlfile, False, [p2] conffile = path.join(self.testdir, 'testLoadFile.conf') helpers.writeFile( conffile, '\n'.join([ '[PARAM1]', 'a = 2', 'a.desc = Option a', 'a.type = int', 'a.required = f', '[PARAM2]', 'a.type = str', 'b:', ' 1', ' 2', 'b.type = list', ])) p3 = Parameter('a', '2') p3.desc = "Option a" p3.required = False p4 = Parameter('b', ['1', '2']) yield conffile, True, [p3, p4]
def testNoYaml(self, testdir): PyPPL.DEFAULT_CFGFILES = [] ymlfile = path.join(testdir, 'config.yaml') helpers.writeFile(ymlfile, ['default:', ' forks: 10']) import sys if helpers.moduleInstalled('yaml'): import yaml del sys.modules['yaml'] paths = [] paths.extend(sys.path) del sys.path[:] #while sys.path: # paths.append(sys.path.pop(0)) with helpers.log2str(levels='all') as (out, err): pp = PyPPL(config={'_log': {'file': True}}, cfgfile=ymlfile) #for p in paths: sys.path.append(p) sys.path = paths self.assertDictEqual(pp.config, {}) logfiles = glob(path.splitext(sys.argv[0])[0] + "*.pyppl.log") self.assertTrue(logfiles) for logfile in logfiles: remove(logfile)
import testly, sys, helpers if helpers.moduleInstalled('graphviz'): from os import path, makedirs from shutil import rmtree from tempfile import gettempdir from pyppl import Proc from pyppl.flowchart import Flowchart from graphviz import Digraph class TestFlowchart(testly.TestCase): def setUpMeta(self): self.testdir = path.join(gettempdir(), 'PyPPL_unittest', 'TestFlowchart') if path.exists(self.testdir): rmtree(self.testdir) makedirs(self.testdir) def dataProvider_testInit(self): dotfile = path.join(self.testdir, 'test.dot') svgfile = path.join(self.testdir, 'test.svg') yield dotfile, svgfile yield 'x', 'y' yield 'x', svgfile yield dotfile, 'y' def testInit(self, fcfile, dotfile): fc = Flowchart(fcfile, dotfile) self.assertEqual(fc.fcfile, fcfile) self.assertEqual(fc.dotfile, dotfile) self.assertIsInstance(fc, Flowchart)
def dataProvider_testSaveSettings(self): pSaveSettings = Proc() pSaveSettings.ppldir = self.testdir yield pSaveSettings, [ # '[brings]', '[channel]', 'value: []', '[depends]', 'procs: []', '[echo]', 'jobs: []', 'type: {"stderr": null, "stdout": null}', '[expart]', 'value_0: TemplateLiquid < >', '[expect]', 'value: TemplateLiquid < >', '[input]', '[output]', '[procvars]', 'args: {}', 'proc: {', '[rc]', 'value: [0]', '[runner]', 'value: local', '[script]', 'value:', ' "TemplateLiquid < #!/usr/bin/env bash >"', '[sets]', 'value: [\'ppldir\']', '[size]', 'value: 0', '[suffix]', 'value: ', '[template]', 'name: TemplateLiquid', '[workdir]', 'value: ', ] pSaveSettings1 = Proc() pSaveSettings1.ppldir = self.testdir infile1 = path.join(self.testdir, 'pSaveSettings1-in1.txt') infile2 = path.join(self.testdir, 'pSaveSettings1-in2.txt') brfile1 = path.join(self.testdir, 'pSaveSettings1-in1.br') brfile2 = path.join(self.testdir, 'pSaveSettings1-in2.br') helpers.writeFile(infile1) helpers.writeFile(infile2) helpers.writeFile(brfile1) helpers.writeFile(brfile2) pSaveSettings1.input = { 'a': 1, 'b:file': [infile1, infile2], 'c:files': [[infile1, infile2]] } #pSaveSettings1.brings = {'b': '{{fn(i.b)}}.br'} pSaveSettings1.output = 'out:file:{{fn(i.b)}}-{{i.a}}.out' pSaveSettings1.echo = {'jobs': [0, 1]} pSaveSettings1.expart = '*-1.out' pSaveSettings1.expect = 'grep 1 {{o.out}}' pSaveSettings1.args.a = 'a' pSaveSettings1.rc = '0,1' pSaveSettings1.script = 'echo {{i.a}} > {{o.out}}' pSaveSettings1.template = 'jinja2' if helpers.moduleInstalled('jinja2'): yield pSaveSettings1, [ #'[brings]', #'b: [\'TemplateJinja2 < {{fn(i.b)}}.br >\']', '[channel]', 'value: []', '[depends]', 'procs: []', '[echo]', 'jobs: [0, 1]', 'type: {"stderr": null, "stdout": null}', '[expart]', 'value_0: TemplateJinja2 < *-1.out >', '[expect]', 'value: TemplateJinja2 < grep 1 {{o.out}} >', '[input]', 'a.type: var', 'a.data#0', ' 1', 'a.data#1', ' 1', 'b.type: file', 'b.data#0', 'pSaveSettings1-in1.txt', 'b.data#1', 'pSaveSettings1-in2.txt', '[output]', 'out.type: file', 'out.data: TemplateJinja2 < {{fn(i.b)}}-{{i.a}}.out >', '[procvars]', 'args: {"a": "a"}', 'proc: {', '[rc]', 'value: [0, 1]', '[runner]', 'value: local', '[script]', 'value:', ' "TemplateJinja2 <<<"', ' "\\t#!/usr/bin/env bash"', ' "\\techo {{i.a}} > {{o.out}}"', ' ">>>"', '[sets]', 'value: [\'ppldir\', \'input\', \'output\', \'echo\', \'expart\', \'expect\', \'rc\', \'script\', \'template\']', '[size]', 'value: 2', '[suffix]', 'value: ', '[template]', 'name: TemplateJinja2', '[workdir]', 'value: ', ]
import helpers, testly, json, sys import copy as pycopy from os import path, makedirs from shutil import rmtree from tempfile import gettempdir from collections import OrderedDict from multiprocessing import cpu_count from pyppl import Proc, Box, Aggr, utils, ProcTree, Channel, Job, logger from pyppl.exception import ProcTagError, ProcAttributeError, ProcTreeProcExists, ProcInputError, ProcOutputError, ProcScriptError, ProcRunCmdError from pyppl.template import TemplateLiquid if helpers.moduleInstalled('jinja2'): from pyppl.template import TemplateJinja2 from pyppl.runners import RunnerLocal __folder__ = path.realpath(path.dirname(__file__)) class TestProc(testly.TestCase): def setUpMeta(self): self.testdir = path.join(gettempdir(), 'PyPPL_unittest', 'TestProc') if path.exists(self.testdir): rmtree(self.testdir) makedirs(self.testdir) def dataProvider_testInit(self): yield 'notag', 'No description', None, { # 'brings': {}, 'channel': [], 'depends': [], 'echo': {},
def dataProvider_testInit(self): yield { '_log': { 'file': False } }, None, {}, { 'theme': 'default' }, [], ['PYPPL', 'TIPS'] yield { 'default': { 'forks': 8 }, '_log': { 'file': False } }, None, { 'default': { 'forks': 8 } }, { 'theme': 'default' }, [], ['PYPPL', 'TIPS'] # default conf files if helpers.moduleInstalled('yaml'): ymlfile = path.join(self.testdir, 'config.yaml') helpers.writeFile(ymlfile, ['default:', ' forks: 10']) j1file = path.join(self.testdir, 'config1.json') helpers.writeFile(j1file, '{"default": {"forks": 8}}') j2file = path.join(self.testdir, 'config2.json') helpers.writeFile(j2file, '{"default": {"forks": 6}}') logfile = path.join(self.testdir, 'init.log') yield { '_flowchart': { 'theme': 'dark' }, '_log': { 'file': False } }, None, { 'default': { 'forks': 8 } }, { 'theme': 'dark' }, [j1file] yield { '_log': { 'file': False } }, None, { 'default': { 'forks': 6 } }, { 'theme': 'default' }, [j1file, j2file] yield { '_log': { 'file': False } }, None, { 'default': { 'forks': 8 } }, { 'theme': 'default' }, [j2file, j1file] yield { '_log': { 'file': False } }, j1file, { 'default': { 'forks': 8 } }, { 'theme': 'default' }, [j2file] yield { 'default': { 'forks': 4 }, '_log': { 'file': False } }, j1file, { 'default': { 'forks': 4 } }, { 'theme': 'default' }, [j2file] if helpers.moduleInstalled('yaml'): yield { '_log': { 'file': False } }, ymlfile, { 'default': { 'forks': 10 } }, { 'theme': 'default' }, [j2file, j1file] yield { 'default': { 'forks': 4 }, '_log': { 'file': False } }, ymlfile, { 'default': { 'forks': 4 } }, { 'theme': 'default' }, [j2file, j1file] yield { 'default': { 'forks': 4 }, '_log': { 'file': False } }, j1file, { 'default': { 'forks': 4 } }, { 'theme': 'default' }, [j2file, ymlfile]
class TestPyPPL(testly.TestCase): def setUpMeta(self): self.testdir = path.join(gettempdir(), 'PyPPL_unittest', 'TestPyPPL') if path.exists(self.testdir): rmtree(self.testdir) makedirs(self.testdir) def dataProvider_testInit(self): yield { '_log': { 'file': False } }, None, {}, { 'theme': 'default' }, [], ['PYPPL', 'TIPS'] yield { 'default': { 'forks': 8 }, '_log': { 'file': False } }, None, { 'default': { 'forks': 8 } }, { 'theme': 'default' }, [], ['PYPPL', 'TIPS'] # default conf files if helpers.moduleInstalled('yaml'): ymlfile = path.join(self.testdir, 'config.yaml') helpers.writeFile(ymlfile, ['default:', ' forks: 10']) j1file = path.join(self.testdir, 'config1.json') helpers.writeFile(j1file, '{"default": {"forks": 8}}') j2file = path.join(self.testdir, 'config2.json') helpers.writeFile(j2file, '{"default": {"forks": 6}}') logfile = path.join(self.testdir, 'init.log') yield { '_flowchart': { 'theme': 'dark' }, '_log': { 'file': False } }, None, { 'default': { 'forks': 8 } }, { 'theme': 'dark' }, [j1file] yield { '_log': { 'file': False } }, None, { 'default': { 'forks': 6 } }, { 'theme': 'default' }, [j1file, j2file] yield { '_log': { 'file': False } }, None, { 'default': { 'forks': 8 } }, { 'theme': 'default' }, [j2file, j1file] yield { '_log': { 'file': False } }, j1file, { 'default': { 'forks': 8 } }, { 'theme': 'default' }, [j2file] yield { 'default': { 'forks': 4 }, '_log': { 'file': False } }, j1file, { 'default': { 'forks': 4 } }, { 'theme': 'default' }, [j2file] if helpers.moduleInstalled('yaml'): yield { '_log': { 'file': False } }, ymlfile, { 'default': { 'forks': 10 } }, { 'theme': 'default' }, [j2file, j1file] yield { 'default': { 'forks': 4 }, '_log': { 'file': False } }, ymlfile, { 'default': { 'forks': 4 } }, { 'theme': 'default' }, [j2file, j1file] yield { 'default': { 'forks': 4 }, '_log': { 'file': False } }, j1file, { 'default': { 'forks': 4 } }, { 'theme': 'default' }, [j2file, ymlfile] def testInit(self, config, cfgfile, outconf, outfcconf, dftconffiles=[], errs=[]): PyPPL.DEFAULT_CFGFILES = dftconffiles with helpers.log2str(levels='all') as (out, err): pp = PyPPL(config, cfgfile) stderr = err.getvalue() self.assertIsInstance(pp, PyPPL) self.assertIsInstance(pp.tree, ProcTree) self.assertDictEqual(pp.config, outconf) self.assertDictEqual(pp.fcconfig, outfcconf) for err in errs: self.assertIn(err, stderr) stderr = stderr[(stderr.find(err) + len(err)):] def dataProvider_testNoYaml(self): yield self.testdir, def testNoYaml(self, testdir): PyPPL.DEFAULT_CFGFILES = [] ymlfile = path.join(testdir, 'config.yaml') helpers.writeFile(ymlfile, ['default:', ' forks: 10']) import sys if helpers.moduleInstalled('yaml'): import yaml del sys.modules['yaml'] paths = [] paths.extend(sys.path) del sys.path[:] #while sys.path: # paths.append(sys.path.pop(0)) with helpers.log2str(levels='all') as (out, err): pp = PyPPL(config={'_log': {'file': True}}, cfgfile=ymlfile) #for p in paths: sys.path.append(p) sys.path = paths self.assertDictEqual(pp.config, {}) logfiles = glob(path.splitext(sys.argv[0])[0] + "*.pyppl.log") self.assertTrue(logfiles) for logfile in logfiles: remove(logfile) def dataProvider_testRegisterProc(self): pRegisterProc = Proc() pRegisterProc1 = Proc() yield pRegisterProc, yield pRegisterProc1, def testRegisterProc(self, p): PyPPL._registerProc(p) node = ProcTree.NODES[p] self.assertIs(node.proc, p) def dataProvider_testAny2Procs(self): pAny2Procs1 = Proc() pAny2Procs2 = Proc() pAny2Procs3 = Proc() pAny2Procs4 = Proc() pAny2Procs51 = Proc(tag='51', id='pAny2Procs5') pAny2Procs52 = Proc(tag='52', id='pAny2Procs5') pAny2Procs6 = Proc() pAny2Procs7 = Proc() aAggr = Aggr(pAny2Procs6, pAny2Procs7) aAggr.starts = [aAggr.pAny2Procs6, aAggr.pAny2Procs7] yield [pAny2Procs1], [pAny2Procs1] yield ['abc'], [], PyPPLProcFindError, 'Failed to find process' yield [aAggr], [aAggr.pAny2Procs6, aAggr.pAny2Procs7] yield ['pAny2Procs5'], [pAny2Procs51, pAny2Procs52] yield ['pAny2Procs5.51'], [pAny2Procs51] yield ['pAny2Procs1.notag'], [pAny2Procs1] yield ['pAny2Procs5', aAggr, [pAny2Procs2, 'pAny2Procs1.notag']], [ pAny2Procs51, pAny2Procs52, pAny2Procs1, pAny2Procs2, aAggr.pAny2Procs6, aAggr.pAny2Procs7 ] def testAny2Procs(self, args, procs, exception=None, msg=None): if exception: self.assertRaisesRegex(exception, msg, PyPPL._any2procs, *args) else: self.assertCountEqual(PyPPL._any2procs(*args), procs) def dataProvider_testStart(self): ''' / p3 --- \ p1 -- p2 \ / p8 \ \ p4 \ p7 p10 p6 / \ p9 p5 / ''' with helpers.log2str(): pp = PyPPL({'_log': {'file': None}}) pStart1 = Proc() pStart2 = Proc() pStart3 = Proc() pStart4 = Proc() pStart5 = Proc() pStart6 = Proc() pStart7 = Proc() pStart8 = Proc() pStart9 = Proc() pStart10 = Proc() pStart9.depends = pStart7 pStart8.depends = pStart7 pStart7.depends = pStart3, pStart6 pStart3.depends = pStart2 pStart6.depends = pStart4, pStart5 pStart3.depends = pStart2 pStart4.depends = pStart2 pStart2.depends = pStart1 pStart10.depends = pStart1 yield pp, pStart1, [pStart1] yield pp, [pStart1, pStart5], [pStart1, pStart5] yield pp, [pStart1, pStart5, pStart2], [pStart1, pStart5], [ 'WARNING', 'Start process pStart2 ignored, depending on [pStart1]' ] def testStart(self, pp, starts, outstarts, errs=[]): with helpers.log2str(levels='all') as (out, err): self.assertIs(pp.start(starts), pp) for outstart in outstarts: self.assertTrue(ProcTree.NODES[outstart].start) stderr = err.getvalue() for err in errs: self.assertIn(err, stderr) stderr = stderr[(stderr.find(err) + len(err)):] def dataProvider_test_resume(self): ''' / p3 --- \ p1 -- p2 \ / p8 \ \ p4 \ p7 p10 p6 / \ p9 p5 / ''' with helpers.log2str(): pp = PyPPL({'_log': {'_file': None}}) p_resume1 = Proc() p_resume2 = Proc() p_resume3 = Proc() p_resume4 = Proc() p_resume5 = Proc() p_resume6 = Proc() p_resume7 = Proc() p_resume8 = Proc() p_resume9 = Proc() p_resume10 = Proc() p_resume9.depends = p_resume7 p_resume8.depends = p_resume7 p_resume7.depends = p_resume3, p_resume6 p_resume3.depends = p_resume2 p_resume6.depends = p_resume4, p_resume5 p_resume3.depends = p_resume2 p_resume4.depends = p_resume2 p_resume2.depends = p_resume1 p_resume10.depends = p_resume1 yield pp, [p_resume1, p_resume5], [p_resume2, p_resume6], True, [ p_resume1, p_resume5 ], PyPPLProcRelationError, 'One of the routes cannot be achived from resumed processes: \'p_resume10 <- \[p_resume1\]\'' yield pp, [p_resume1, p_resume5], [p_resume1, p_resume6], True, [p_resume5] yield pp, [p_resume1, p_resume5], [p_resume1, p_resume3, p_resume6 ], False, [p_resume5, p_resume2, p_resume4] def test_resume(self, pp, starts, resumes, plus, skips, exception=None, msg=None): pp.tree = ProcTree() for node in ProcTree.NODES.values(): node.proc.resume = '' pp.start(starts) helpers.log2sys() skip = 'skip+' if plus else 'skip' resume = 'resume+' if plus else 'resume' if exception: self.assertRaisesRegex(exception, msg, pp._resume, *resumes, plus=plus) else: pp._resume(*resumes, plus=plus) for r in resumes: self.assertEqual(r.resume, resume) for s in skips: self.assertEqual(s.resume, skip) def dataProvider_testResume1(self): with helpers.log2str(): pp = PyPPL({'_log': {'file': None}}) pResume11 = Proc() pResume12 = Proc() pResume13 = Proc() pResume13.depends = pResume12 pResume12.depends = pResume11 yield pp, pResume11, [] yield pp, pResume11, [pResume12] def testResume1(self, pp, start, procs): pp.tree = ProcTree() pp.start(start).resume(procs) if not procs: for node in ProcTree.NODES.values(): self.assertEqual(node.proc.resume, '') else: for node in ProcTree.NODES.values(): self.assertIn(node.proc.resume, ['', 'skip', 'resume']) def dataProvider_testResume2(self): with helpers.log2str(): pp = PyPPL({'_log': {'file': None}}) pResume21 = Proc() pResume22 = Proc() pResume23 = Proc() pResume23.depends = pResume22 pResume22.depends = pResume21 yield pp, pResume21, [] yield pp, pResume21, [pResume22] def testResume2(self, pp, start, procs): pp.tree = ProcTree() pp.start(start).resume2(procs) if not procs: for proc in ProcTree.NODES.keys(): if not proc.id.startswith('pResume2'): continue self.assertEqual(proc.resume, '') else: for proc in ProcTree.NODES.keys(): if not proc.id.startswith('pResume2'): continue self.assertIn(proc.resume, ['', 'skip+', 'resume+']) ''' def dataProvider_testGetProfile(self): yield {'log': {'file': None}, 'proc': {'id': 'a'}}, 'proc', {}, PyPPLConfigError, 'Cannot set a universal id for all process in configuration: \'a\'' yield {'log': {'file': None}, 'proc': {}, 'sge': {}}, 'sge', {'runner': 'sge'} yield {'log': {'file': None}, 'proc': {}, 'haha': {'tag': 'new'}}, 'haha', {'runner': 'local', 'tag': 'new'}, None, None, [ 'WARNING', 'No runner specified in profile \'haha\', will use local runner.' ] def testGetProfile(self, inconf, profile, config, exception = None, msg = None, errs = []): PyPPL.DEFAULT_CFGFILES = [] with helpers.log2str(): pp = PyPPL(inconf) if exception: self.assertRaisesRegex(exception, msg, pp._getProfile, profile) else: with helpers.log2str(levels = 'all') as (out, err): c = pp._getProfile(profile) stderr = err.getvalue() self.assertEqual(c, config) for err in errs: self.assertIn(err, stderr) ''' def dataProvider_testShowAllRoutes(self): ''' / p3 --- \ p1 -- p2 \ / p8 \ \ p4 \ p7 p10 p6 / \ p9 p5 / ''' pShowAllRoutes1 = Proc() pShowAllRoutes2 = Proc() pShowAllRoutes3 = Proc() pShowAllRoutes4 = Proc() pShowAllRoutes5 = Proc() pShowAllRoutes6 = Proc() pShowAllRoutes7 = Proc() pShowAllRoutes8 = Proc() pShowAllRoutes9 = Proc() pShowAllRoutes10 = Proc() pShowAllRoutes3.depends = pShowAllRoutes2 pShowAllRoutes6.depends = pShowAllRoutes4, pShowAllRoutes5 pShowAllRoutes3.depends = pShowAllRoutes2 pShowAllRoutes4.depends = pShowAllRoutes2 pShowAllRoutes2.depends = pShowAllRoutes1 pShowAllRoutes10.depends = pShowAllRoutes1 aAggr = Aggr(pShowAllRoutes7, pShowAllRoutes8, pShowAllRoutes9) aAggr.starts = [aAggr.pShowAllRoutes7] aAggr.pShowAllRoutes8.depends = aAggr.pShowAllRoutes7 aAggr.pShowAllRoutes9.depends = aAggr.pShowAllRoutes7 aAggr.depends2 = [pShowAllRoutes3, pShowAllRoutes6] yield [pShowAllRoutes1, pShowAllRoutes5], [ 'DEBUG', '* pShowAllRoutes1 -> pShowAllRoutes10', 'pShowAllRoutes1 -> pShowAllRoutes2 -> pShowAllRoutes3 -> [aAggr]', '* pShowAllRoutes1 -> pShowAllRoutes2 -> pShowAllRoutes4 -> pShowAllRoutes6 -> [aAggr]', '* pShowAllRoutes5 -> pShowAllRoutes6 -> [aAggr]' ] def testShowAllRoutes(self, start, errs): with helpers.log2str(): pp = PyPPL({'_log': {'file': None}}) pp.start(start) with helpers.log2str(levels='all') as (out, err): pp.showAllRoutes() stderr = err.getvalue() for err in errs: self.assertIn(err, stderr) stderr = stderr[(stderr.find(err) + len(err)):] def dataProvider_testFlowchart(self): ''' / p3 --- \ p1 -- p2 \ / p8 \ \ p4 \ p7 p10 p6 / \ p9 p5 / ''' pFlowchart1 = Proc() pFlowchart2 = Proc() pFlowchart3 = Proc() pFlowchart4 = Proc() pFlowchart5 = Proc() pFlowchart6 = Proc() pFlowchart7 = Proc() pFlowchart8 = Proc() pFlowchart9 = Proc() pFlowchart10 = Proc() pFlowchart3.depends = pFlowchart2 pFlowchart6.depends = pFlowchart4, pFlowchart5 pFlowchart3.depends = pFlowchart2 pFlowchart4.depends = pFlowchart2 pFlowchart2.depends = pFlowchart1 pFlowchart10.depends = pFlowchart1 aAggr = Aggr(pFlowchart7, pFlowchart8, pFlowchart9) aAggr.starts = [aAggr.pFlowchart7] aAggr.pFlowchart8.depends = aAggr.pFlowchart7 aAggr.pFlowchart9.depends = aAggr.pFlowchart7 aAggr.depends2 = [pFlowchart3, pFlowchart6] dotfile = path.join(self.testdir, 'test.dot') fcfile = path.join(self.testdir, 'test.svg') yield [pFlowchart1, pFlowchart5], fcfile, dotfile, [ 'DEBUG', '* pFlowchart1 -> pFlowchart10', 'pFlowchart1 -> pFlowchart2 -> pFlowchart3 -> [aAggr]', '* pFlowchart1 -> pFlowchart2 -> pFlowchart4 -> pFlowchart6 -> [aAggr]', '* pFlowchart5 -> pFlowchart6 -> [aAggr]', 'INFO', 'Flowchart file saved to: %s' % fcfile, 'DOT file saved to: %s' % dotfile, ] yield [pFlowchart1, pFlowchart5], fcfile, None, [ 'DEBUG', '* pFlowchart1 -> pFlowchart10', 'pFlowchart1 -> pFlowchart2 -> pFlowchart3 -> [aAggr]', '* pFlowchart1 -> pFlowchart2 -> pFlowchart4 -> pFlowchart6 -> [aAggr]', '* pFlowchart5 -> pFlowchart6 -> [aAggr]', 'INFO', 'Flowchart file saved to: %s' % fcfile, 'DOT file saved to: %s' % dotfile, ] @unittest.skipIf(not helpers.moduleInstalled('graphviz'), 'graphviz not installed.') def testFlowchart(self, start, fcfile, dotfile, errs=[]): with helpers.log2str(): pp = PyPPL({'_log': {'file': None}}) pp.start(start) with helpers.log2str(levels='all') as (out, err): pp.flowchart(fcfile=fcfile, dotfile=dotfile) self.assertTrue(path.isfile(fcfile)) self.assertTrue( path.isfile(dotfile or path.splitext(fcfile)[0] + '.dot')) stderr = err.getvalue() for err in errs: self.assertIn(err, stderr) stderr = stderr[(stderr.find(err) + len(err)):] def dataProvider_testRun(self): ''' / p3 --- \ p1 -- p2 \ / p8 \ \ p4 \ p7 p10 p6 / \ p9 p5 / ''' pRun0 = Proc() pRun1 = pRun0.copy() pRun2 = Proc() pRun3 = Proc() pRun4 = Proc() pRun5 = Proc() pRun6 = Proc() pRun7 = Proc() pRun8 = Proc() pRun9 = Proc() pRun10 = Proc() pRun3.depends = pRun2 pRun6.depends = pRun4, pRun5 pRun3.depends = pRun2 pRun4.depends = pRun2 pRun2.depends = pRun1 pRun10.depends = pRun1 aAggr = Aggr(pRun7, pRun8, pRun9) aAggr.starts = [aAggr.pRun7] aAggr.pRun8.depends = aAggr.pRun7 aAggr.pRun9.depends = aAggr.pRun7 aAggr.depends2 = [pRun3, pRun6] yield [pRun1, pRun5], 'profile', 'sge', [ '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', 'pRun1 (pRun0): No description.', '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', 'pRun2: No description.', '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', 'pRun3: No description.', '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', 'pRun4: No description.', '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', 'pRun5: No description.', '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', 'pRun6: No description.', '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', 'pRun10: No description.', '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', 'pRun7.5gPF@aAggr: No description.', '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', 'pRun8.5gPF@aAggr: No description.', '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', 'pRun9.5gPF@aAggr: No description.', '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', ] # pRun21 -> pRun22 # pRun23 -/ # \-> pRun24 pRun21 = Proc() pRun23 = Proc() pRun22 = Proc() pRun22.depends = pRun21, pRun23 yield [pRun21], 'profile', 'local', [ "pRun22 won't run as path can't be reached: pRun22 <- pRun23" ] def testRun(self, start, profile, runner, errs=[]): with helpers.log2str(): pp = PyPPL({ '_log': { 'file': None }, 'profile': { 'ppldir': self.testdir, 'runner': runner } }) import sys pp.start(start) argv = sys.argv sys.argv = [sys.argv[0]] helpers.log2sys() with helpers.log2str(levels='all') as (out, err): pp.run(profile) sys.argv = argv for s in start: self.assertEqual(s.runner, runner) stderr = err.getvalue() for err in errs: self.assertIn(err, stderr) stderr = stderr[(stderr.find(err) + len(err)):] def dataProvider_testRegisterRunner(self): yield RunnerLocal, 'local' yield RunnerSsh, 'ssh' yield RunnerDry, 'dry' yield RunnerSge, 'sge' yield RunnerSlurm, 'slurm' class xxx(object): pass yield xxx, 'xxx' class RunnerYyy(object): pass yield RunnerYyy, 'yyy' def testRegisterRunner(self, runner, name): if name in PyPPL.RUNNERS: del PyPPL.RUNNERS[name] PyPPL.registerRunner(runner) self.assertIs(PyPPL.RUNNERS[name], runner)
class TestTemplateJinja2(testly.TestCase): def dataProvider_testInit(self): yield '', {} yield '{{a}}', {'a': 1} @unittest.skipIf(not helpers.moduleInstalled('jinja2'), 'Jinja2 not installed.') def testInit(self, source, envs): import jinja2 tpl = TemplateJinja2(source, **envs) helpers.assertTextEqual(self, tpl.source, source) self.assertDictContains(Template.DEFAULT_ENVS, tpl.envs) self.assertDictContains(envs, tpl.envs) self.assertIsInstance(tpl.engine, jinja2.Template) def dataProvider_testStr(self): yield '', yield 'a', yield '{{a}}', yield '{{a}}\n{{b}}', @unittest.skipIf(not helpers.moduleInstalled('jinja2'), 'Jinja2 not installed.') def testStr(self, source): tpl = TemplateJinja2(source) lines = source.splitlines() if len(lines) <= 1: self.assertEqual(str(tpl), 'TemplateJinja2 < %s >' % ''.join(lines)) else: helpers.assertTextEqual( self, str(tpl), '\n'.join(['TemplateJinja2 <<<'] + ['\t' + line for line in tpl.source.splitlines()] + ['>>>'])) def dataProvider_testRender(self): # 0 yield '{{name}}', {'name': 'John'}, 'John' yield '{{names[0]}}', {'names': ['John', 'Tom']}, 'John' yield '{{concate(v1, v2)}}', { 'v1': 'hello', 'v2': 'world', 'concate': lambda x, y: x + y }, 'helloworld' yield '{{R(v23)}}', {'v23': '"FALSE"'}, '\'"FALSE"\'' yield '{{R(v3)}}', {'v3': 'false'}, "FALSE" # 5 yield '{{realpath(v4)}}', {'v4': __file__}, path.realpath(__file__) #yield ('{{readlink(v5)}}', {'v5': path.join(path.dirname(path.realpath(path.abspath(__file__))), 'helpers.py')yield , path.relpath(path.jo(path.dirname(path.dirname(path.abspath(__file__))), 'bin', 'helpers.py'))), yield '{{dirname(v6)}}', {'v6': '/a/b/c'}, '/a/b' yield '{{basename(v7)}}{{bn(v7)}}', {'v7': '/a/b/c.txt'}, 'c.txtc.txt' yield '{{basename(v8)}}{{bn(v8)}}', { 'v8': '/a/b/c[1].txt' }, 'c.txtc.txt' yield '{{basename(v9, v9b)}}{{bn(v9, v9b)}}', { 'v9': '/a/b/c.txt', 'v9b': True }, 'c.txtc.txt' # 10 yield '{{basename(v10, v10b)}}{{bn(v10, v10b)}} {{ext(v10)}}', { 'v10': '/a/b/c[1].txt', 'v10b': True }, 'c[1].txtc[1].txt .txt' yield '{{filename(v11)}}{{fn(v11)}} {{prefix(v11)}}', { 'v11': '/a/b/a.txt' }, 'aa /a/b/a' yield '{{filename(v12)}}{{fn(v12)}}', {'v12': '/a/b/b[1].txt'}, 'bb' yield '{{filename(v13, v13b)}}{{fn(v13, v13b)}}', { 'v13': '/a/b/d.txt', 'v13b': True }, 'dd' yield '{{filename(v14, v14b)}}{{fn(v14, v14b)}}', { 'v14': '/a/b/c[1].txt', 'v14b': True }, 'c[1]c[1]' # 15 yield '{{R(var1)}}', {'var1': 'NULL'}, 'NULL' yield '{{R(var2)}}', {'var2': 'abc'}, "'abc'" yield '{% for var in varlist %}{{R(var)}}{% endfor %}', { 'varlist': ['abc', 'True', 1, False] }, "'abc'TRUE1FALSE" yield '{% if bool(var3) %}1{% else %}0{% endif %}', { 'var3': 'abc', 'bool': bool }, '1' yield '{% for k,v in data.items() %}{{k}}:{{v}}{% endfor %}', { 'data': { 'a': 1, 'b': 2 } }, 'a:1b:2' # 20 yield '{{quote(a)}}', {'a': ''}, '""' yield '{{R(x)}}', { 'x': OrderedDict([(u'key1', 'val1'), ('key2', u'val2')]) }, "list(key1='val1',key2='val2')" yield '{{asquote(b)}}', {'b': [1, 2]}, '"1" "2"' yield '{{acquote(c)}}', {'c': [1, 2]}, '"1", "2"' yield '{{squote(d)}}', {'d': '1'}, "'1'" # 25 yield '{{json(e["f"])}}', {'e': {'f': [1, 2]}}, '[1, 2]' yield '{{os.path.join(g,h)}}', { 'g': 'a', 'h': 'b', 'os': __import__('os') }, 'a/b' yield """ #!/usr/bin/env python {% if x %} {% for y in ylist %} {{y}} {% endfor %} {% endif %} """, { 'x': True, 'ylist': [1, 2, 3, 4, 5] }, """ #!/usr/bin/env python\n\t\t\n\t\t 1\n\t\t 2\n\t\t 3\n\t\t 4\n\t\t 5\n\t\t \n\t\t""" yield '{{read(a)}}', {'a': __file__}, helpers.readFile(__file__) file2read = path.join(path.dirname(__file__), 'helpers.py') yield '{{"\\n".join(readlines(a))}}', { 'a': file2read }, helpers.readFile( file2read, lambda x: '\n'.join(str(y) for y in x.splitlines() if y)) @unittest.skipIf(not helpers.moduleInstalled('jinja2'), 'Jinja2 not installed.') def testRender(self, s, e, out): t = TemplateJinja2(s) helpers.assertTextEqual(self, t.render(e), out)