def test_buildscript(tmpdir, caplog):
p13 = Proc()
p13.props.template = TemplateLiquid
p13._buildScript()
assert 'No script specified' in caplog.text
scriptfile = tmpdir / 'test_buildscript.txt'
scriptfile.write_text('# script')
p13.script = 'file:%s' % scriptfile
fs.remove(scriptfile)
with pytest.raises(ProcScriptError):
p13._buildScript()
scriptfile.write_text('''# script
# PYPPL INDENT REMOVE
abs
bin
# PYPPL INDENT KEEP
callable
def''')
caplog.clear()
p13.lang = 'python'
p13._buildScript()
assert 'Using template file: ' in caplog.text
assert p13.script.render() == '''#!/usr/bin/env python
def test_readconfig_preload(tmpdir):
config._load({'xyz': {'runner': 'sge', 'forks': 50}})
p18 = Proc()
p18._readConfig('xyz', Config())
assert p18.runner == 'sge'
assert p18.forks == 50
assert p18.config.runner == 'xyz'
def test_suffix_input(tmpdir):
p82 = Proc()
p82.input = 'a'
p82.input = [1]
sigs = OBox()
sigs.argv0 = str(Path(sys.argv[0]).resolve())
sigs.id = 'p82'
sigs.tag = 'notag'
assert p82.config.input == {'a': [1]}
sigs.input = {'a': "[1]"}
assert p82.suffix == uid(sigs.to_json())
def test_savesettings(tmpdir, caplog):
import yaml
p14 = Proc()
p14.props.template = TemplateLiquid
p14.props.workdir = tmpdir / 'test_savesettings_p14'
p14.props.workdir.mkdir()
p14._saveSettings()
assert 'Settings saved to: ' in caplog.text
with (p14.props.workdir / 'proc.settings.yaml').open() as f:
saved = yaml.load(f, Loader=yaml.Loader)
assert saved['workdir'] == p14.props.workdir
def test_suffix_compute(tmpdir):
p76 = Proc()
p81 = Proc()
p81.depends = p76
p81.input = 'a'
sigs = OBox()
sigs.argv0 = str(Path(sys.argv[0]).resolve())
sigs.id = 'p81'
sigs.tag = 'notag'
sigs.input = 'a'
sigs.depends = ['p76#' + p76.suffix]
assert p81.suffix == uid(sigs.to_json())
def test_proc_getattr(tmpdir):
p2 = Proc()
with pytest.raises(ProcAttributeError):
p2.x
# alias
p2.afterCmd = 'after cmd'
assert p2.postCmd == 'after cmd'
assert p2.id == 'p2'
assert p2.envs == {}
def test_runcmd(tmpdir, caplog):
p19 = Proc()
p19.props.template = TemplateLiquid
p19._runCmd('pre')
assert 'Running ' not in caplog.text
p19.preCmd = 'echo "Hello world!"'
p19._runCmd('before')
assert 'Running <beforeCmd> ...' in caplog.text
assert 'Hello world!' in caplog.text
p19.postCmd = 'nosuchcommand'
with pytest.raises(ProcRunCmdError):
p19._runCmd('post')
def test_readconfig_preset(tmpdir):
p181 = Proc()
p181.runner = 'xyz'
config = Config()
cfile = tmpdir / 'test_readconfig_preset.ini'
cfile.write_text("""
[xyz]
runner: sge
forks: 50
""")
config._load(cfile)
p181._readConfig('', config)
assert p181.runner == 'sge'
assert p181.forks == 50
assert p181.config.runner == 'xyz'
def test_preruntidy(tmpdir, caplog):
p20 = Proc()
p20.input = 'a, b'
p20.input = [(1, 2), (3, 4), (5, 6)]
p20.callfront = lambda p: setattr(p, 'forks', 20)
p20._preRunTidy()
assert 'Calling callfront ...' in caplog.text
assert p20.forks == 20
assert p20.size == 3
def test_proc_init(tmpdir):
p1 = Proc(id='p1', tag='tag', desc='desc', preCmd='ls')
assert p1.config.id == 'p1'
assert p1.config.args == {}
assert p1.config.callfront == None
assert p1.config.callback == None
assert p1.props.channel == []
assert p1.config.depends == []
assert p1.props.depends == []
assert p1.props.echo == {}
assert p1.props.expart == []
assert p1.props.expect == None
#assert p1.config.hide == False
assert p1.config.input == ''
assert p1.props.input == {}
assert p1.props.jobs == []
assert p1.props.lock == None
assert p1.props.ncjobids == []
assert p1.props.origin == 'p1'
assert p1.config.output == ''
assert p1.props.output == {}
assert p1.props.procvars == {}
assert p1.props.rc == [0]
assert p1.config.resume == ''
assert p1.props.runner == 'local'
assert p1.props.script == None
assert p1.props._suffix == ''
assert p1.props.template == None
assert p1.props.timer == None
assert p1.config.envs == {}
assert p1.props.workdir == ''
assert p1.props.sets == set(['beforeCmd'])
with pytest.raises(ProcTagError):
Proc(tag='a b')
with pytest.raises(ProcAttributeError):
Proc(depends=1)
with pytest.raises(ProcAttributeError):
Proc(xxx=1)
def test_copy(tmpdir):
p5 = Proc()
p6 = p5.copy(id='p6', tag='new', desc='desc')
assert p6.sets == {'id', 'tag', 'desc'}
assert p6.id == 'p6'
assert p6.tag == 'new'
assert p6.desc == 'desc'
assert p6.workdir == ''
assert p6.resume == ''
assert p6.config.args == {}
assert p6.config.args is not p5.config.args
assert p6.envs == {}
assert p6.envs is not p5.envs
assert p6.depends == []
assert p6.jobs == []
assert p6.ncjobids == []
assert p6.origin == 'p5'
assert p6.props._suffix == ''
assert p6.channel == []
assert p6.procvars == {}
assert p6.procvars is not p5.procvars
def test_buildjobs(tmpdir, caplog):
p15 = Proc()
p15.props.jobs = [None, None]
p15._buildJobs()
assert 'No data found for jobs, process will be skipped.' in caplog.text
p15.props.input['a'] = ('var', [1, 2, 3])
p15.props.jobs = [1, 2, 3]
p15.props.runner = 'nosuchrunner'
with pytest.raises(ProcAttributeError):
p15._buildJobs()
p15.props.runner = 'local'
p15._buildJobs()
assert len(p15.props.jobs) == 3
assert p15.props.jobs[0].__class__.__name__ == 'RunnerLocal'
def test_buildprocvars(tmpdir, caplog):
p11 = Proc()
p11.args = {'a': 1}
p11._buildProcVars()
assert 'p11: a => 1' in caplog.text
assert 'p11: runner => local' in caplog.text
assert 'p11: size => 0' in caplog.text
assert 'exdir' not in caplog.text
assert p11.procvars['args'] is p11.args
p11.config.args = {'a': 1}
p11.exdir = 'abc'
caplog.clear()
p11._buildProcVars()
assert 'p11: exdir => abc' in caplog.text
assert p11.procvars['args'] == p11.args
assert p11.procvars['args'] is not p11.args
def test_run(tmpdir, caplog):
sys.argv = ['pytest']
p24 = Proc()
p24.resume = 'resume'
p24.props.workdir = tmpdir / 'test_run_p24'
fs.mkdir(p24.workdir)
(p24.workdir / 'proc.settings.yaml').write_text('input: ')
p24.run('dry', Config())
assert 'Previous processes skipped.' in caplog.text
assert p24.runner == 'dry'
p25 = Proc()
p25.resume = 'skip'
caplog.clear()
p25.run(None, Config())
assert 'Pipeline will resume from future processes.' in caplog.text
p25.resume = 'skip+'
caplog.clear()
p25.props.workdir = tmpdir / 'test_run_p25'
fs.mkdir(p25.workdir)
(p25.workdir / 'proc.settings.yaml').write_text('input: ')
p25.run(None, Config())
assert 'Data loaded, pipeline will resume from future processes.' in caplog.text
def test_runjobs(tmpdir):
p21 = Proc()
p21.forks = 3
p21.input = 'a, b'
p21.input = [(1, 2), (3, 4), (5, 6)]
p21.output = 'a:{{i.a}}, b:{{i.b}}'
p21.script = 'echo Hello world!'
p21._preRunTidy()
p21._runJobs()
assert p21.channel == [('1', '2'), ('3', '4'), ('5', '6')]
assert p21.channel.a.flatten() == ['1', '3', '5']
assert p21.channel.b.flatten() == ['2', '4', '6']
def test_postruntidy(tmpdir, caplog):
p22 = Proc()
p22.resume = 'skip+'
p22.callback = lambda p: p.props.update({'channel': p.channel.cbind(1, 2)})
p22._postRunTidy()
assert p22.channel == [(1, 2)]
p23 = Proc()
p23.forks = 5
p23.input = 'a, b'
p23.input = [(1, 2), (3, 4), (5, 6), (7, 8), (9, 10)]
p23.output = 'a:{{i.a}}, b:{{i.b}}'
p23.script = 'echo Hello world!'
p23.errhow = 'ignore'
p23.callback = lambda p: setattr(p, 'forks', 10)
p23._preRunTidy()
p23._runJobs()
p23.jobs[0].state = STATES.BUILTFAILED
p23.jobs[1].state = STATES.SUBMITFAILED
p23.jobs[2].state = STATES.DONE
p23.jobs[3].state = STATES.DONECACHED
p23.jobs[4].state = STATES.ENDFAILED
p23._postRunTidy()
assert ' Jobs (Cached: 1, Succ: 1, B.Fail: 1, S.Fail: 1, R.Fail: 1)' in caplog.text
assert 'Failed but ignored (totally 3).' in caplog.text
assert p23.forks == 10
p23.errhow = 'terminate'
caplog.clear()
with pytest.raises(SystemExit):
p23._postRunTidy()
assert 'Cached: 4' in caplog.text
assert 'Succeeded: 3' in caplog.text
assert 'Building failed: 1' in caplog.text
assert 'Submission failed: 2' in caplog.text
assert 'Running failed: 5' in caplog.text
def test_procset():
p62 = Proc()
assert p62.procset is None
p62.config.tag = 'notag@ps'
assert p62.procset == 'ps'
def test_name():
p61 = Proc()
assert p61.name() == 'p61'
ps = ProcSet(p61)
assert ps.p61.tag == 'notag@ps'
assert ps.p61.name(True) == 'p61@ps'
def test_suffix_preset(tmpdir):
p75 = Proc()
p8 = Proc()
p8.depends = p75
p8.props._suffix = '123'
assert p8.suffix == '123'
def test_size(tmpdir):
p7 = Proc()
assert p7.size == 0
p7.props.jobs = [1, 2, 3]
assert p7.size == 3
def test_readconfig(tmpdir):
p16 = Proc()
assert p16.id == 'p16'
# nothing updated
p16._readConfig(None, Config())
assert p16.id == 'p16'
config = Config()
config._load({'f20': {'forks': 20}})
p16.forks = 10
p16._readConfig({'forks': 30}, config)
assert p16.id == 'p16'
assert p16.forks == 10
assert p16.runner == 'local'
assert p16.config.runner == '__tmp__'
p17 = Proc()
p17.forks = 10
config = Config()
config._load({'f30': {'forks': 20}})
# no such profile in config
p17._readConfig('dry', config)
assert p17.forks == 10
assert p17.runner == 'dry'
assert p17.config.runner == 'dry'
def test_buildprops(tmpdir):
from pyppl import ProcTree
p9 = Proc()
p91 = Proc(id='p9')
ProcTree.register(p9)
ProcTree.register(p91)
with pytest.raises(ProcTreeProcExists):
p91._buildProps()
p9.id = 'p89'
p9.template = TemplateLiquid
p9.ppldir = Path(tmpdir / 'test_buildprops')
p9.rc = '0,1'
p9.workdir = tmpdir / 'p8'
p9.exdir = tmpdir / 'p8.exdir'
p9.echo = True
p9.expect = 'ls'
p9.expart = 'outfile'
p9._buildProps()
assert p9.template is TemplateLiquid
assert p9.rc == [0, 1]
assert p9.workdir.exists()
assert fs.exists(p9.exdir)
assert p9.echo == dict(jobs=[0], type=dict(stderr=None, stdout=None))
assert p9.expect.render() == 'ls'
assert len(p9.expart) == 1
assert p9.expart[0].render() == 'outfile'
p9.template = None
p9.rc = 1
p9.sets.remove('workdir')
p9.props.workdir = None
p9.echo = False
p9._buildProps()
assert p9.template is TemplateLiquid
assert p9.rc == [1]
assert Path(
p9.workdir) == Path(p9.ppldir) / ('PyPPL.p89.notag.%s' % p9.suffix)
assert p9.echo == dict(jobs=[], type=dict(stderr=None, stdout=None))
p9.template = 'liquid'
p9.rc = [0, 1, 2]
p9.echo = 'stderr'
p9._buildProps()
assert p9.template is TemplateLiquid
assert p9.rc == [0, 1, 2]
assert p9.echo == dict(jobs=[0], type=dict(stderr=None))
fs.remove(p9.workdir)
p9.resume = 'resume'
with pytest.raises(ProcAttributeError):
p9._buildProps()
p9.echo = dict(type='stderr')
p9.resume = ''
p9._buildProps()
assert p9.echo == dict(jobs=[0], type=dict(stderr=None))
p9.echo = dict(jobs='0,1')
p9._buildProps()
assert p9.echo == dict(jobs=[0, 1], type=dict(stderr=None, stdout=None))
p9.echo = dict(jobs='0,1', type=dict(all=r'^log'))
p9._buildProps()
assert p9.echo == dict(jobs=[0, 1],
type=dict(stderr=r'^log', stdout=r'^log'))
def test_buildinput(tmpdir, caplog):
p10 = Proc()
p10.input = 'a, b:file, '
p10.input = ('1', 'infile')
p10._buildInput()
assert len(p10.input) == 2
assert p10.input['a'] == ('var', ['1'])
assert p10.input['b'] == ('file', ['infile'])
assert p10.size == 1
p10.input = 'a:x:y'
with pytest.raises(ProcInputError):
p10._buildInput()
p101 = Proc()
p101.props.channel = Channel.create([(1, 3), (2, 4)])
p10.depends = p101
p10.input = 'a, b, c'
p10.input = lambda ch: ch.cbind(1).cbind(2)
caplog.clear()
p10._buildInput()
assert 'Not all data are used as input' in caplog.text
assert len(p10.input) == 3
assert p10.size == 2
assert p10.input['a'] == ('var', [1, 2])
assert p10.input['b'] == ('var', [3, 4])
assert p10.input['c'] == ('var', [1, 1])
p10.input = 'a:files, b:files, c'
p10.input = Channel.create([['infile1'], ['infile2']])
p10._buildInput()
assert 'No data found for input key "b"' in caplog.text
assert 'No data found for input key "c"' in caplog.text
caplog.clear()
assert len(p10.input) == 3
assert p10.size == 2
assert p10.input['a'] == ('files', [['infile1'], ['infile2']])
assert p10.input['b'] == ('files', [[], []])
assert p10.input['c'] == ('var', ['', ''])
p10.props.template = TemplateLiquid
p10.props.workdir = tmpdir / 'test_buildinput_p10'
p10.resume = 'resume'
fs.remove(Path(p10.workdir) / 'proc.settings.yaml')
with pytest.raises(ProcInputError):
p10._buildInput()
fs.mkdir(p10.workdir)
p10.props.input = OBox()
p10.input['a'] = ('files', [['infile1'], ['infile2']])
p10.input['b'] = ('files', [[], []])
p10.input['c'] = ('var', ['', ''])
p10._saveSettings()
p10.props.input = None
p10._buildInput()
assert len(p10.input) == 3
assert p10.size == 2
assert p10.input['a'] == ('files', [['infile1'], ['infile2']])
assert p10.input['b'] == ('files', [[], []])
assert p10.input['c'] == ('var', ['', ''])
def test_buildinput_empty(tmpdir):
sys.argv = ['pytest']
p102 = Proc()
p102._buildInput()
assert p102.size == 0
assert p102.jobs == []
def test_repr(tmpdir):
p4 = Proc()
assert p4.id == 'p4'
assert p4.tag == 'notag'
assert repr(p4).startswith('<Proc(p4) @')
def test_proc_setattr(tmpdir, caplog):
p3 = Proc()
with pytest.raises(ProcAttributeError):
p3.x = 1
p3.preCmd = 'ls'
assert 'beforeCmd' in p3.sets
p4 = Proc()
ps = ProcSet(p4)
p3.depends = p4
assert p3.depends == [p4]
p3.depends = [p4], ps
assert p3.depends == [p4, ps.p4]
assert p4.tag == 'notag'
assert ps.p4.tag == 'notag@ps'
with pytest.raises(ProcAttributeError):
p3.depends = p3
with pytest.raises(ProcAttributeError):
p3.depends = object()
p3.script = 'file:%s' % Path(__file__).name
assert p3.config.script == 'file:%s' % Path(__file__).resolve()
with pytest.raises(ProcAttributeError):
p3.script = 'file:nosuchfile'
p3.args = {'a': 1}
assert p3.args.a == 1
p3.input = 'a, b'
p3.input = [1, 2]
assert p3.config.input == {'a, b': [1, 2]}
p3.input = {'a': [1], 'b': [2]}
caplog.clear()
p3.input = [3, 4]
assert 'Previous input is a dict with multiple keys and key order may be changed.' in caplog.text
assert 'Now the key order is:' in caplog.text
p3.runner = 'sge'
assert p3.config.runner == 'sge'
assert p3.props.runner == 'sge'
with pytest.raises(ProcAttributeError):
p3.tag = 'a@b'
def test_buildoutput(tmpdir):
p12 = Proc()
p12.output = 'a'
with pytest.raises(ProcOutputError):
p12._buildOutput()
p12.output = 'a:b:c:d'
with pytest.raises(ProcOutputError):
p12._buildOutput()
p12.output = {'a': '1', 'b': 2}
with pytest.raises(ProcOutputError):
p12._buildOutput()
p12.output = 'a:b:c'
with pytest.raises(ProcOutputError):
p12._buildOutput()
p12.output = 'a:1, b:file:infile'
p12.props.template = TemplateLiquid
p12._buildOutput()
assert len(p12.output) == 2
assert p12.output['a'][0] == 'var'
assert p12.output['a'][1].render() == '1'
assert p12.output['b'][0] == 'file'
assert p12.output['b'][1].render() == 'infile'
