def test_buildinput(tmpdir, caplog):
p10 = Proc()
p10.input = 'a, b:file, '
p10.input = ('1', 'infile')
p10._buildInput()
assert len(p10.input) == 2
assert p10.input['a'] == ('var', ['1'])
assert p10.input['b'] == ('file', ['infile'])
assert p10.size == 1
p10.input = 'a:x:y'
with pytest.raises(ProcInputError):
p10._buildInput()
p101 = Proc()
p101.props.channel = Channel.create([(1, 3), (2, 4)])
p10.depends = p101
p10.input = 'a, b, c'
p10.input = lambda ch: ch.cbind(1).cbind(2)
caplog.clear()
p10._buildInput()
assert 'Not all data are used as input' in caplog.text
assert len(p10.input) == 3
assert p10.size == 2
assert p10.input['a'] == ('var', [1, 2])
assert p10.input['b'] == ('var', [3, 4])
assert p10.input['c'] == ('var', [1, 1])
p10.input = 'a:files, b:files, c'
p10.input = Channel.create([['infile1'], ['infile2']])
p10._buildInput()
assert 'No data found for input key "b"' in caplog.text
assert 'No data found for input key "c"' in caplog.text
caplog.clear()
assert len(p10.input) == 3
assert p10.size == 2
assert p10.input['a'] == ('files', [['infile1'], ['infile2']])
assert p10.input['b'] == ('files', [[], []])
assert p10.input['c'] == ('var', ['', ''])
p10.props.template = TemplateLiquid
p10.props.workdir = tmpdir / 'test_buildinput_p10'
p10.resume = 'resume'
fs.remove(Path(p10.workdir) / 'proc.settings.yaml')
with pytest.raises(ProcInputError):
p10._buildInput()
fs.mkdir(p10.workdir)
p10.props.input = OBox()
p10.input['a'] = ('files', [['infile1'], ['infile2']])
p10.input['b'] = ('files', [[], []])
p10.input['c'] = ('var', ['', ''])
p10._saveSettings()
p10.props.input = None
p10._buildInput()
assert len(p10.input) == 3
assert p10.size == 2
assert p10.input['a'] == ('files', [['infile1'], ['infile2']])
assert p10.input['b'] == ('files', [[], []])
assert p10.input['c'] == ('var', ['', ''])
def test_run(tmpdir, caplog):
sys.argv = ['pytest']
p24 = Proc()
p24.resume = 'resume'
p24.props.workdir = tmpdir / 'test_run_p24'
fs.mkdir(p24.workdir)
(p24.workdir / 'proc.settings.yaml').write_text('input: ')
p24.run('dry', Config())
assert 'Previous processes skipped.' in caplog.text
assert p24.runner == 'dry'
p25 = Proc()
p25.resume = 'skip'
caplog.clear()
p25.run(None, Config())
assert 'Pipeline will resume from future processes.' in caplog.text
p25.resume = 'skip+'
caplog.clear()
p25.props.workdir = tmpdir / 'test_run_p25'
fs.mkdir(p25.workdir)
(p25.workdir / 'proc.settings.yaml').write_text('input: ')
p25.run(None, Config())
assert 'Data loaded, pipeline will resume from future processes.' in caplog.text
def test_postruntidy(tmpdir, caplog):
p22 = Proc()
p22.resume = 'skip+'
p22.callback = lambda p: p.props.update({'channel': p.channel.cbind(1, 2)})
p22._postRunTidy()
assert p22.channel == [(1, 2)]
p23 = Proc()
p23.forks = 5
p23.input = 'a, b'
p23.input = [(1, 2), (3, 4), (5, 6), (7, 8), (9, 10)]
p23.output = 'a:{{i.a}}, b:{{i.b}}'
p23.script = 'echo Hello world!'
p23.errhow = 'ignore'
p23.callback = lambda p: setattr(p, 'forks', 10)
p23._preRunTidy()
p23._runJobs()
p23.jobs[0].state = STATES.BUILTFAILED
p23.jobs[1].state = STATES.SUBMITFAILED
p23.jobs[2].state = STATES.DONE
p23.jobs[3].state = STATES.DONECACHED
p23.jobs[4].state = STATES.ENDFAILED
p23._postRunTidy()
assert ' Jobs (Cached: 1, Succ: 1, B.Fail: 1, S.Fail: 1, R.Fail: 1)' in caplog.text
assert 'Failed but ignored (totally 3).' in caplog.text
assert p23.forks == 10
p23.errhow = 'terminate'
caplog.clear()
with pytest.raises(SystemExit):
p23._postRunTidy()
assert 'Cached: 4' in caplog.text
assert 'Succeeded: 3' in caplog.text
assert 'Building failed: 1' in caplog.text
assert 'Submission failed: 2' in caplog.text
assert 'Running failed: 5' in caplog.text
def test_buildprops(tmpdir):
from pyppl import ProcTree
p9 = Proc()
p91 = Proc(id='p9')
ProcTree.register(p9)
ProcTree.register(p91)
with pytest.raises(ProcTreeProcExists):
p91._buildProps()
p9.id = 'p89'
p9.template = TemplateLiquid
p9.ppldir = Path(tmpdir / 'test_buildprops')
p9.rc = '0,1'
p9.workdir = tmpdir / 'p8'
p9.exdir = tmpdir / 'p8.exdir'
p9.echo = True
p9.expect = 'ls'
p9.expart = 'outfile'
p9._buildProps()
assert p9.template is TemplateLiquid
assert p9.rc == [0, 1]
assert p9.workdir.exists()
assert fs.exists(p9.exdir)
assert p9.echo == dict(jobs=[0], type=dict(stderr=None, stdout=None))
assert p9.expect.render() == 'ls'
assert len(p9.expart) == 1
assert p9.expart[0].render() == 'outfile'
p9.template = None
p9.rc = 1
p9.sets.remove('workdir')
p9.props.workdir = None
p9.echo = False
p9._buildProps()
assert p9.template is TemplateLiquid
assert p9.rc == [1]
assert Path(
p9.workdir) == Path(p9.ppldir) / ('PyPPL.p89.notag.%s' % p9.suffix)
assert p9.echo == dict(jobs=[], type=dict(stderr=None, stdout=None))
p9.template = 'liquid'
p9.rc = [0, 1, 2]
p9.echo = 'stderr'
p9._buildProps()
assert p9.template is TemplateLiquid
assert p9.rc == [0, 1, 2]
assert p9.echo == dict(jobs=[0], type=dict(stderr=None))
fs.remove(p9.workdir)
p9.resume = 'resume'
with pytest.raises(ProcAttributeError):
p9._buildProps()
p9.echo = dict(type='stderr')
p9.resume = ''
p9._buildProps()
assert p9.echo == dict(jobs=[0], type=dict(stderr=None))
p9.echo = dict(jobs='0,1')
p9._buildProps()
assert p9.echo == dict(jobs=[0, 1], type=dict(stderr=None, stdout=None))
p9.echo = dict(jobs='0,1', type=dict(all=r'^log'))
p9._buildProps()
assert p9.echo == dict(jobs=[0, 1],
type=dict(stderr=r'^log', stdout=r'^log'))
