def test_prerun(caplog):
sys.argv = [sys.argv[0]]
with pytest.raises(plugin.PyPPLFuncWrongPositionError):
PyPPL().start(Proc(id='pPreRun1')).run().preRun()
assert not any('PYPPL PRERUN' in msg for _, _, msg in caplog.record_tuples)
PyPPL().start(Proc(id='pPreRun2')).preRun().run()
assert any('PYPPL PRERUN' in msg for _, _, msg in caplog.record_tuples)
def test_nosuch_template(proc, reportfile, tmp_path):
ppl = PyPPL(forks=5).start(proc).run()
tmpdir = tmp_path.joinpath('nosuchstatic')
tmpdir.mkdir(exist_ok=True)
tmpdir.joinpath('template.html').write_text('')
with pytest.raises(ValueError):
ppl.report(outfile=reportfile,
standalone=False,
template=str(tmpdir.joinpath('template.html')))
def test_report_cached(proc, reportfile, caplog):
proc.config.report_template = '# Report'
PyPPL(forks=5).start(proc).run().report(outfile=reportfile)
caplog.clear()
procX = proc.copy(id=proc.id, tag='another')
procX.workdir = proc.workdir
procX.tag = proc.tag
PyPPL(forks=5).start(procX).run().report(outfile=reportfile)
assert 'Report markdown file cached, skip.' in caplog.text
def test_unlink_prev_report(proc, reportfile):
proc.config.report_template = '# Report'
PyPPL(forks=5).start(proc).run().report(outfile=reportfile)
assert proc.workdir.joinpath('proc.report.md').is_file()
assert_in_file(reportfile, 'Reports for ')
procX = proc.copy()
procX.workdir = proc.workdir
procX.config.report_template = False
PyPPL(forks=5).start(procX).run().report(outfile=reportfile)
assert not procX.workdir.joinpath('proc.report.md').is_file()
def test_run_noprofile(pset, tmp_path):
pset.p14.props.origin = 'pOrig'
for p in pset.values():
p.input = {'a': [1]}
p.output = 'a:var:{{i.a}}'
ppl = PyPPL({
'default': {
'ppldir': tmp_path / 'test_run_noprofile'
}
}).start(pset.p14, pset.p15)
ppl.run()
assert pset.p14.ppldir == tmp_path / 'test_run_noprofile'
def test_registerRunner():
with pytest.raises(RunnerClassNameError):
PyPPL.registerRunner(object)
class RunnerX:
pass
PyPPL.registerRunner(RunnerX)
assert PyPPL.RUNNERS['x'] is RunnerX
assert PyPPL.RUNNERS['local'] is runner.RunnerLocal
assert PyPPL.RUNNERS['dry'] is runner.RunnerDry
assert PyPPL.RUNNERS['ssh'] is runner.RunnerSsh
assert PyPPL.RUNNERS['sge'] is runner.RunnerSge
assert PyPPL.RUNNERS['slurm'] is runner.RunnerSlurm
def test_appendix(proc, reportfile):
proc.report = '''
# Title
Some results
## Appendix
### Full detail
Some table
'''
p2 = Proc()
p2.depends = proc
p2.report = '''
# Title2
Some other results
## Appendix
### Full detail2
Some other table
'''
PyPPL().start(proc).run().report(outfile=reportfile)
assertInfile(
reportfile,
'<h2 id="title">Title</h2>',
'<p>Some results</p>',
'<h2 id="title2">Title2</h2>',
'<p>Some other results</p>',
'<h2 id="appendix">Appendix</h2>',
'<h3 id="full-detail">Full detail</h3>',
'<p>Some table</p>',
'<h3 id="full-detail2">Full detail2</h3>',
'<p>Some other table</p>',
)
def test_run_defaultcfg(tmp_path):
os.environ['PYPPL2_f100_forks'] = '100'
config._load('PYPPL2.osenv')
pF100 = Proc()
pF100.input = {'a': [0]}
PyPPL().start(pF100).run('f100')
assert pF100.forks == 100
def test_template_rel(proc, reportfile):
proc.config.report_template = 'file:data/rel_template.md'
proc.config.report_envs.level = 3
PyPPL(forks=5).start(proc).run().report(outfile=reportfile)
assert_in_file(
reportfile,
'<h3 id="a-relative-path-template">A relative path template</h3>')
def test_jobfail(caplog):
p = Proc(id='pPluginJobFail')
p.input = {'a': ['1']}
p.script = 'exit 1'
with pytest.raises(SystemExit):
PyPPL().start(p).run()
assert any('Job 0 failed' in msg for _, _, msg in caplog.record_tuples)
def dataProvider_testStart(self):
'''
/ p3 --- \
p1 -- p2 \ / p8
\ \ p4 \ p7
p10 p6 / \ p9
p5 /
'''
with helpers.log2str():
pp = PyPPL({'_log': {'file': None}})
pStart1 = Proc()
pStart2 = Proc()
pStart3 = Proc()
pStart4 = Proc()
pStart5 = Proc()
pStart6 = Proc()
pStart7 = Proc()
pStart8 = Proc()
pStart9 = Proc()
pStart10 = Proc()
pStart9.depends = pStart7
pStart8.depends = pStart7
pStart7.depends = pStart3, pStart6
pStart3.depends = pStart2
pStart6.depends = pStart4, pStart5
pStart3.depends = pStart2
pStart4.depends = pStart2
pStart2.depends = pStart1
pStart10.depends = pStart1
yield pp, pStart1, [pStart1]
yield pp, [pStart1, pStart5], [pStart1, pStart5]
yield pp, [pStart1, pStart5, pStart2], [pStart1, pStart5], [
'WARNING', 'Start process pStart2 ignored, depending on [pStart1]'
]
def ceQTL_manhattan(args):
pSortByRs.input = [args.cefile]
pAggregate = Proc(desc='Aggregate pvalues each snp.')
pAggregate.depends = pSortByRs
pAggregate.input = 'infile:file'
pAggregate.output = 'outfile:file:{{i.infile | bn}}'
pAggregate.args.method = args.aggr
pAggregate.lang = params.python.value
pAggregate.script = 'file:scripts/ceQTL-tool-pAggregate.py'
pGTMat2Bed.depends = pAggregate
pGTMat2Bed.args.inopts.cnames = False
pGTMat2Bed.args.ncol = 66
pBedSort.depends = pGTMat2Bed
pBedSort.args.chrorder = params.chrorder.value
pManhattan.depends = pBedSort
if args.hifile:
pManhattan.input = lambda ch: ch.cbind(args.hifile)
pManhattan.args.gsize = params.gsize.value
setOutfile(pManhattan, args.outfile)
PyPPL().start(pSortByRs).run()
def test_collapses_in_collapses(proc, reportfile):
proc.report = '''
# Title
{% for i in range(5) %}
:::: {.tab}
## Tab{{i}}
::: {.tab}
### subtab1_{{i}}
content1_{{i}}
:::
::: {.tab}
### subtab2_{{i}}
content2_{{i}}
:::
::: {.tab}
### subtab3_{{i}}
content3_{{i}}
:::
::: {.tab}
### subtab4_{{i}}
content4_{{i}}
:::
::::
{% endfor %}
'''
PyPPL().start(proc).run().report(outfile=reportfile)
def test_modals(proc, reportfile):
proc.report = '''
# Title
Click [here](modal#detailinfo) for details.
{# size: default, small, large, xlarge #}
::: {.modal id="detailinfo" title="Detailed information" closebtn="true" size="xlarge"}
```table
file: %s/filetable.txt
caption: Sample table
header: true
width: 1
total_width: .8
align: default
rows: 10
cols: 0
csvargs:
dialect: unix
#delimiter: " "
```
:::
''' % (HERE, HERE)
PyPPL().start(proc).run().report(outfile=reportfile)
def test_template_rel(proc, reportfile):
proc.report = 'file:rel_template.md'
proc.envs.report.level = 3
PyPPL().start(proc).run().report(outfile=reportfile)
assertInfile(
reportfile,
'<h3 id="a-relative-path-template">A relative path template</h3>')
def test_collapses_in_collapses(proc, reportfile):
proc.config.report_template = '''
# Title
{% for i in range(5) %}
:::: {.panel}
## Tab{{i}}
::: {.panel}
### subtab1_{{i}}
content1_{{i}}
:::
::: {.panel}
### subtab2_{{i}}
content2_{{i}}
:::
::: {.panel}
### subtab3_{{i}}
content3_{{i}}
:::
::: {.panel}
### subtab4_{{i}}
content4_{{i}}
:::
::::
{% endfor %}
'''
PyPPL(forks=5).start(proc).run().report(outfile=reportfile)
def test_showallroutes(pset, caplog, tmp_path):
fcfile = tmp_path / 'test_showallroutes.svg'
# p15 -> p16 -> ps -> p19
# p14 _/ \_ p18_/ \_ p20
# hide
pset.p17.depends = []
ps = ProcSet(Proc(id='p1'), Proc(id='p2'))
ps.depends = pset.p16, pset.p18
pset.p19.depends = ps
pset.p20.depends = ps
ppl = PyPPL().start(pset.p14, pset.p15)
ppl.flowchart(fcfile)
assert 'ALL ROUTES:' in caplog.text
assert ' p14 -> p16 -> [@ps] -> p19' in caplog.text
assert ' p14 -> p16 -> [@ps] -> p20' in caplog.text
assert ' p15 -> p16 -> [@ps] -> p19' in caplog.text
assert ' p15 -> p16 -> [@ps] -> p20' in caplog.text
def test_pandoc_cmderror(proc):
with pytest.raises(SystemExit):
PyPPL().start(proc).run().report(template='nonexist')
from glob import glob
assert glob('./pytest.*.report.md')
for rmd in glob('./pytest.*.report.md'):
Path(rmd).unlink()
assert not glob('./pytest.*.report.md')
def main(opts):
"""Main function"""
if opts.outform == 'log2':
opts.outform = 'function(x) log2(x+1)'
PyPPL(name='Quality control for expression data') \
.start(pipeline(opts)) \
.run()
def dataProvider_testResume2(self):
with helpers.log2str():
pp = PyPPL({'_log': {'file': None}})
pResume21 = Proc()
pResume22 = Proc()
pResume23 = Proc()
pResume23.depends = pResume22
pResume22.depends = pResume21
yield pp, pResume21, []
yield pp, pResume21, [pResume22]
def test_coded_heading(proc, reportfile):
proc.config.report_template = '''
# Level2
```
# Level1
```
'''
proc.config.report_envs.level = 2
PyPPL(forks=5).start(proc).run().report(outfile=reportfile)
assert_in_file(reportfile, '<h2 id="level2">Level2</h2>',
'<code># Level1</code>')
def test_coded_heading(proc, reportfile):
proc.report = '''
# Level2
```
# Level1
```
'''
proc.envs.report.level = 2
PyPPL().start(proc).run().report(outfile=reportfile)
assertInfile(reportfile, '<h2 id="level2">Level2</h2>',
'<code># Level1</code>')
def test_force_collapses(proc, reportfile):
proc.config.report_template = '''
# Title
{% for i in range(3) %}
::: {.panel .accordion}
content{{i}}
:::
{% endfor %}
'''
PyPPL(forks=5).start(proc).run().report(outfile=reportfile)
def test_filetable(proc, reportfile):
proc.config.report_template = '''
```table
file: %s/data/filetable.txt
caption: Sample table
header: true
rows: 0
cols: 0
```
''' % HERE
PyPPL(forks=5).start(proc).run().report(outfile=reportfile)
def test_force_tabs(proc, reportfile):
proc.report = '''
# Title
{% for i in range(6) %}
::: {.tab .force-tab}
content{{i}}
:::
{% endfor %}
'''
PyPPL().start(proc).run().report(outfile=reportfile)
def test_force_tabs(proc, reportfile):
proc.config.report_template = '''
# Title
{% for i in range(6) %}
::: {.tab .panel}
content{{i}}
:::
{% endfor %}
'''
PyPPL(forks=5).start(proc).run().report(outfile=reportfile)
def test_collapse(proc, reportfile):
proc.report = '''
# Title
{% for i in range(10) %}
::: {.tab}
content{{i}}
:::
{% endfor %}
'''
PyPPL().start(proc).run().report(outfile=reportfile)
def test_run_extrafile(tmp_path):
cfile = tmp_path / 'test_run_extrafile.ini'
cfile.write_text('''
[default]
forks = 2
[f10]
forks = 10
''')
pCfile = Proc()
pCfile.input = {'a': [0]}
PyPPL(cfgfile=cfile).start(pCfile).run('f10')
assert pCfile.forks == 10
def test_checkProc():
py = Proc()
pz = Proc(id='py')
PyPPL._registerProc(py)
PyPPL._registerProc(pz)
with pytest.raises(ProcTreeProcExists):
PyPPL._checkProc(pz)
def test_init(tmp_path, caplog):
config._log.file = False
ppl = PyPPL()
assert PyPPL.COUNTER == 1
assert ppl.config == config
assert ppl.config._log.file == False
os.environ['PYPPL2_default__log'] = 'py:{"file": True}'
ppl = PyPPL(cfgfile='PYPPL2.osenv')
assert ppl.counter == 1
assert ppl.config._log.file == Path('./%s.1.pyppl.log' %
(Path(sys.argv[0]).stem))
caplog.clear()
ppl = PyPPL(cfgfile=tmp_path / 'nosuchfile.ini')
assert 'Configuration file does not exist: ' in caplog.text
tomlfile = tmp_path / 'test_init.toml'
tomlfile.write_text('[default]\nforks = 2')
ppl = PyPPL(cfgfile=tomlfile)
assert ppl.config.forks == 2
ppl = PyPPL({'default': {'forks': 3}}, cfgfile=tomlfile)
assert ppl.config.forks == 3
yamlfile = tmp_path / 'test_init.yaml'
yamlfile.write_text('default:\n forks: 4')
ppl = PyPPL(cfgfile=yamlfile)
assert ppl.config.forks == 4
