def test_box(self):
with pytest.raises(LiquidRenderError):
TemplateLiquid('{{x|box}}').render({'x': []})
with pytest.raises(LiquidRenderError):
TemplateLiquid('{{x|obox}}').render({'x': []})
TemplateLiquid('{{x|box}}').render({'x': {}}) == {}
TemplateLiquid('{{x|obox}}').render({'x': {}}) == {}
def test_succeed(job0, caplog):
job0.rc = 1
job0.proc.rc = [0]
assert not job0.succeed()
job0.proc.rc = [0, 1]
(job0.dir / 'output').mkdir()
job0.proc.expect = TemplateLiquid('')
assert job0.succeed()
job0.output = OrderedDiot(outfile=('file',
job0.dir / 'output' / 'notexists'))
job0.rc = 1
caplog.clear()
assert not job0.succeed()
assert 'Outfile not generated' in caplog.text
assert job0.rc == 1 + (1 << 9)
(job0.dir / 'output' / 'notexists').write_text('')
job0.proc.expect = TemplateLiquid('grep abc {{o.outfile}}')
job0.rc = 1
caplog.clear()
assert not job0.succeed()
assert 'Check expectation' in caplog.text
assert job0.rc == 1 + (1 << 10)
def test_diot(self, default_envs):
with pytest.raises(LiquidRenderError):
TemplateLiquid('{{x|diot}}', **default_envs).render({'x': []})
with pytest.raises(LiquidRenderError):
TemplateLiquid('{{x|odiot}}', **default_envs).render({'x': []})
TemplateLiquid('{{x|diot}}', **default_envs).render({'x': {}}) == {}
TemplateLiquid('{{x|odiot}}', **default_envs).render({'x': {}}) == {}
def test_prepscript(job0, tmpdir):
job0.proc.script = TemplateLiquid(str("# python script"))
job0._prepScript()
assert (job0.dir / 'job.script').read_text() == "# python script"
job0.proc.script = TemplateLiquid(str("# python script2"))
job0._prepScript()
assert (job0.dir / 'job.script').read_text() == "# python script2"
assert fs.exists(job0.dir / 'job.script._bak')
def test_prepoutput(job0, tmpdir):
job0.proc.output = OBox()
job0._prepOutput()
assert len(job0.output) == 0
job0.proc.output.out = ('var', TemplateLiquid('abc'))
job0.proc.output.outfile = ('file', TemplateLiquid('outfile{{job.index}}.txt'))
job0._prepOutput()
assert len(job0.output) == 2
assert job0.output.out == ('var', 'abc')
assert job0.output.outfile == ('file', job0.dir / 'output' / 'outfile0.txt')
job0.proc.output.clear()
job0.proc.output.abs = ('file', TemplateLiquid('/a/b/c'))
with pytest.raises(JobOutputParseError):
job0._prepOutput()
def test_build(job0, tmpdir, caplog):
job0.proc.input = {}
job0.proc.output = {}
job0.proc.cache = True
job0.proc.script = TemplateLiquid('# script')
fs.remove(job0.dir)
assert job0.build()
assert fs.isdir(job0.dir)
assert not fs.exists(job0.dir / 'job.stdout.bak')
assert not fs.exists(job0.dir / 'job.stderr.bak')
(job0.dir / 'job.stdout').write_text('')
(job0.dir / 'job.stderr').write_text('')
assert job0.build()
assert fs.exists(job0.dir / 'job.stdout.bak')
assert fs.exists(job0.dir / 'job.stderr.bak')
fs.remove(job0.dir / FILE_STDERR)
fs.remove(job0.dir / FILE_STDOUT)
job0._signature = None
job0.rc = 0
job0.cache()
assert job0.build()
assert fs.isfile(job0.dir / FILE_STDERR)
assert fs.isfile(job0.dir / FILE_STDOUT)
job0._signature = None
job0.rc = 0
job0.cache()
assert job0.build() == 'cached'
# raise exception while building
del job0.proc.input
assert not job0.build()
assert 'KeyError' in (job0.dir / 'job.stderr').read_text()
def testStr(self, source): tpl = TemplateLiquid(source) lines = source.splitlines() if len(lines) <= 1: assert str(tpl) == 'TemplateLiquid < %s >' % ''.join(lines) else: assert str(tpl) == '\n'.join( ['TemplateLiquid <<<'] + ['\t' + line for line in tpl.source.splitlines()] + ['>>>'])
def __init__(self, *args, **kwargs):
kwargs['nthread'] = 10
kwargs['name'] = lambda procset = True: 'pProc'
kwargs['errhow'] = 'terminate'
kwargs['errntry'] = 3
kwargs['forks'] = 1
kwargs['expect'] = TemplateLiquid('')
kwargs['size'] = 1
kwargs['rc'] = [0]
kwargs['config'] = Diot(_log = {})
super(Proc, self).__init__(*args, **kwargs)
def testStr(self, source): tpl = TemplateLiquid(source) lines = source.splitlines() if len(lines) <= 1: self.assertEqual(str(tpl), 'TemplateLiquid < %s >' % ''.join(lines)) else: helpers.assertTextEqual(self, str(tpl), '\n'.join( ['TemplateLiquid <<<'] + ['\t' + line for line in tpl.source.splitlines()] + ['>>>']) )
def test_poll(job0, caplog):
fs.remove(job0.dir / 'job.stderr')
fs.remove(job0.dir / 'job.stdout')
assert job0.poll() == 'running'
(job0.dir / 'output').mkdir()
(job0.dir / 'job.stderr').write_text('')
(job0.dir / 'job.stdout').write_text('')
job0.rc = 0
job0.proc.rc = [0]
job0.proc.expect = TemplateLiquid('')
job0.proc.echo = {'jobs': [0], 'type': {'stdout': '', 'stderr': ''}}
assert job0.poll()
job0._rc = None
fs.remove(job0.dir / 'job.rc')
assert job0.poll() == 'running'
def testRender(self, source, data, out): tpl = TemplateLiquid(source) helpers.assertTextEqual(self, tpl.render(data), out)
def test_render_func(self, default_envs):
assert TemplateLiquid('{{x | render}}', **default_envs).render({'x': '{{i}}', 'i': 2}) == '2'
assert TemplateLiquid('{{x | render}}', **default_envs).render({'x': [], 'i': 2}) == '[]'
liquid = TemplateLiquid('{{x | render}}', **default_envs)
with pytest.raises(LiquidRenderError):
liquid.render({'x': '', '__engine': None})
def test_filename_no_ext(self, default_envs):
assert TemplateLiquid('{{a|fn}}', **default_envs).render({'a': 'abc'}) == "abc"
def test_readlines_skip_empty(self, tmp_path, default_envs):
tmpfile = tmp_path / 'test_readlines_skip_empty.txt'
tmpfile.write_text("a\n\nb\n")
assert TemplateLiquid('{{readlines(a) | @join: "."}}', **default_envs).render({'a': tmpfile}) == "a.b"
assert TemplateLiquid('{{readlines(a, False) | @join: "."}}', **default_envs).render({'a': tmpfile}) == "a..b"
def testRender(self, source, data, out, default_envs): tpl = TemplateLiquid(source, **default_envs) assert tpl.render(data) == out
def testInit(self, source, envs): tpl = TemplateLiquid(source, **envs) assert tpl.source == source assertDictContains(Template.DEFAULT_ENVS, tpl.envs) assertDictContains(envs, tpl.envs) assert isinstance(tpl.engine, Liquid)
def test_export(job0, tmpdir, caplog):
job0.proc.exdir = ''
job0.export()
assert 'Exported' not in caplog.text
job0.proc.exdir = '/path/not/exists'
with pytest.raises(AssertionError):
job0.export()
job0.proc.exdir = tmpdir / 'test_export'
job0.proc.exdir.mkdir()
job0.proc.expart = None
with pytest.raises(AssertionError):
job0.export()
job0.proc.expart = []
job0.export()
assert 'Exported' not in caplog.text
# export everything
outfile1 = job0.dir / 'output' / 'test_export_outfile.txt'
outfile1.parent.mkdir()
outfile1.write_text('')
job0.output = OrderedDiot(outfile=('file', outfile1))
job0.proc.exhow = 'copy'
job0.proc.exow = True
job0.proc._log.shorten = 0
job0.export()
assert fs.exists(job0.proc.exdir / outfile1.name)
assert not fs.islink(outfile1)
assert not fs.samefile(outfile1, job0.proc.exdir / outfile1.name)
assert ('Exported: %s' % (job0.proc.exdir / outfile1.name)) in caplog.text
job0.proc.exhow = 'move'
job0.export()
assert fs.exists(job0.proc.exdir / outfile1.name)
assert fs.islink(outfile1)
assert fs.samefile(outfile1, job0.proc.exdir / outfile1.name)
assert ('Exported: %s' % (job0.proc.exdir / outfile1.name)) in caplog.text
# outfile is a link, then copy the file
job0.export()
assert fs.exists(job0.proc.exdir / outfile1.name)
assert not fs.islink(job0.proc.exdir / outfile1.name)
assert fs.islink(outfile1)
assert fs.samefile(outfile1, job0.proc.exdir / outfile1.name)
job0.proc.exhow = 'link'
job0.export()
assert fs.exists(job0.proc.exdir / outfile1.name)
assert fs.islink(job0.proc.exdir / outfile1.name)
assert not fs.islink(outfile1)
assert fs.samefile(outfile1, job0.proc.exdir / outfile1.name)
job0.proc.exhow = 'gzip'
job0.export()
assert fs.exists(job0.proc.exdir / (outfile1.name + '.gz'))
job0.proc.expart = [TemplateLiquid('outfile')]
fs.remove(job0.proc.exdir / (outfile1.name + '.gz'))
job0.export()
assert fs.exists(job0.proc.exdir / (outfile1.name + '.gz'))
job0.proc.expart = [TemplateLiquid('*.txt')]
fs.remove(job0.proc.exdir / (outfile1.name + '.gz'))
job0.export()
assert fs.exists(job0.proc.exdir / (outfile1.name + '.gz'))
def testInit(self, source, envs): tpl = TemplateLiquid(source, **envs) helpers.assertTextEqual(self, tpl.source, source) self.assertDictContains(Template.DEFAULT_ENVS, tpl.envs) self.assertDictContains(envs, tpl.envs) self.assertIsInstance(tpl.engine, Liquid)
def test_isexptcached(job0, tmpdir, caplog):
job0.proc.cache = False
assert not job0.isExptCached()
job0.proc.cache = 'export'
job0.proc.exhow = 'link'
assert not job0.isExptCached()
assert 'Job is not export-cached using symlink export.' in caplog.text
caplog.clear()
job0.proc.exhow = 'copy'
job0.proc.expart = [TemplateLiquid('outfile')]
assert not job0.isExptCached()
assert 'Job is not export-cached using partial export.' in caplog.text
caplog.clear()
job0.proc.expart = None
job0.proc.exdir = ''
assert not job0.isExptCached()
assert 'Job is not export-cached since export directory is not set.' in caplog.text
caplog.clear()
job0.proc.exdir = tmpdir / 'test_isexptcached_exdir'
job0.proc.exdir.mkdir()
outfile1 = tmpdir / 'test_isexptcached_outfile1.txt'
outfile1.write_text('')
outfile2 = tmpdir / 'test_isexptcached_outfile_not_exists.txt'
outdir1 = tmpdir / 'test_isexptcached_outdir1'
outdir1.mkdir()
fs.gzip(outfile1, job0.proc.exdir / (outfile1.name + '.gz'))
fs.gzip(outdir1, job0.proc.exdir / (outdir1.name + '.tgz'))
job0.output = OrderedDiot(outfile=('file', outfile1),
outdir=('dir', outdir1),
out=('var', 'abc'))
# overwriting existing
(job0.dir / 'output').mkdir()
(job0.dir / 'output' / outfile1.name).write_text('')
job0.proc.exhow = 'gzip'
assert job0.isExptCached()
assert 'Overwrite file for export-caching:' in caplog.text
assert job0.isTrulyCached()
caplog.clear()
fs.remove(job0.proc.exdir / (outfile1.name + '.gz'))
assert not job0.isExptCached()
assert 'Job is not export-cached since exported file not exists:' in caplog.text
caplog.clear()
job0.output = OrderedDiot(outfile=('file', outfile1))
job0.proc.exhow = 'move'
assert not job0.isExptCached()
assert 'Job is not export-cached since exported file not exists:' in caplog.text
fs.link(outfile1, job0.proc.exdir / outfile1.name)
assert job0.isExptCached()
caplog.clear()
# overwriting existing
fs.remove(job0.proc.exdir / outfile1.name)
(job0.proc.exdir / outfile1.name).write_text('')
assert job0.isExptCached()
assert 'Overwrite file for export-caching: ' in caplog.text
def testRender(self, source, data, out): tpl = TemplateLiquid(source) assert tpl.render(data) == out
def test_filename_no_ext(self):
assert TemplateLiquid('{{a|fn}}').render({'a': 'abc'}) == "abc"
