def test_succeed(job0, caplog):
job0.rc = 1
job0.proc.rc = [0]
assert not job0.succeed()
job0.proc.rc = [0, 1]
(job0.dir / 'output').mkdir()
job0.proc.expect = TemplateLiquid('')
assert job0.succeed()
job0.output = OBox(
outfile = ('file', job0.dir / 'output' / 'notexists')
)
job0.rc = 1
caplog.clear()
assert not job0.succeed()
assert 'Outfile not generated' in caplog.text
assert job0.rc == 1 + (1<<9)
(job0.dir / 'output' / 'notexists').write_text('')
job0.proc.expect = TemplateLiquid('grep abc {{o.outfile}}')
job0.rc = 1
caplog.clear()
assert not job0.succeed()
assert 'Check expectation' in caplog.text
assert job0.rc == 1 + (1<<10)
def test_buildinput(tmpdir, caplog):
p10 = Proc()
p10.input = 'a, b:file, '
p10.input = ('1', 'infile')
p10._buildInput()
assert len(p10.input) == 2
assert p10.input['a'] == ('var', ['1'])
assert p10.input['b'] == ('file', ['infile'])
assert p10.size == 1
p10.input = 'a:x:y'
with pytest.raises(ProcInputError):
p10._buildInput()
p101 = Proc()
p101.props.channel = Channel.create([(1, 3), (2, 4)])
p10.depends = p101
p10.input = 'a, b, c'
p10.input = lambda ch: ch.cbind(1).cbind(2)
caplog.clear()
p10._buildInput()
assert 'Not all data are used as input' in caplog.text
assert len(p10.input) == 3
assert p10.size == 2
assert p10.input['a'] == ('var', [1, 2])
assert p10.input['b'] == ('var', [3, 4])
assert p10.input['c'] == ('var', [1, 1])
p10.input = 'a:files, b:files, c'
p10.input = Channel.create([['infile1'], ['infile2']])
p10._buildInput()
assert 'No data found for input key "b"' in caplog.text
assert 'No data found for input key "c"' in caplog.text
caplog.clear()
assert len(p10.input) == 3
assert p10.size == 2
assert p10.input['a'] == ('files', [['infile1'], ['infile2']])
assert p10.input['b'] == ('files', [[], []])
assert p10.input['c'] == ('var', ['', ''])
p10.props.template = TemplateLiquid
p10.props.workdir = tmpdir / 'test_buildinput_p10'
p10.resume = 'resume'
fs.remove(Path(p10.workdir) / 'proc.settings.yaml')
with pytest.raises(ProcInputError):
p10._buildInput()
fs.mkdir(p10.workdir)
p10.props.input = OBox()
p10.input['a'] = ('files', [['infile1'], ['infile2']])
p10.input['b'] = ('files', [[], []])
p10.input['c'] = ('var', ['', ''])
p10._saveSettings()
p10.props.input = None
p10._buildInput()
assert len(p10.input) == 3
assert p10.size == 2
assert p10.input['a'] == ('files', [['infile1'], ['infile2']])
assert p10.input['b'] == ('files', [[], []])
assert p10.input['c'] == ('var', ['', ''])
def test_reest(job0):
job0.ntry = 0
(job0.dir / 'output').mkdir()
(job0.dir / 'output' / 'outfile.txt').write_text('')
(job0.dir / 'output' / '.jobcache').mkdir()
(job0.dir / 'job.rc').write_text('')
(job0.dir / 'job.stdout').write_text('out')
(job0.dir / 'job.stderr').write_text('err')
(job0.dir / 'job.pid').write_text('')
(job0.dir / 'retry.1').mkdir()
job0.reset()
assert not fs.exists(job0.dir / 'retry.1')
assert not fs.exists(job0.dir / 'job.rc')
# recreated
assert (job0.dir / 'job.stdout').read_text() == ''
assert (job0.dir / 'job.stderr').read_text() == ''
assert not fs.exists(job0.dir / 'job.pid')
assert fs.exists(job0.dir / 'output')
# recreated
assert not fs.exists(job0.dir / 'output' / 'outfile.txt')
job0.ntry = 1
(job0.dir / 'output' / 'outfile.txt').write_text('')
(job0.dir / 'output' / '.jobcache' / 'cached.txt').write_text('')
job0.reset()
assert fs.exists(job0.dir / 'retry.1')
assert not fs.exists(job0.dir / 'retry.1' / '.jobcache')
assert fs.exists(job0.dir / 'output' / '.jobcache' / 'cached.txt')
# remove whole output directory
job0.ntry = 0
fs.remove(job0.dir / 'output' / '.jobcache')
(job0.dir / 'output' / 'outfile.txt').write_text('')
job0.reset()
assert not fs.exists(job0.dir / 'output' / 'outfile.txt')
# move whole output directory
job0.ntry = 1
fs.remove(job0.dir / 'output' / '.jobcache')
(job0.dir / 'output' / 'outfile.txt').write_text('')
job0.reset()
assert not fs.exists(job0.dir / 'output' / 'outfile.txt')
# restore output directory and stdout, stderr
job0.output = OBox(
outdir = ('dir', job0.dir / 'output' / 'outdir'),
outfile = ('stdout', job0.dir / 'output' / 'outfile'),
errfile = ('stderr', job0.dir / 'output' / 'errfile'),
)
job0.ntry = 0
job0.reset()
assert fs.isdir(job0.dir / 'output' / 'outdir')
assert fs.islink(job0.dir / 'output' / 'outfile')
assert fs.islink(job0.dir / 'output' / 'errfile')
assert fs.samefile(job0.dir / 'job.stdout', job0.dir / 'output' / 'outfile')
assert fs.samefile(job0.dir / 'job.stderr', job0.dir / 'output' / 'errfile')
# what if outdir exists
job0.reset()
def _docSecs(self):
ret = OBox(desc=[])
name = 'desc'
for line in self.docs:
if not line.startswith('@'):
ret[name].append(line)
else:
name = line.strip('@: ')
ret[name] = []
return ret
def test_suffix_compute(tmpdir):
p76 = Proc()
p81 = Proc()
p81.depends = p76
p81.input = 'a'
sigs = OBox()
sigs.argv0 = str(Path(sys.argv[0]).resolve())
sigs.id = 'p81'
sigs.tag = 'notag'
sigs.input = 'a'
sigs.depends = ['p76#' + p76.suffix]
assert p81.suffix == uid(sigs.to_json())
def test_suffix_input(tmpdir):
p82 = Proc()
p82.input = 'a'
p82.input = [1]
sigs = OBox()
sigs.argv0 = str(Path(sys.argv[0]).resolve())
sigs.id = 'p82'
sigs.tag = 'notag'
assert p82.config.input == {'a': [1]}
sigs.input = {'a': "[1]"}
assert p82.suffix == uid(sigs.to_json())
def test_prepoutput(job0, tmpdir):
job0.proc.output = OBox()
job0._prepOutput()
assert len(job0.output) == 0
job0.proc.output.out = ('var', TemplateLiquid('abc'))
job0.proc.output.outfile = ('file', TemplateLiquid('outfile{{job.index}}.txt'))
job0._prepOutput()
assert len(job0.output) == 2
assert job0.output.out == ('var', 'abc')
assert job0.output.outfile == ('file', job0.dir / 'output' / 'outfile0.txt')
job0.proc.output.clear()
job0.proc.output.abs = ('file', TemplateLiquid('/a/b/c'))
with pytest.raises(JobOutputParseError):
job0._prepOutput()
def test_export(job0, tmpdir, caplog):
job0.proc.exdir = ''
job0.export()
assert 'Exported' not in caplog.text
job0.proc.exdir = '/path/not/exists'
with pytest.raises(AssertionError):
job0.export()
job0.proc.exdir = tmpdir / 'test_export'
job0.proc.exdir.mkdir()
job0.proc.expart = None
with pytest.raises(AssertionError):
job0.export()
job0.proc.expart = []
job0.export()
assert 'Exported' not in caplog.text
# export everything
outfile1 = job0.dir / 'output' / 'test_export_outfile.txt'
outfile1.parent.mkdir()
outfile1.write_text('')
job0.output = OBox(
outfile = ('file', outfile1)
)
job0.proc.exhow = 'copy'
job0.proc.exow = True
job0.proc._log.shorten = 0
job0.export()
assert fs.exists(job0.proc.exdir / outfile1.name)
assert not fs.islink(outfile1)
assert not fs.samefile(outfile1, job0.proc.exdir / outfile1.name)
assert ('Exported: %s' % (job0.proc.exdir / outfile1.name)) in caplog.text
job0.proc.exhow = 'move'
job0.export()
assert fs.exists(job0.proc.exdir / outfile1.name)
assert fs.islink(outfile1)
assert fs.samefile(outfile1, job0.proc.exdir / outfile1.name)
assert ('Exported: %s' % (job0.proc.exdir / outfile1.name)) in caplog.text
# outfile is a link, then copy the file
job0.export()
assert fs.exists(job0.proc.exdir / outfile1.name)
assert not fs.islink(job0.proc.exdir / outfile1.name)
assert fs.islink(outfile1)
assert fs.samefile(outfile1, job0.proc.exdir / outfile1.name)
job0.proc.exhow = 'link'
job0.export()
assert fs.exists(job0.proc.exdir / outfile1.name)
assert fs.islink(job0.proc.exdir / outfile1.name)
assert not fs.islink(outfile1)
assert fs.samefile(outfile1, job0.proc.exdir / outfile1.name)
job0.proc.exhow = 'gzip'
job0.export()
assert fs.exists(job0.proc.exdir / (outfile1.name + '.gz'))
job0.proc.expart = [TemplateLiquid('outfile')]
fs.remove(job0.proc.exdir / (outfile1.name + '.gz'))
job0.export()
assert fs.exists(job0.proc.exdir / (outfile1.name + '.gz'))
job0.proc.expart = [TemplateLiquid('*.txt')]
fs.remove(job0.proc.exdir / (outfile1.name + '.gz'))
job0.export()
assert fs.exists(job0.proc.exdir / (outfile1.name + '.gz'))
def test_isexptcached(job0, tmpdir, caplog):
job0.proc.cache = False
assert not job0.isExptCached()
job0.proc.cache = 'export'
job0.proc.exhow = 'link'
assert not job0.isExptCached()
assert 'Job is not export-cached using symlink export.' in caplog.text
caplog.clear()
job0.proc.exhow = 'copy'
job0.proc.expart = [TemplateLiquid('outfile')]
assert not job0.isExptCached()
assert 'Job is not export-cached using partial export.' in caplog.text
caplog.clear()
job0.proc.expart = None
job0.proc.exdir = ''
assert not job0.isExptCached()
assert 'Job is not export-cached since export directory is not set.' in caplog.text
caplog.clear()
job0.proc.exdir = tmpdir / 'test_isexptcached_exdir'
job0.proc.exdir.mkdir()
outfile1 = tmpdir / 'test_isexptcached_outfile1.txt'
outfile1.write_text('')
outfile2 = tmpdir / 'test_isexptcached_outfile_not_exists.txt'
outdir1 = tmpdir / 'test_isexptcached_outdir1'
outdir1.mkdir()
fs.gzip(outfile1, job0.proc.exdir / (outfile1.name + '.gz'))
fs.gzip(outdir1, job0.proc.exdir / (outdir1.name + '.tgz'))
job0.output = OBox(
outfile = ('file', outfile1),
outdir = ('dir', outdir1),
out = ('var', 'abc')
)
# overwriting existing
(job0.dir / 'output').mkdir()
(job0.dir / 'output' / outfile1.name).write_text('')
job0.proc.exhow = 'gzip'
assert job0.isExptCached()
assert 'Overwrite file for export-caching:' in caplog.text
assert job0.isTrulyCached()
caplog.clear()
fs.remove(job0.proc.exdir / (outfile1.name + '.gz'))
assert not job0.isExptCached()
assert 'Job is not export-cached since exported file not exists:' in caplog.text
caplog.clear()
job0.output = OBox(
outfile = ('file', outfile1)
)
job0.proc.exhow = 'move'
assert not job0.isExptCached()
assert 'Job is not export-cached since exported file not exists:' in caplog.text
fs.link(outfile1, job0.proc.exdir / outfile1.name)
assert job0.isExptCached()
caplog.clear()
# overwriting existing
fs.remove(job0.proc.exdir / outfile1.name)
(job0.proc.exdir / outfile1.name).write_text('')
assert job0.isExptCached()
assert 'Overwrite file for export-caching: ' in caplog.text
def test_signature(job0, tmpdir, caplog):
fs.remove(job0.dir / 'job.script')
assert job0.signature() == ''
(job0.dir / 'job.script').write_text('')
assert job0.signature() == Box(
script = filesig(job0.dir / 'job.script'),
i = {'var': {}, 'file': {}, 'files': {}},
o = {'var': {}, 'file': {}, 'dir':{}})
infile = tmpdir / 'test_signature_input.txt'
infile.write_text('')
infile1 = tmpdir / 'test_signature_input_not_exists.txt'
job0.input = Box(
invar = ('var', 'abc'),
infile = ('file', infile),
infiles = ('files', [infile])
)
assert job0.signature().i == {
'var': {'invar': 'abc'},
'file': {'infile': filesig(infile)},
'files': {'infiles': [filesig(infile)]},
}
job0.input = Box(
invar = ('var', 'abc'),
infile = ('file', infile1)
)
assert job0.signature() == ''
assert 'Empty signature because of input file' in caplog.text
job0.input = Box(
invar = ('var', 'abc'),
infiles = ('files', [infile1])
)
assert job0.signature() == ''
assert 'Empty signature because of one of input files' in caplog.text
job0.input = {}
outfile = tmpdir / 'test_signature_outfile.txt'
outfile.write_text('')
outfile1 = tmpdir / 'test_signature_outfile_not_exists.txt'
outdir = tmpdir / 'test_signature_outdir'
outdir.mkdir()
outdir1 = tmpdir / 'test_signature_outdir_not_exists'
job0.output = OBox(
out = ('var', 'abc'),
outfile = ('file', outfile),
outdir = ('dir', outdir)
)
assert job0.signature().o == {
'var': {'out': 'abc'},
'file': {'outfile': filesig(outfile)},
'dir': {'outdir': filesig(outdir, dirsig = job0.proc.dirsig)}
}
job0.output = OBox(
outfile = ('file', outfile1)
)
assert job0.signature() == ''
assert 'Empty signature because of output file:' in caplog.text
job0.output = OBox(
outdir = ('dir', outdir1)
)
assert job0.signature() == ''
assert 'Empty signature because of output dir:' in caplog.text