def test_succeed(job0, caplog): job0.rc = 1 job0.proc.rc = [0] assert not job0.succeed() job0.proc.rc = [0, 1] (job0.dir / 'output').mkdir() job0.proc.expect = TemplateLiquid('') assert job0.succeed() job0.output = OBox( outfile = ('file', job0.dir / 'output' / 'notexists') ) job0.rc = 1 caplog.clear() assert not job0.succeed() assert 'Outfile not generated' in caplog.text assert job0.rc == 1 + (1<<9) (job0.dir / 'output' / 'notexists').write_text('') job0.proc.expect = TemplateLiquid('grep abc {{o.outfile}}') job0.rc = 1 caplog.clear() assert not job0.succeed() assert 'Check expectation' in caplog.text assert job0.rc == 1 + (1<<10)
def test_buildinput(tmpdir, caplog): p10 = Proc() p10.input = 'a, b:file, ' p10.input = ('1', 'infile') p10._buildInput() assert len(p10.input) == 2 assert p10.input['a'] == ('var', ['1']) assert p10.input['b'] == ('file', ['infile']) assert p10.size == 1 p10.input = 'a:x:y' with pytest.raises(ProcInputError): p10._buildInput() p101 = Proc() p101.props.channel = Channel.create([(1, 3), (2, 4)]) p10.depends = p101 p10.input = 'a, b, c' p10.input = lambda ch: ch.cbind(1).cbind(2) caplog.clear() p10._buildInput() assert 'Not all data are used as input' in caplog.text assert len(p10.input) == 3 assert p10.size == 2 assert p10.input['a'] == ('var', [1, 2]) assert p10.input['b'] == ('var', [3, 4]) assert p10.input['c'] == ('var', [1, 1]) p10.input = 'a:files, b:files, c' p10.input = Channel.create([['infile1'], ['infile2']]) p10._buildInput() assert 'No data found for input key "b"' in caplog.text assert 'No data found for input key "c"' in caplog.text caplog.clear() assert len(p10.input) == 3 assert p10.size == 2 assert p10.input['a'] == ('files', [['infile1'], ['infile2']]) assert p10.input['b'] == ('files', [[], []]) assert p10.input['c'] == ('var', ['', '']) p10.props.template = TemplateLiquid p10.props.workdir = tmpdir / 'test_buildinput_p10' p10.resume = 'resume' fs.remove(Path(p10.workdir) / 'proc.settings.yaml') with pytest.raises(ProcInputError): p10._buildInput() fs.mkdir(p10.workdir) p10.props.input = OBox() p10.input['a'] = ('files', [['infile1'], ['infile2']]) p10.input['b'] = ('files', [[], []]) p10.input['c'] = ('var', ['', '']) p10._saveSettings() p10.props.input = None p10._buildInput() assert len(p10.input) == 3 assert p10.size == 2 assert p10.input['a'] == ('files', [['infile1'], ['infile2']]) assert p10.input['b'] == ('files', [[], []]) assert p10.input['c'] == ('var', ['', ''])
def test_reest(job0): job0.ntry = 0 (job0.dir / 'output').mkdir() (job0.dir / 'output' / 'outfile.txt').write_text('') (job0.dir / 'output' / '.jobcache').mkdir() (job0.dir / 'job.rc').write_text('') (job0.dir / 'job.stdout').write_text('out') (job0.dir / 'job.stderr').write_text('err') (job0.dir / 'job.pid').write_text('') (job0.dir / 'retry.1').mkdir() job0.reset() assert not fs.exists(job0.dir / 'retry.1') assert not fs.exists(job0.dir / 'job.rc') # recreated assert (job0.dir / 'job.stdout').read_text() == '' assert (job0.dir / 'job.stderr').read_text() == '' assert not fs.exists(job0.dir / 'job.pid') assert fs.exists(job0.dir / 'output') # recreated assert not fs.exists(job0.dir / 'output' / 'outfile.txt') job0.ntry = 1 (job0.dir / 'output' / 'outfile.txt').write_text('') (job0.dir / 'output' / '.jobcache' / 'cached.txt').write_text('') job0.reset() assert fs.exists(job0.dir / 'retry.1') assert not fs.exists(job0.dir / 'retry.1' / '.jobcache') assert fs.exists(job0.dir / 'output' / '.jobcache' / 'cached.txt') # remove whole output directory job0.ntry = 0 fs.remove(job0.dir / 'output' / '.jobcache') (job0.dir / 'output' / 'outfile.txt').write_text('') job0.reset() assert not fs.exists(job0.dir / 'output' / 'outfile.txt') # move whole output directory job0.ntry = 1 fs.remove(job0.dir / 'output' / '.jobcache') (job0.dir / 'output' / 'outfile.txt').write_text('') job0.reset() assert not fs.exists(job0.dir / 'output' / 'outfile.txt') # restore output directory and stdout, stderr job0.output = OBox( outdir = ('dir', job0.dir / 'output' / 'outdir'), outfile = ('stdout', job0.dir / 'output' / 'outfile'), errfile = ('stderr', job0.dir / 'output' / 'errfile'), ) job0.ntry = 0 job0.reset() assert fs.isdir(job0.dir / 'output' / 'outdir') assert fs.islink(job0.dir / 'output' / 'outfile') assert fs.islink(job0.dir / 'output' / 'errfile') assert fs.samefile(job0.dir / 'job.stdout', job0.dir / 'output' / 'outfile') assert fs.samefile(job0.dir / 'job.stderr', job0.dir / 'output' / 'errfile') # what if outdir exists job0.reset()
def _docSecs(self): ret = OBox(desc=[]) name = 'desc' for line in self.docs: if not line.startswith('@'): ret[name].append(line) else: name = line.strip('@: ') ret[name] = [] return ret
def test_suffix_compute(tmpdir): p76 = Proc() p81 = Proc() p81.depends = p76 p81.input = 'a' sigs = OBox() sigs.argv0 = str(Path(sys.argv[0]).resolve()) sigs.id = 'p81' sigs.tag = 'notag' sigs.input = 'a' sigs.depends = ['p76#' + p76.suffix] assert p81.suffix == uid(sigs.to_json())
def test_suffix_input(tmpdir): p82 = Proc() p82.input = 'a' p82.input = [1] sigs = OBox() sigs.argv0 = str(Path(sys.argv[0]).resolve()) sigs.id = 'p82' sigs.tag = 'notag' assert p82.config.input == {'a': [1]} sigs.input = {'a': "[1]"} assert p82.suffix == uid(sigs.to_json())
def test_prepoutput(job0, tmpdir): job0.proc.output = OBox() job0._prepOutput() assert len(job0.output) == 0 job0.proc.output.out = ('var', TemplateLiquid('abc')) job0.proc.output.outfile = ('file', TemplateLiquid('outfile{{job.index}}.txt')) job0._prepOutput() assert len(job0.output) == 2 assert job0.output.out == ('var', 'abc') assert job0.output.outfile == ('file', job0.dir / 'output' / 'outfile0.txt') job0.proc.output.clear() job0.proc.output.abs = ('file', TemplateLiquid('/a/b/c')) with pytest.raises(JobOutputParseError): job0._prepOutput()
def test_export(job0, tmpdir, caplog): job0.proc.exdir = '' job0.export() assert 'Exported' not in caplog.text job0.proc.exdir = '/path/not/exists' with pytest.raises(AssertionError): job0.export() job0.proc.exdir = tmpdir / 'test_export' job0.proc.exdir.mkdir() job0.proc.expart = None with pytest.raises(AssertionError): job0.export() job0.proc.expart = [] job0.export() assert 'Exported' not in caplog.text # export everything outfile1 = job0.dir / 'output' / 'test_export_outfile.txt' outfile1.parent.mkdir() outfile1.write_text('') job0.output = OBox( outfile = ('file', outfile1) ) job0.proc.exhow = 'copy' job0.proc.exow = True job0.proc._log.shorten = 0 job0.export() assert fs.exists(job0.proc.exdir / outfile1.name) assert not fs.islink(outfile1) assert not fs.samefile(outfile1, job0.proc.exdir / outfile1.name) assert ('Exported: %s' % (job0.proc.exdir / outfile1.name)) in caplog.text job0.proc.exhow = 'move' job0.export() assert fs.exists(job0.proc.exdir / outfile1.name) assert fs.islink(outfile1) assert fs.samefile(outfile1, job0.proc.exdir / outfile1.name) assert ('Exported: %s' % (job0.proc.exdir / outfile1.name)) in caplog.text # outfile is a link, then copy the file job0.export() assert fs.exists(job0.proc.exdir / outfile1.name) assert not fs.islink(job0.proc.exdir / outfile1.name) assert fs.islink(outfile1) assert fs.samefile(outfile1, job0.proc.exdir / outfile1.name) job0.proc.exhow = 'link' job0.export() assert fs.exists(job0.proc.exdir / outfile1.name) assert fs.islink(job0.proc.exdir / outfile1.name) assert not fs.islink(outfile1) assert fs.samefile(outfile1, job0.proc.exdir / outfile1.name) job0.proc.exhow = 'gzip' job0.export() assert fs.exists(job0.proc.exdir / (outfile1.name + '.gz')) job0.proc.expart = [TemplateLiquid('outfile')] fs.remove(job0.proc.exdir / (outfile1.name + '.gz')) job0.export() assert fs.exists(job0.proc.exdir / (outfile1.name + '.gz')) job0.proc.expart = [TemplateLiquid('*.txt')] fs.remove(job0.proc.exdir / (outfile1.name + '.gz')) job0.export() assert fs.exists(job0.proc.exdir / (outfile1.name + '.gz'))
def test_isexptcached(job0, tmpdir, caplog): job0.proc.cache = False assert not job0.isExptCached() job0.proc.cache = 'export' job0.proc.exhow = 'link' assert not job0.isExptCached() assert 'Job is not export-cached using symlink export.' in caplog.text caplog.clear() job0.proc.exhow = 'copy' job0.proc.expart = [TemplateLiquid('outfile')] assert not job0.isExptCached() assert 'Job is not export-cached using partial export.' in caplog.text caplog.clear() job0.proc.expart = None job0.proc.exdir = '' assert not job0.isExptCached() assert 'Job is not export-cached since export directory is not set.' in caplog.text caplog.clear() job0.proc.exdir = tmpdir / 'test_isexptcached_exdir' job0.proc.exdir.mkdir() outfile1 = tmpdir / 'test_isexptcached_outfile1.txt' outfile1.write_text('') outfile2 = tmpdir / 'test_isexptcached_outfile_not_exists.txt' outdir1 = tmpdir / 'test_isexptcached_outdir1' outdir1.mkdir() fs.gzip(outfile1, job0.proc.exdir / (outfile1.name + '.gz')) fs.gzip(outdir1, job0.proc.exdir / (outdir1.name + '.tgz')) job0.output = OBox( outfile = ('file', outfile1), outdir = ('dir', outdir1), out = ('var', 'abc') ) # overwriting existing (job0.dir / 'output').mkdir() (job0.dir / 'output' / outfile1.name).write_text('') job0.proc.exhow = 'gzip' assert job0.isExptCached() assert 'Overwrite file for export-caching:' in caplog.text assert job0.isTrulyCached() caplog.clear() fs.remove(job0.proc.exdir / (outfile1.name + '.gz')) assert not job0.isExptCached() assert 'Job is not export-cached since exported file not exists:' in caplog.text caplog.clear() job0.output = OBox( outfile = ('file', outfile1) ) job0.proc.exhow = 'move' assert not job0.isExptCached() assert 'Job is not export-cached since exported file not exists:' in caplog.text fs.link(outfile1, job0.proc.exdir / outfile1.name) assert job0.isExptCached() caplog.clear() # overwriting existing fs.remove(job0.proc.exdir / outfile1.name) (job0.proc.exdir / outfile1.name).write_text('') assert job0.isExptCached() assert 'Overwrite file for export-caching: ' in caplog.text
def test_signature(job0, tmpdir, caplog): fs.remove(job0.dir / 'job.script') assert job0.signature() == '' (job0.dir / 'job.script').write_text('') assert job0.signature() == Box( script = filesig(job0.dir / 'job.script'), i = {'var': {}, 'file': {}, 'files': {}}, o = {'var': {}, 'file': {}, 'dir':{}}) infile = tmpdir / 'test_signature_input.txt' infile.write_text('') infile1 = tmpdir / 'test_signature_input_not_exists.txt' job0.input = Box( invar = ('var', 'abc'), infile = ('file', infile), infiles = ('files', [infile]) ) assert job0.signature().i == { 'var': {'invar': 'abc'}, 'file': {'infile': filesig(infile)}, 'files': {'infiles': [filesig(infile)]}, } job0.input = Box( invar = ('var', 'abc'), infile = ('file', infile1) ) assert job0.signature() == '' assert 'Empty signature because of input file' in caplog.text job0.input = Box( invar = ('var', 'abc'), infiles = ('files', [infile1]) ) assert job0.signature() == '' assert 'Empty signature because of one of input files' in caplog.text job0.input = {} outfile = tmpdir / 'test_signature_outfile.txt' outfile.write_text('') outfile1 = tmpdir / 'test_signature_outfile_not_exists.txt' outdir = tmpdir / 'test_signature_outdir' outdir.mkdir() outdir1 = tmpdir / 'test_signature_outdir_not_exists' job0.output = OBox( out = ('var', 'abc'), outfile = ('file', outfile), outdir = ('dir', outdir) ) assert job0.signature().o == { 'var': {'out': 'abc'}, 'file': {'outfile': filesig(outfile)}, 'dir': {'outdir': filesig(outdir, dirsig = job0.proc.dirsig)} } job0.output = OBox( outfile = ('file', outfile1) ) assert job0.signature() == '' assert 'Empty signature because of output file:' in caplog.text job0.output = OBox( outdir = ('dir', outdir1) ) assert job0.signature() == '' assert 'Empty signature because of output dir:' in caplog.text