def fixt_chmodx(request, tmp_path):
if request.param == 'not_a_file':
xfile = tmp_path / 'chmodxtest_not_a_file'
xfile.mkdir()
return Box(file=xfile, expt=OSError)
elif request.param == 'success':
xfile = tmp_path / 'chmodxtest_success'
xfile.write_text('')
return Box(file=xfile, expt=[str(xfile)])
elif getuid() == 0:
pytest.skip('I am root, I cannot fail chmod and read from shebang')
elif request.param == 'failed_to_chmod':
xfile = '/etc/passwd'
return Box(file=xfile, expt=OSError)
elif request.param == 'from_shebang':
if not path.isfile('/bin/zcat'):
pytest.skip('/bin/zcat not exists.')
else:
return Box(file='/bin/zcat', expt=['/bin/sh', '/bin/zcat'])
elif request.param == 'unierr_shebang':
xfile = '/bin/bash'
if not path.isfile('/bin/bash'):
pytest.skip('/bin/bash not exists.')
else:
return Box(file=xfile, expt=OSError) # UnicodeDecodeError
def fixt_fileflush(request, fd_fileflush):
fd_read, fd_append = fd_fileflush
if request.param == 0:
return Box(filed=fd_read, residue='', expt_lines=[], expt_residue='')
if request.param == 1:
fd_append.write('abcde')
fd_append.flush()
return Box(filed=fd_read,
residue='',
expt_lines=[],
expt_residue='abcde')
if request.param == 2:
fd_append.write('ccc\ne1')
fd_append.flush()
return Box(filed=fd_read,
residue='abcde',
expt_lines=['abcdeccc\n'],
expt_residue='e1')
if request.param == 3:
fd_append.write('ccc')
fd_append.flush()
return Box(filed=fd_read,
residue='',
end=True,
expt_lines=['ccc\n'],
expt_residue='')
if request.param == 4:
return Box(filed=fd_read,
residue='end',
end=True,
expt_lines=['end\n'],
expt_residue='')
def test_prepinput_exc(job0, tmpdir):
infile1 = tmpdir / 'test_prepinput_not_exists.txt'
job0.proc.input = Box(
infile = ('file', [[]]), # no a strin gor path or input [infile:file]
)
with pytest.raises(JobInputParseError):
job0._prepInput()
job0.proc.input = Box(
nefile = ('file', [infile1]), # not exists
)
with pytest.raises(JobInputParseError):
job0._prepInput()
job0.proc.input = Box(
nlfiles = ('files', [1]), # not a list
)
with pytest.raises(JobInputParseError):
job0._prepInput()
job0.proc.input = Box(
npfiles = ('files', [[None]]), # not a path
)
with pytest.raises(JobInputParseError):
job0._prepInput()
job0.proc.input = Box(
nefiles = ('files', [[infile1]])
)
with pytest.raises(JobInputParseError):
job0._prepInput()
def test_submit(job0, caplog): job0.isRunningImpl = lambda: True assert job0.submit() assert 'is already running at' in caplog.text job0.isRunningImpl = lambda: False job0.submitImpl = lambda: Box(rc = 0) assert job0.submit() job0.submitImpl = lambda: Box(rc = 1, cmd = '', stderr = '') caplog.clear() assert not job0.submit() assert 'Submission failed' in caplog.text
def fixt_funcsig(request):
if request.param == 0:
def func1():
pass
return Box(func=func1, expt="def func1(): pass")
if request.param == 1:
func2 = lambda: True
return Box(func=func2, expt='func2 = lambda: True')
if request.param == 2:
return Box(func="", expt="None")
if request.param == 3:
return Box(func="A", expt="None")
def procPostRun(proc):
"""Generate report for the process"""
fs.remove(proc.report)
if not proc.config.report:
return
logger.debug('Rendering report template ...')
report = proc.config.report
if report.startswith('file:'):
tplfile = Path(report[5:])
## checked at __setattr__
## and we are not able to change it on the run
# if not fs.exists (tplfile):
# raise ProcAttributeError(tplfile, 'No such report template file')
logger.debug("Using report template: %s", tplfile, proc=proc.id)
report = tplfile.read_text()
report = proc.template(report, **proc.envs)
rptdata = Box(jobs=[], **proc.procvars)
for job in proc.jobs:
jobdata = job.data
datafile = job.dir / 'output' / 'job.report.data.yaml'
data = {}
data.update(jobdata.job)
if datafile.is_file():
with datafile.open() as fdata:
data.update(yaml.safe_load(fdata))
rptdata.jobs.append(Box(i=jobdata.i, o=jobdata.o, **data))
rptenvs = Box(level=2, pre='', post='', title=proc.desc)
rptenvs.update(proc.envs.get('report', {}))
rptdata.title = rptenvs.title
try:
reportmd = report.render(rptdata)
except Exception as exc:
raise RuntimeError('Failed to render report markdown for process: %s' %
(proc)) from exc
reportmd = reportmd.splitlines()
codeblock = False
appendix = False
for i, line in enumerate(reportmd):
if line.startswith('## Appendix') or appendix:
appendix = True
continue
if line.startswith('#') and not codeblock:
reportmd[i] = '#' * (rptenvs.level - 1) + line
elif codeblock:
if line.startswith('```') and len(line) - len(
line.lstrip('`')) == codeblock:
codeblock = False
elif line.startswith('```'):
codeblock = len(line) - len(line.lstrip('`'))
proc.report.write_text(
proc.template(rptenvs.pre, **proc.envs).render(rptdata) + '\n\n' +
'\n'.join(reportmd) + '\n\n' +
proc.template(rptenvs.post, **proc.envs).render(rptdata) + '\n')
def fixt_filesig(request, tmp_path):
if not request.param:
return Box(file='', expt=['', 0])
if request.param == 'a_file':
afile = tmp_path / 'filesig_afile'
afile.write_text('')
return Box(file=afile, expt=[str(afile), int(path.getmtime(afile))])
if request.param == 'nonexists':
return Box(file='/path/to/__non_exists__', expt=False)
if request.param == 'a_link':
alink = tmp_path / 'filesig_alink'
alink_orig = tmp_path / 'filesig_alink_orig'
alink_orig.write_text('')
alink.symlink_to(alink_orig)
return Box(file=alink,
expt=[str(alink),
int(path.getmtime(alink_orig))])
if request.param == 'a_link_to_dir':
alink = tmp_path / 'filesig_alink_to_dir'
adir = tmp_path / 'filesig_adir'
adir.mkdir()
alink.symlink_to(adir)
return Box(file=alink, expt=[str(alink), int(path.getmtime(adir))])
if request.param == 'a_dir_with_subdir':
adir = tmp_path / 'filesig_another_dir'
adir.mkdir()
utime(adir, (path.getmtime(adir) + 100, ) * 2)
asubdir = adir / 'filesig_another_subdir'
asubdir.mkdir()
return Box(file=adir, expt=[str(adir), int(path.getmtime(adir))])
if request.param == 'a_dir_with_file':
adir = tmp_path / 'filesig_another_dir4'
adir.mkdir()
utime(adir, (path.getmtime(adir) - 100, ) * 2)
afile = adir / 'filesig_another_file4'
afile.write_text('')
return Box(file=adir, expt=[str(adir), int(path.getmtime(afile))])
if request.param == 'a_dir_subdir_newer':
adir = tmp_path / 'filesig_another_dir2'
adir.mkdir()
utime(adir, (path.getmtime(adir) - 100, ) * 2)
asubdir = adir / 'filesig_another_subdir2'
asubdir.mkdir()
return Box(file=adir, expt=[str(adir), int(path.getmtime(asubdir))])
if request.param == 'a_dir_subdir_newer_dirsig_false':
adir = tmp_path / 'filesig_another_dir3'
adir.mkdir()
utime(adir, (path.getmtime(adir) - 100, ) * 2)
asubdir = adir / 'filesig_another_subdir3'
asubdir.mkdir()
return Box(file=adir,
dirsig=False,
expt=[str(adir), int(path.getmtime(adir))])
def test_prepinput(job0, tmpdir, caplog):
infile1 = tmpdir / 'test_prepinput.txt'
infile1.write_text('')
infile2 = tmpdir / 'renaming' / 'test_prepinput.txt'
infile2.parent.mkdir()
infile2.write_text('')
job0.proc.input = Box(
invar = ('var', ['abc']),
infile = ('file', [infile1]),
infiles = ('files', [[infile1]]),
emptyfile = ('file', ['']),
emptyfiles = ('files', [['']]),
renamed = ('file', [infile2]),
nodatafiles = ('files', [[]]),
renamedfiles = ('files', [[infile2]]),
)
job0._prepInput()
assert len(job0.input) == 8
assert job0.input['invar'] == ('var', 'abc')
assert job0.input['infile'] == ('file', str(job0.dir / 'input' / 'test_prepinput.txt'))
assert job0.input['infiles'] == ('files', [str(job0.dir / 'input' / 'test_prepinput.txt')])
assert job0.input['emptyfile'] == ('file', '')
assert job0.input['emptyfiles'] == ('files', [''])
assert job0.input['renamed'] == ('file', str(job0.dir / 'input' / '[1]test_prepinput.txt'))
assert job0.input['nodatafiles'] == ('files', [])
assert job0.input['renamedfiles'] == ('files', [str(job0.dir / 'input' / '[1]test_prepinput.txt')])
assert 'pProc: [1/1] Input file renamed: test_prepinput.txt -> [1]test_prepinput.txt' in caplog.text
assert 'No data provided for [nodatafiles:files], use empty list instead.' in caplog.text
def test_report(job0, caplog):
job0.proc._log.shorten = 10
job0.input = Box(
a = ('var', 'abcdefghijkmnopq'),
bc = ('files', ['/long/path/to/file1']),
de = ('file', '/long/path/to/file2'),
)
job0.output = Box(
outfile = ('file', '/path/to/output/file1'),
outfiles = ('files', ['/path/to/output/file2'])
)
job0.report()
assert 'pProc: [1/1] a => ab ... pq' in caplog.text
assert 'pProc: [1/1] bc => [ /l/p/t/file1 ]' in caplog.text
assert 'pProc: [1/1] de => /l/p/t/file2' in caplog.text
assert 'pProc: [1/1] outfile => /p/t/o/file1' in caplog.text
assert 'pProc: [1/1] outfiles => [ /p/t/o/file2 ]' in caplog.text
def __init__(self, *args, **kwargs):
kwargs['nthread'] = 10
kwargs['name'] = lambda procset=True: 'pProc'
kwargs['errhow'] = 'terminate'
kwargs['errntry'] = 3
kwargs['forks'] = 1
kwargs['size'] = 1
kwargs['config'] = Box(_log={})
super(Proc, self).__init__(*args, **kwargs)
def setup(config):
for colors in THEMES.values():
colors['REPORT'] = colors['DONE']
LEVELS_ALWAYS.add('REPORT')
config['report'] = ''
config['envs'].update(report=Box(
level=2,
pre='',
post='',
))
def fixt_threadex(request):
if request.param == 'noexc':
def worker():
pass
return Box(worker=worker, expt_ex=None)
if request.param == 'oserror':
def worker():
raise OSError('oserr')
return Box(worker=worker, expt_ex=OSError)
if request.param == 'runtimeerror':
def worker():
cmdy.ls('file_not_exists', _raise=True)
return Box(worker=worker, expt_ex=RuntimeError)
def fixt_killtree(request):
# spawn subprocesses
c = cmdy.python(c='import cmdy; cmdy.sleep(100)', _hold=True)
c = cmdy.python(c='import cmdy; cmdy.bash(c = "%s")' % c.cmd, _bg=True)
proc = psutil.Process(c.pid)
# take some time to spawn
while len(proc.children(recursive=True)) < 2:
sleep(.05)
return Box(pid=c.pid,
children=[p.pid for p in proc.children(recursive=True)],
killme=request.param == 'killme')
def fixt_threadpool(request):
if request.param == 'donot_raise_init':
def initializer():
sleep(.5)
return Box(initializer=initializer, nthread=3, expt_exc=None)
if request.param == 'raise_original':
def initializer():
raise IOError('')
return Box(initializer=initializer, nthread=3, expt_exc=IOError)
if request.param == 'donot_raise':
def initializer():
raise IOError('')
def cleanup(ex):
pass
return Box(initializer=initializer,
cleanup=cleanup,
nthread=3,
expt_exc=None)
if request.param == 'raise_from_cleanup':
def initializer():
raise IOError('')
def cleanup(ex):
if isinstance(ex, IOError):
raise OSError('')
return Box(initializer=initializer,
cleanup=cleanup,
nthread=3,
expt_exc=OSError)
def submit(self):
"""
Submit the job
@returns:
The `utils.cmd.Cmd` instance if succeed
else a `Box` object with stderr as the exception and rc as 1
"""
cmdlist = [self.cmds['qsub'], self.script]
try:
r = cmd.run(cmdlist)
# Your job 6556149 ("pSort.notag.3omQ6NdZ.0") has been submitted
m = re.search(r'\s(\d+)\s', r.stdout)
if not m:
r.rc = 1
else:
self.pid = m.group(1)
return r
except (OSError, subprocess.CalledProcessError) as ex:
r = Box()
r.stderr = str(ex)
r.rc = 1
return r
def submit(self): """ Submit the job @returns: The `utils.cmd.Cmd` instance if succeed else a `Box` object with stderr as the exception and rc as 1 """ cmdlist = [self.cmds['sbatch'], self.script] try: r = cmd.run(cmdlist) # sbatch: Submitted batch job 99999999 m = re.search(r'\s(\d+)$', r.stdout) if not m: # pragma: no cover r.rc = 1 else: self.pid = m.group(1) return r except (OSError, subprocess.CalledProcessError) as ex: # pragma: no cover r = Box() r.stderr = str(ex) r.rc = 1 return r
def fixt_varname(request):
#varname.index = 0
class Klass(object):
def __init__(self, default='', d2=''):
self.id = varname()
def copy(self, *arg):
return varname()
def func():
return varname()
param = request.param
if param.klass and not param.multi and \
not param.docopy and not param.array:
klass = Klass()
return Box(var=klass.id, expt='klass')
if param.klass and param.multi and \
not param.docopy and not param.array:
klass = Klass(
default='a',
d2='b',
)
return Box(var=klass.id, expt='klass')
if param.klass and not param.multi and \
param.docopy and not param.array:
klass = Klass()
klass_copy = klass.copy()
return Box(var=klass_copy, expt='klass_copy')
if param.klass and param.multi and \
param.docopy and not param.array:
klass = Klass()
klass_copy = klass.copy(1, 2)
return Box(var=klass_copy, expt='klass_copy')
if param.klass and not param.multi and \
not param.docopy and param.array:
klass = [Klass()]
return Box(var=klass[0].id, expt='var_0')
if not param.klass and not param.multi and not param.array:
fun = func()
return Box(var=fun, expt='fun')
if not param.klass and param.multi and not param.array:
fun = func()
return Box(var=fun, expt='fun')
if not param.klass and not param.multi and param.array:
fun = [func()]
return Box(var=fun[0], expt='var_1')
def test_signature(job0, tmpdir, caplog):
fs.remove(job0.dir / 'job.script')
assert job0.signature() == ''
(job0.dir / 'job.script').write_text('')
assert job0.signature() == Box(
script = filesig(job0.dir / 'job.script'),
i = {'var': {}, 'file': {}, 'files': {}},
o = {'var': {}, 'file': {}, 'dir':{}})
infile = tmpdir / 'test_signature_input.txt'
infile.write_text('')
infile1 = tmpdir / 'test_signature_input_not_exists.txt'
job0.input = Box(
invar = ('var', 'abc'),
infile = ('file', infile),
infiles = ('files', [infile])
)
assert job0.signature().i == {
'var': {'invar': 'abc'},
'file': {'infile': filesig(infile)},
'files': {'infiles': [filesig(infile)]},
}
job0.input = Box(
invar = ('var', 'abc'),
infile = ('file', infile1)
)
assert job0.signature() == ''
assert 'Empty signature because of input file' in caplog.text
job0.input = Box(
invar = ('var', 'abc'),
infiles = ('files', [infile1])
)
assert job0.signature() == ''
assert 'Empty signature because of one of input files' in caplog.text
job0.input = {}
outfile = tmpdir / 'test_signature_outfile.txt'
outfile.write_text('')
outfile1 = tmpdir / 'test_signature_outfile_not_exists.txt'
outdir = tmpdir / 'test_signature_outdir'
outdir.mkdir()
outdir1 = tmpdir / 'test_signature_outdir_not_exists'
job0.output = OBox(
out = ('var', 'abc'),
outfile = ('file', outfile),
outdir = ('dir', outdir)
)
assert job0.signature().o == {
'var': {'out': 'abc'},
'file': {'outfile': filesig(outfile)},
'dir': {'outdir': filesig(outdir, dirsig = job0.proc.dirsig)}
}
job0.output = OBox(
outfile = ('file', outfile1)
)
assert job0.signature() == ''
assert 'Empty signature because of output file:' in caplog.text
job0.output = OBox(
outdir = ('dir', outdir1)
)
assert job0.signature() == ''
assert 'Empty signature because of output dir:' in caplog.text
@pytest.mark.parametrize('numbers,expt', [
('1,2,3,4', [1, 2, 3, 4]),
('1-4', [1, 2, 3, 4]),
('1-4,7,8-10', [1, 2, 3, 4, 7, 8, 9, 10]),
])
def test_expandNumbers(numbers, expt):
assert expandNumbers(numbers) == expt
@pytest.mark.parametrize(
'val,keylen,alias,expt',
[
('a', 0, None, "a"),
('a', 0, 'b', "[b] a"),
(Box(), 0, 'l', '[l] <Box> { }'),
({
"a": 1
}, 0, 'l', '[l] { a: 1 }'),
(
{
"a": 1,
"b": 2
},
0,
'x',
# =>
'[x] { a: 1,\n'
' b: 2, }'),
(
{
def test_box_init(construct):
assert Box(construct).__dict__['_box_config']['box_intact_types'] == (
list, )
def test_box_repr(construct, expect, strexpt):
assert repr(Box(construct)) == expect
assert str(Box(construct)) == strexpt
def test_box_copy(construct):
box = Box(construct)
box2 = box.copy()
assert id(box) != id(box2)
box3 = box.__copy__()
assert id(box) != id(box3)
from os import path, getuid, utime
from time import sleep
import cmdy
import pytest
import psutil
from pyppl.utils import Box, varname
@pytest.fixture(params=[
Box(klass=True, multi=False, docopy=False, array=False),
Box(klass=True, multi=True, docopy=False, array=False),
Box(klass=True, multi=False, docopy=True, array=False),
Box(klass=True, multi=True, docopy=True, array=False),
Box(klass=True, multi=False, docopy=False, array=True),
Box(klass=False, multi=False, docopy=False, array=False),
Box(klass=False, multi=True, docopy=False, array=False),
Box(klass=False, multi=False, docopy=False, array=True),
])
def fixt_varname(request):
#varname.index = 0
class Klass(object):
def __init__(self, default='', d2=''):
self.id = varname()
def copy(self, *arg):
return varname()
def func():
return varname()
param = request.param