def testFileSig (self):
if os.path.exists ("./test/"):
shutil.rmtree ("./test/")
os.makedirs ("./test/")
thefile = "./test/filesig.txt"
open(thefile, 'w').write('')
sig = utils.filesig (thefile)
from time import sleep
sleep (.1)
utime (thefile, None)
open(thefile, 'w').write('')
self.assertNotEqual (sig, utils.filesig(thefile))
shutil.rmtree ("./test/")
def test_signature(job0, tmpdir, caplog):
fs.remove(job0.dir / 'job.script')
assert job0.signature() == ''
(job0.dir / 'job.script').write_text('')
assert job0.signature() == Diot(script=filesig(job0.dir / 'job.script'),
i={
'var': {},
'file': {},
'files': {}
},
o={
'var': {},
'file': {},
'dir': {}
})
infile = tmpdir / 'test_signature_input.txt'
infile.write_text('')
infile1 = tmpdir / 'test_signature_input_not_exists.txt'
job0.input = Diot(invar=('var', 'abc'),
infile=('file', infile),
infiles=('files', [infile]))
job0._signature = None
assert job0.signature().i == {
'var': {
'invar': 'abc'
},
'file': {
'infile': filesig(infile)
},
'files': {
'infiles': [filesig(infile)]
},
}
job0.input = Diot(invar=('var', 'abc'), infile=('file', infile1))
job0._signature = None
assert job0.signature() == ''
assert 'Empty signature because of input file' in caplog.text
job0.input = Diot(invar=('var', 'abc'), infiles=('files', [infile1]))
job0._signature = None
assert job0.signature() == ''
assert 'Empty signature because of one of input files' in caplog.text
job0.input = {}
outfile = tmpdir / 'test_signature_outfile.txt'
outfile.write_text('')
outfile1 = tmpdir / 'test_signature_outfile_not_exists.txt'
outdir = tmpdir / 'test_signature_outdir'
outdir.mkdir()
outdir1 = tmpdir / 'test_signature_outdir_not_exists'
job0.output = OrderedDiot(out=('var', 'abc'),
outfile=('file', outfile),
outdir=('dir', outdir))
job0._signature = None
assert job0.signature().o == {
'var': {
'out': 'abc'
},
'file': {
'outfile': filesig(outfile)
},
'dir': {
'outdir': filesig(outdir, dirsig=job0.proc.dirsig)
}
}
job0.output = OrderedDiot(outfile=('file', outfile1))
job0._signature = None
assert job0.signature() == ''
assert 'Empty signature because of output file:' in caplog.text
job0.output = OrderedDiot(outdir=('dir', outdir1))
job0._signature = None
assert job0.signature() == ''
assert 'Empty signature because of output dir:' in caplog.text
def test_filesig(fixt_filesig):
fixt_filesig.dirsig = fixt_filesig.get('dirsig', True)
assert filesig(fixt_filesig.file, fixt_filesig.dirsig) == fixt_filesig.expt
def proc_postrun(proc, status):
"""Generate report for the process"""
# skip if process failed or cached
# pylint: disable=too-many-locals,too-many-branches,too-many-statements
report_file = proc.workdir.joinpath('proc.report.md')
template = proc.config.report_template
template_file = None
if template and template.startswith('file:'):
template_file = Path(template[5:])
logger.debug("Using report template: %s", template_file, proc=proc.id)
if not template and report_file.is_file():
report_file.unlink()
if not template or status == 'failed':
return
signature = OrderedDiot([(key, value)
for key, value in sorted(proc.config.items())
if key.startswith('report_')])
if template_file and template_file.is_file():
signature.template = filesig(template_file)
signature = sha256(toml.dumps(signature).encode()).hexdigest()
if status == 'cached' and report_file.is_file():
with report_file.open() as frpt:
if frpt.readline().strip() == '<!--- %s -->' % signature:
logger.debug("Report markdown file cached, skip.", proc=proc.id)
return
fs.remove(report_file)
logger.debug('Rendering report template ...', proc=proc.id)
if template_file:
template = template_file.read_text()
template = proc.template(textwrap.dedent(template), **proc.envs)
rptdata = dict(jobs=[None] * proc.size, proc=proc, args=proc.args)
for i, job in enumerate(proc.jobs):
rptdata['jobs'][i] = job.data.job.copy()
rptdata['jobs'][i]['i'] = job.data.i
rptdata['jobs'][i]['o'] = job.data.o
datafile = job.dir / 'output/job.report.data.toml'
if datafile.is_file():
with datafile.open() as fdata:
rptdata['jobs'][i].update(toml.load(fdata))
rptenvs = Diot(level=1, pre='', post='', title=proc.desc)
rptenvs.update(proc.config.report_envs)
rptdata['report'] = rptenvs
try:
reportmd = template.render(rptdata)
except Exception as exc:
raise RuntimeError('Failed to render report markdown for process: %s' %
(proc)) from exc
reportmd = reportmd.splitlines()
codeblock = False
for i, line in enumerate(reportmd):
if line.startswith('#') and not codeblock:
reportmd[i] = '#' * (rptenvs.level - 1) + line
elif codeblock:
if line.startswith('```') and len(line) - len(
line.lstrip('`')) == codeblock:
codeblock = False
elif line.startswith('```'):
codeblock = len(line) - len(line.lstrip('`'))
report_file.write_text(
'<!--- %s -->' % signature +
proc.template(textwrap.dedent(rptenvs.pre),
**proc.envs).render(rptdata) + '\n\n' +
'\n'.join(reportmd) + '\n\n' +
proc.template(textwrap.dedent(rptenvs.post),
**proc.envs).render(rptdata) + '\n'
)