def test_reest(job0):
job0.ntry = 0
(job0.dir / 'output').mkdir()
(job0.dir / 'output' / 'outfile.txt').write_text('')
(job0.dir / 'output' / '.jobcache').mkdir()
(job0.dir / 'job.rc').write_text('')
(job0.dir / 'job.stdout').write_text('out')
(job0.dir / 'job.stderr').write_text('err')
(job0.dir / 'job.pid').write_text('')
(job0.dir / 'retry.1').mkdir()
job0.reset()
assert not fs.exists(job0.dir / 'retry.1')
assert not fs.exists(job0.dir / 'job.rc')
# recreated
assert (job0.dir / 'job.stdout').read_text() == ''
assert (job0.dir / 'job.stderr').read_text() == ''
assert not fs.exists(job0.dir / 'job.pid')
assert fs.exists(job0.dir / 'output')
# recreated
assert not fs.exists(job0.dir / 'output' / 'outfile.txt')
job0.ntry = 1
(job0.dir / 'output' / 'outfile.txt').write_text('')
(job0.dir / 'output' / '.jobcache' / 'cached.txt').write_text('')
job0.reset()
assert fs.exists(job0.dir / 'retry.1')
assert not fs.exists(job0.dir / 'retry.1' / '.jobcache')
assert fs.exists(job0.dir / 'output' / '.jobcache' / 'cached.txt')
# remove whole output directory
job0.ntry = 0
fs.remove(job0.dir / 'output' / '.jobcache')
(job0.dir / 'output' / 'outfile.txt').write_text('')
job0.reset()
assert not fs.exists(job0.dir / 'output' / 'outfile.txt')
# move whole output directory
job0.ntry = 1
fs.remove(job0.dir / 'output' / '.jobcache')
(job0.dir / 'output' / 'outfile.txt').write_text('')
job0.reset()
assert not fs.exists(job0.dir / 'output' / 'outfile.txt')
# restore output directory and stdout, stderr
job0.output = OrderedDiot(
outdir=('dir', job0.dir / 'output' / 'outdir'),
outfile=('stdout', job0.dir / 'output' / 'outfile'),
errfile=('stderr', job0.dir / 'output' / 'errfile'),
)
job0.ntry = 0
job0.reset()
assert fs.isdir(job0.dir / 'output' / 'outdir')
assert fs.islink(job0.dir / 'output' / 'outfile')
assert fs.islink(job0.dir / 'output' / 'errfile')
assert fs.samefile(job0.dir / 'job.stdout',
job0.dir / 'output' / 'outfile')
assert fs.samefile(job0.dir / 'job.stderr',
job0.dir / 'output' / 'errfile')
# what if outdir exists
job0.reset()
def test_copy(tmpdir):
tmpdir = Path(tmpdir)
test1 = tmpdir / 'test1'
test2 = tmpdir / 'test2'
test1.write_text('1')
test2.write_text('2')
with pytest.raises(OSError):
fs.copy(test1, test2, False)
assert test1.read_text() == '1'
assert test2.read_text() == '2'
fs.copy(test1, test2, True)
assert test1.read_text() == '1'
assert test2.read_text() == '1'
dir1 = tmpdir / 'dir1'
dir2 = tmpdir / 'dir2'
dir1.mkdir()
dir2.mkdir()
fs.copy(test1, dir1 / 'test1')
with pytest.raises(OSError):
fs.copy(dir1, dir2, False)
fs.copy(dir1, dir2)
fs.exists(dir1 / 'test1')
fs.exists(dir2 / 'test2')
# copy same file, don't do anything
fs.link(test1, test2)
assert fs.samefile(test1, test2)
assert fs.islink(test2)
fs.copy(test2, test1)
assert fs.samefile(test1, test2)
assert fs.islink(test2)
def test_samefile(tmpdir):
tmpdir = Path(tmpdir)
test1 = tmpdir / 'testlink1'
test2 = tmpdir / 'testlink1'
test3 = tmpdir / 'testlink3'
assert fs.samefile(test1, test2)
test1.write_text('')
assert not fs.samefile(test1, test3)
fs.link(test1, test3)
assert fs.samefile(test1, test3)
def test_link(tmpdir):
tmpdir = Path(tmpdir)
test1 = tmpdir / 'testlink1'
test2 = tmpdir / 'testlink2'
test1.write_text('')
test2.write_text('')
with pytest.raises(OSError):
fs.link(test1, test2, False)
fs.link(test1, test2)
assert fs.islink(test2)
# same file
fs.link(test2, test1)
assert fs.samefile(test1, test2)
assert fs.islink(test1)
fs.link(test1, test2)
assert fs.samefile(test1, test2)
assert fs.islink(test2)
def test_export(job0, tmpdir, caplog):
job0.proc.exdir = ''
job0.export()
assert 'Exported' not in caplog.text
job0.proc.exdir = '/path/not/exists'
with pytest.raises(AssertionError):
job0.export()
job0.proc.exdir = tmpdir / 'test_export'
job0.proc.exdir.mkdir()
job0.proc.expart = None
with pytest.raises(AssertionError):
job0.export()
job0.proc.expart = []
job0.export()
assert 'Exported' not in caplog.text
# export everything
outfile1 = job0.dir / 'output' / 'test_export_outfile.txt'
outfile1.parent.mkdir()
outfile1.write_text('')
job0.output = OrderedDiot(outfile=('file', outfile1))
job0.proc.exhow = 'copy'
job0.proc.exow = True
job0.proc._log.shorten = 0
job0.export()
assert fs.exists(job0.proc.exdir / outfile1.name)
assert not fs.islink(outfile1)
assert not fs.samefile(outfile1, job0.proc.exdir / outfile1.name)
assert ('Exported: %s' % (job0.proc.exdir / outfile1.name)) in caplog.text
job0.proc.exhow = 'move'
job0.export()
assert fs.exists(job0.proc.exdir / outfile1.name)
assert fs.islink(outfile1)
assert fs.samefile(outfile1, job0.proc.exdir / outfile1.name)
assert ('Exported: %s' % (job0.proc.exdir / outfile1.name)) in caplog.text
# outfile is a link, then copy the file
job0.export()
assert fs.exists(job0.proc.exdir / outfile1.name)
assert not fs.islink(job0.proc.exdir / outfile1.name)
assert fs.islink(outfile1)
assert fs.samefile(outfile1, job0.proc.exdir / outfile1.name)
job0.proc.exhow = 'link'
job0.export()
assert fs.exists(job0.proc.exdir / outfile1.name)
assert fs.islink(job0.proc.exdir / outfile1.name)
assert not fs.islink(outfile1)
assert fs.samefile(outfile1, job0.proc.exdir / outfile1.name)
job0.proc.exhow = 'gzip'
job0.export()
assert fs.exists(job0.proc.exdir / (outfile1.name + '.gz'))
job0.proc.expart = [TemplateLiquid('outfile')]
fs.remove(job0.proc.exdir / (outfile1.name + '.gz'))
job0.export()
assert fs.exists(job0.proc.exdir / (outfile1.name + '.gz'))
job0.proc.expart = [TemplateLiquid('*.txt')]
fs.remove(job0.proc.exdir / (outfile1.name + '.gz'))
job0.export()
assert fs.exists(job0.proc.exdir / (outfile1.name + '.gz'))
def job_prebuild(job):
"""See if we can extract output from export directory"""
if job.proc.cache != 'export' or not job.proc.config.export_dir:
return
if job.proc.config.export_how in EX_LINK:
job.logger("Job is not export-cached using symlink export.",
slevel="EXPORT_CACHE_USING_SYMLINK",
level="warning",
plugin="export")
return
if job.proc.config.export_part and \
job.proc.config.export_part[0].render(job.data):
job.logger("Job is not export-cached using partial export.",
slevel="EXPORT_CACHE_USING_EXPARTIAL",
level="warning",
plugin="export")
return
for outtype, outdata in job.output.values():
if outtype in OUT_VARTYPE:
continue
exfile = job.proc.config.export_dir / outdata.name
if job.proc.config.export_how in EX_GZIP:
exfile = (exfile.with_suffix(exfile.suffix + '.tgz')
if fs.isdir(outdata) or outtype in OUT_DIRTYPE else
exfile.with_suffix(exfile.suffix + '.gz'))
with fs.lock(exfile, outdata):
if not fs.exists(exfile):
job.logger("Job is not export-cached since exported "
"file not exists: %s" % exfile,
slevel="EXPORT_CACHE_EXFILE_NOTEXISTS",
level="debug",
plugin="export")
return
if fs.exists(outdata):
job.logger('Overwrite file for export-caching: %s' %
outdata,
slevel="EXPORT_CACHE_OUTFILE_EXISTS",
level="warning",
plugin="export")
fs.gunzip(exfile, outdata)
else: # exhow not gzip
with fs.lock(exfile, outdata):
if not fs.exists(exfile):
job.logger("Job is not export-cached since "
"exported file not exists: %s" % exfile,
slevel="EXPORT_CACHE_EXFILE_NOTEXISTS",
level="debug",
plugin="export")
return
if fs.samefile(exfile, outdata):
continue
if fs.exists(outdata):
job.logger("Overwrite file for "
"export-caching: %s" % outdata,
slevel="EXPORT_CACHE_OUTFILE_EXISTS",
level="warning",
plugin="export")
fs.link(exfile.resolve(), outdata)
job.rc = 0
job.cache()
def test_export(job0, tmp_path, caplog):
job0.proc.config.export_dir = ''
job_done(job0, 'succeeded')
assert 'Exported' not in caplog.text
job0.proc.config.export_dir = tmp_path / 'test_export'
proc_prerun(job0.proc)
job0.proc.config.export_part = []
job_done(job0, 'succeeded')
assert 'Exported' not in caplog.text
# export everything
outfile1 = job0.dir / 'output' / 'test_export_outfile.txt'
outfile1.parent.mkdir(exist_ok=True)
outfile1.write_text('')
job0.__attrs_property_cached__['output'] = OrderedDiot(outfile=('file',
outfile1))
job0.proc.config.export_how = 'copy'
job0.proc.config.export_ow = True
job_done(job0, 'succeeded')
assert fs.exists(job0.proc.config.export_dir / outfile1.name)
assert not fs.islink(outfile1)
assert not fs.samefile(outfile1,
job0.proc.config.export_dir / outfile1.name)
assert ('Exported: %s' %
(job0.proc.config.export_dir / outfile1.name)) in caplog.text
job0.proc.config.export_how = 'move'
job_done(job0, 'succeeded')
assert fs.exists(job0.proc.config.export_dir / outfile1.name)
assert fs.islink(outfile1)
assert fs.samefile(outfile1, job0.proc.config.export_dir / outfile1.name)
assert ('Exported: %s' %
(job0.proc.config.export_dir / outfile1.name)) in caplog.text
# outfile is a link, then copy the file
job_done(job0, 'succeeded')
assert fs.exists(job0.proc.config.export_dir / outfile1.name)
assert not fs.islink(job0.proc.config.export_dir / outfile1.name)
assert fs.islink(outfile1)
assert fs.samefile(outfile1, job0.proc.config.export_dir / outfile1.name)
job0.proc.config.export_how = 'link'
job_done(job0, 'succeeded')
assert fs.exists(job0.proc.config.export_dir / outfile1.name)
assert fs.islink(job0.proc.config.export_dir / outfile1.name)
assert not fs.islink(outfile1)
assert fs.samefile(outfile1, job0.proc.config.export_dir / outfile1.name)
job0.proc.config.export_how = 'gzip'
job_done(job0, 'succeeded')
assert fs.exists(job0.proc.config.export_dir / (outfile1.name + '.gz'))
job0.proc.config.export_part = ['outfile']
fs.remove(job0.proc.config.export_dir / (outfile1.name + '.gz'))
job_done(job0, 'succeeded')
assert fs.exists(job0.proc.config.export_dir / (outfile1.name + '.gz'))
job0.proc.config.export_part = ['*.txt']
fs.remove(job0.proc.config.export_dir / (outfile1.name + '.gz'))
job_done(job0, 'succeeded')
assert fs.exists(job0.proc.config.export_dir / (outfile1.name + '.gz'))