def test_copy(tmpdir):
tmpdir = Path(tmpdir)
test1 = tmpdir / 'test1'
test2 = tmpdir / 'test2'
test1.write_text('1')
test2.write_text('2')
with pytest.raises(OSError):
fs.copy(test1, test2, False)
assert test1.read_text() == '1'
assert test2.read_text() == '2'
fs.copy(test1, test2, True)
assert test1.read_text() == '1'
assert test2.read_text() == '1'
dir1 = tmpdir / 'dir1'
dir2 = tmpdir / 'dir2'
dir1.mkdir()
dir2.mkdir()
fs.copy(test1, dir1 / 'test1')
with pytest.raises(OSError):
fs.copy(dir1, dir2, False)
fs.copy(dir1, dir2)
fs.exists(dir1 / 'test1')
fs.exists(dir2 / 'test2')
# copy same file, don't do anything
fs.link(test1, test2)
assert fs.samefile(test1, test2)
assert fs.islink(test2)
fs.copy(test2, test1)
assert fs.samefile(test1, test2)
assert fs.islink(test2)
def job_done(job, status): # pylint: disable=too-many-branches
"""Export the output if job succeeded"""
if status == 'failed' or not job.proc.config.export_dir:
return
# output files to export
files2ex = []
# no partial export
if (not job.proc.config.export_part
or (len(job.proc.config.export_part) == 1
and not job.proc.config.export_part[0].render(job.data))):
files2ex.extend(
Path(outdata) for outtype, outdata in job.output.values()
if outtype not in OUT_VARTYPE)
else:
for expart in job.proc.config.export_part:
expart = expart.render(job.data)
if expart in job.output:
files2ex.append(Path(job.output[expart][1]))
else:
files2ex.extend(job.dir.joinpath('output').glob(expart))
files2ex = set(files2ex)
for file2ex in files2ex:
# don't export if file2ex does not exist
# it might be a dead link
# then job should fail
if not file2ex.exists():
return
# exported file
exfile = job.proc.config.export_dir.joinpath(file2ex.name)
if job.proc.config.export_how in EX_GZIP:
exfile = exfile.with_suffix(exfile.suffix + '.tgz') \
if fs.isdir(file2ex) \
else exfile.with_suffix(exfile.suffix + '.gz')
# If job is cached and exported file exists, skip exporting
if status == 'cached' and exfile.exists():
continue
with fs.lock(file2ex, exfile):
if job.proc.config.export_how in EX_GZIP:
fs.gzip(file2ex, exfile, overwrite=job.proc.config.export_ow)
elif job.proc.config.export_how in EX_COPY:
fs.copy(file2ex, exfile, overwrite=job.proc.config.export_ow)
elif job.proc.config.export_how in EX_LINK:
fs.link(file2ex, exfile, overwrite=job.proc.config.export_ow)
else: # move
if fs.islink(file2ex):
fs.copy(file2ex,
exfile,
overwrite=job.proc.config.export_ow)
else:
fs.move(file2ex,
exfile,
overwrite=job.proc.config.export_ow)
fs.link(exfile.resolve(), file2ex)
job.logger('Exported: %s' % exfile, level='EXPORT', plugin='export')
def test_samefile(tmpdir):
tmpdir = Path(tmpdir)
test1 = tmpdir / 'testlink1'
test2 = tmpdir / 'testlink1'
test3 = tmpdir / 'testlink3'
assert fs.samefile(test1, test2)
test1.write_text('')
assert not fs.samefile(test1, test3)
fs.link(test1, test3)
assert fs.samefile(test1, test3)
def test_move(tmpdir):
tmpdir = Path(tmpdir)
test1 = tmpdir / 'test1'
test2 = tmpdir / 'test2'
test1.write_text('1')
test2.write_text('2')
with pytest.raises(OSError):
fs.move(test1, test2, False)
assert test2.read_text() == '2'
assert fs.exists(test1)
test1.write_text('1')
fs.move(test1, test2, True)
assert test2.read_text() == '1'
assert not fs.exists(test1)
# same file
fs.link(test2, test1)
fs.move(test1, test2)
assert not fs.exists(test1)
assert fs.isfile(test2)
def test_link(tmpdir):
tmpdir = Path(tmpdir)
test1 = tmpdir / 'testlink1'
test2 = tmpdir / 'testlink2'
test1.write_text('')
test2.write_text('')
with pytest.raises(OSError):
fs.link(test1, test2, False)
fs.link(test1, test2)
assert fs.islink(test2)
# same file
fs.link(test2, test1)
assert fs.samefile(test1, test2)
assert fs.islink(test1)
fs.link(test1, test2)
assert fs.samefile(test1, test2)
assert fs.islink(test2)
def test_isexptcached(job0, tmpdir, caplog):
job0.proc.cache = False
assert not job0.isExptCached()
job0.proc.cache = 'export'
job0.proc.exhow = 'link'
assert not job0.isExptCached()
assert 'Job is not export-cached using symlink export.' in caplog.text
caplog.clear()
job0.proc.exhow = 'copy'
job0.proc.expart = [TemplateLiquid('outfile')]
assert not job0.isExptCached()
assert 'Job is not export-cached using partial export.' in caplog.text
caplog.clear()
job0.proc.expart = None
job0.proc.exdir = ''
assert not job0.isExptCached()
assert 'Job is not export-cached since export directory is not set.' in caplog.text
caplog.clear()
job0.proc.exdir = tmpdir / 'test_isexptcached_exdir'
job0.proc.exdir.mkdir()
outfile1 = tmpdir / 'test_isexptcached_outfile1.txt'
outfile1.write_text('')
outfile2 = tmpdir / 'test_isexptcached_outfile_not_exists.txt'
outdir1 = tmpdir / 'test_isexptcached_outdir1'
outdir1.mkdir()
fs.gzip(outfile1, job0.proc.exdir / (outfile1.name + '.gz'))
fs.gzip(outdir1, job0.proc.exdir / (outdir1.name + '.tgz'))
job0.output = OrderedDiot(outfile=('file', outfile1),
outdir=('dir', outdir1),
out=('var', 'abc'))
# overwriting existing
(job0.dir / 'output').mkdir()
(job0.dir / 'output' / outfile1.name).write_text('')
job0.proc.exhow = 'gzip'
assert job0.isExptCached()
assert 'Overwrite file for export-caching:' in caplog.text
assert job0.isTrulyCached()
caplog.clear()
fs.remove(job0.proc.exdir / (outfile1.name + '.gz'))
assert not job0.isExptCached()
assert 'Job is not export-cached since exported file not exists:' in caplog.text
caplog.clear()
job0.output = OrderedDiot(outfile=('file', outfile1))
job0.proc.exhow = 'move'
assert not job0.isExptCached()
assert 'Job is not export-cached since exported file not exists:' in caplog.text
fs.link(outfile1, job0.proc.exdir / outfile1.name)
assert job0.isExptCached()
caplog.clear()
# overwriting existing
fs.remove(job0.proc.exdir / outfile1.name)
(job0.proc.exdir / outfile1.name).write_text('')
assert job0.isExptCached()
assert 'Overwrite file for export-caching: ' in caplog.text
def job_prebuild(job):
"""See if we can extract output from export directory"""
if job.proc.cache != 'export' or not job.proc.config.export_dir:
return
if job.proc.config.export_how in EX_LINK:
job.logger("Job is not export-cached using symlink export.",
slevel="EXPORT_CACHE_USING_SYMLINK",
level="warning",
plugin="export")
return
if job.proc.config.export_part and \
job.proc.config.export_part[0].render(job.data):
job.logger("Job is not export-cached using partial export.",
slevel="EXPORT_CACHE_USING_EXPARTIAL",
level="warning",
plugin="export")
return
for outtype, outdata in job.output.values():
if outtype in OUT_VARTYPE:
continue
exfile = job.proc.config.export_dir / outdata.name
if job.proc.config.export_how in EX_GZIP:
exfile = (exfile.with_suffix(exfile.suffix + '.tgz')
if fs.isdir(outdata) or outtype in OUT_DIRTYPE else
exfile.with_suffix(exfile.suffix + '.gz'))
with fs.lock(exfile, outdata):
if not fs.exists(exfile):
job.logger("Job is not export-cached since exported "
"file not exists: %s" % exfile,
slevel="EXPORT_CACHE_EXFILE_NOTEXISTS",
level="debug",
plugin="export")
return
if fs.exists(outdata):
job.logger('Overwrite file for export-caching: %s' %
outdata,
slevel="EXPORT_CACHE_OUTFILE_EXISTS",
level="warning",
plugin="export")
fs.gunzip(exfile, outdata)
else: # exhow not gzip
with fs.lock(exfile, outdata):
if not fs.exists(exfile):
job.logger("Job is not export-cached since "
"exported file not exists: %s" % exfile,
slevel="EXPORT_CACHE_EXFILE_NOTEXISTS",
level="debug",
plugin="export")
return
if fs.samefile(exfile, outdata):
continue
if fs.exists(outdata):
job.logger("Overwrite file for "
"export-caching: %s" % outdata,
slevel="EXPORT_CACHE_OUTFILE_EXISTS",
level="warning",
plugin="export")
fs.link(exfile.resolve(), outdata)
job.rc = 0
job.cache()
def test_prebuild(job0, tmp_path, caplog):
job0.proc.config.export_dir = False
assert not job0.is_cached()
job0.proc.cache = 'export'
job0.proc.config.export_dir = 'export'
job0.proc.config.export_how = 'link'
job_prebuild(job0)
assert not job0.is_cached()
assert 'Job is not export-cached using symlink export.' in caplog.text
caplog.clear()
job0.proc.config.export_how = 'copy'
job0.proc.config.export_part = [('outfile')]
job_prebuild(job0)
assert not job0.is_cached()
assert 'Job is not export-cached using partial export.' in caplog.text
caplog.clear()
job0.proc.config.export_part = None
job0.proc.config.export_dir = ''
job_prebuild(job0)
assert not job0.is_cached()
caplog.clear()
job0.proc.config.export_dir = tmp_path / 'test_is_cached_exdir'
job0.proc.config.export_dir.mkdir()
outfile1 = tmp_path / 'test_is_cached_outfile1.txt'
outfile1.write_text('')
outfile2 = tmp_path / 'test_is_cached_outfile_not_exists.txt'
outdir1 = tmp_path / 'test_is_cached_outdir1'
outdir1.mkdir()
fs.gzip(outfile1, job0.proc.config.export_dir / (outfile1.name + '.gz'))
fs.gzip(outdir1, job0.proc.config.export_dir / (outdir1.name + '.tgz'))
job0.__attrs_property_cached__['output'] = OrderedDiot(outfile=('file',
outfile1),
outdir=('dir',
outdir1),
out=('var', 'abc'))
# overwriting existing
(job0.dir / 'output').mkdir()
(job0.dir / 'output' / outfile1.name).write_text('')
job0.proc.config.export_how = 'gzip'
job_prebuild(job0)
assert job0.is_cached()
assert 'Overwrite file for export-caching:' in caplog.text
assert job0.is_cached()
caplog.clear()
fs.remove(job0.proc.config.export_dir / (outfile1.name + '.gz'))
job_prebuild(job0)
assert 'Job is not export-cached since exported file not exists:' in caplog.text
caplog.clear()
job0.__attrs_property_cached__['output'] = OrderedDiot(outfile=('file',
outfile1))
job0.proc.config.export_how = 'move'
job_prebuild(job0)
assert 'Job is not export-cached since exported file not exists:' in caplog.text
fs.link(outfile1, job0.proc.config.export_dir / outfile1.name)
job_prebuild(job0)
assert job0.is_cached()
caplog.clear()
# overwriting existing
fs.remove(job0.proc.config.export_dir / outfile1.name)
(job0.proc.config.export_dir / outfile1.name).write_text('')
job_prebuild(job0)
assert job0.is_cached()
assert 'Overwrite file for export-caching: ' in caplog.text