def test_copy(tmpdir):
tmpdir = Path(tmpdir)
test1 = tmpdir / 'test1'
test2 = tmpdir / 'test2'
test1.write_text('1')
test2.write_text('2')
with pytest.raises(OSError):
fs.copy(test1, test2, False)
assert test1.read_text() == '1'
assert test2.read_text() == '2'
fs.copy(test1, test2, True)
assert test1.read_text() == '1'
assert test2.read_text() == '1'
dir1 = tmpdir / 'dir1'
dir2 = tmpdir / 'dir2'
dir1.mkdir()
dir2.mkdir()
fs.copy(test1, dir1 / 'test1')
with pytest.raises(OSError):
fs.copy(dir1, dir2, False)
fs.copy(dir1, dir2)
fs.exists(dir1 / 'test1')
fs.exists(dir2 / 'test2')
# copy same file, don't do anything
fs.link(test1, test2)
assert fs.samefile(test1, test2)
assert fs.islink(test2)
fs.copy(test2, test1)
assert fs.samefile(test1, test2)
assert fs.islink(test2)
def test_reest(job0):
job0.ntry = 0
(job0.dir / 'output').mkdir()
(job0.dir / 'output' / 'outfile.txt').write_text('')
(job0.dir / 'output' / '.jobcache').mkdir()
(job0.dir / 'job.rc').write_text('')
(job0.dir / 'job.stdout').write_text('out')
(job0.dir / 'job.stderr').write_text('err')
(job0.dir / 'job.pid').write_text('')
(job0.dir / 'retry.1').mkdir()
job0.reset()
assert not fs.exists(job0.dir / 'retry.1')
assert not fs.exists(job0.dir / 'job.rc')
# recreated
assert (job0.dir / 'job.stdout').read_text() == ''
assert (job0.dir / 'job.stderr').read_text() == ''
assert not fs.exists(job0.dir / 'job.pid')
assert fs.exists(job0.dir / 'output')
# recreated
assert not fs.exists(job0.dir / 'output' / 'outfile.txt')
job0.ntry = 1
(job0.dir / 'output' / 'outfile.txt').write_text('')
(job0.dir / 'output' / '.jobcache' / 'cached.txt').write_text('')
job0.reset()
assert fs.exists(job0.dir / 'retry.1')
assert not fs.exists(job0.dir / 'retry.1' / '.jobcache')
assert fs.exists(job0.dir / 'output' / '.jobcache' / 'cached.txt')
# remove whole output directory
job0.ntry = 0
fs.remove(job0.dir / 'output' / '.jobcache')
(job0.dir / 'output' / 'outfile.txt').write_text('')
job0.reset()
assert not fs.exists(job0.dir / 'output' / 'outfile.txt')
# move whole output directory
job0.ntry = 1
fs.remove(job0.dir / 'output' / '.jobcache')
(job0.dir / 'output' / 'outfile.txt').write_text('')
job0.reset()
assert not fs.exists(job0.dir / 'output' / 'outfile.txt')
# restore output directory and stdout, stderr
job0.output = OrderedDiot(
outdir=('dir', job0.dir / 'output' / 'outdir'),
outfile=('stdout', job0.dir / 'output' / 'outfile'),
errfile=('stderr', job0.dir / 'output' / 'errfile'),
)
job0.ntry = 0
job0.reset()
assert fs.isdir(job0.dir / 'output' / 'outdir')
assert fs.islink(job0.dir / 'output' / 'outfile')
assert fs.islink(job0.dir / 'output' / 'errfile')
assert fs.samefile(job0.dir / 'job.stdout',
job0.dir / 'output' / 'outfile')
assert fs.samefile(job0.dir / 'job.stderr',
job0.dir / 'output' / 'errfile')
# what if outdir exists
job0.reset()
def job_done(job, status): # pylint: disable=too-many-branches
"""Export the output if job succeeded"""
if status == 'failed' or not job.proc.config.export_dir:
return
# output files to export
files2ex = []
# no partial export
if (not job.proc.config.export_part
or (len(job.proc.config.export_part) == 1
and not job.proc.config.export_part[0].render(job.data))):
files2ex.extend(
Path(outdata) for outtype, outdata in job.output.values()
if outtype not in OUT_VARTYPE)
else:
for expart in job.proc.config.export_part:
expart = expart.render(job.data)
if expart in job.output:
files2ex.append(Path(job.output[expart][1]))
else:
files2ex.extend(job.dir.joinpath('output').glob(expart))
files2ex = set(files2ex)
for file2ex in files2ex:
# don't export if file2ex does not exist
# it might be a dead link
# then job should fail
if not file2ex.exists():
return
# exported file
exfile = job.proc.config.export_dir.joinpath(file2ex.name)
if job.proc.config.export_how in EX_GZIP:
exfile = exfile.with_suffix(exfile.suffix + '.tgz') \
if fs.isdir(file2ex) \
else exfile.with_suffix(exfile.suffix + '.gz')
# If job is cached and exported file exists, skip exporting
if status == 'cached' and exfile.exists():
continue
with fs.lock(file2ex, exfile):
if job.proc.config.export_how in EX_GZIP:
fs.gzip(file2ex, exfile, overwrite=job.proc.config.export_ow)
elif job.proc.config.export_how in EX_COPY:
fs.copy(file2ex, exfile, overwrite=job.proc.config.export_ow)
elif job.proc.config.export_how in EX_LINK:
fs.link(file2ex, exfile, overwrite=job.proc.config.export_ow)
else: # move
if fs.islink(file2ex):
fs.copy(file2ex,
exfile,
overwrite=job.proc.config.export_ow)
else:
fs.move(file2ex,
exfile,
overwrite=job.proc.config.export_ow)
fs.link(exfile.resolve(), file2ex)
job.logger('Exported: %s' % exfile, level='EXPORT', plugin='export')
def test_link(tmpdir):
tmpdir = Path(tmpdir)
test1 = tmpdir / 'testlink1'
test2 = tmpdir / 'testlink2'
test1.write_text('')
test2.write_text('')
with pytest.raises(OSError):
fs.link(test1, test2, False)
fs.link(test1, test2)
assert fs.islink(test2)
# same file
fs.link(test2, test1)
assert fs.samefile(test1, test2)
assert fs.islink(test1)
fs.link(test1, test2)
assert fs.samefile(test1, test2)
assert fs.islink(test2)
def test_remove(tmpdir):
test1 = Path(tmpdir / 'link')
test2 = tmpdir / 'file'
test2.write_text('', encoding = 'utf-8')
test1.symlink_to(test2)
test3 = tmpdir / 'dir'
test3.mkdir()
assert fs.exists(test3)
fs.remove(test3)
assert not fs.exists(test3)
assert fs.exists(test1)
assert fs.islink(test1)
fs.remove(test1)
assert not fs.exists(test1)
assert not fs.islink(test1)
assert fs.exists(test2)
fs.remove(test2)
assert not fs.exists(test2)
with pytest.raises(OSError):
fs.remove(tmpdir / 'notexists', False)
def test_alias(tmpdir):
assert fs.exists(tmpdir)
assert fs.isdir(tmpdir)
test1 = tmpdir / 'test1'
test1.write_text('', encoding = 'utf-8')
assert fs.isfile(test1)
test2 = tmpdir / 'test2'
if fs.exists(test2):
test2.unlink()
Path(test2).symlink_to(test1)
assert fs.islink(test2)
def test_export(job0, tmpdir, caplog):
job0.proc.exdir = ''
job0.export()
assert 'Exported' not in caplog.text
job0.proc.exdir = '/path/not/exists'
with pytest.raises(AssertionError):
job0.export()
job0.proc.exdir = tmpdir / 'test_export'
job0.proc.exdir.mkdir()
job0.proc.expart = None
with pytest.raises(AssertionError):
job0.export()
job0.proc.expart = []
job0.export()
assert 'Exported' not in caplog.text
# export everything
outfile1 = job0.dir / 'output' / 'test_export_outfile.txt'
outfile1.parent.mkdir()
outfile1.write_text('')
job0.output = OrderedDiot(outfile=('file', outfile1))
job0.proc.exhow = 'copy'
job0.proc.exow = True
job0.proc._log.shorten = 0
job0.export()
assert fs.exists(job0.proc.exdir / outfile1.name)
assert not fs.islink(outfile1)
assert not fs.samefile(outfile1, job0.proc.exdir / outfile1.name)
assert ('Exported: %s' % (job0.proc.exdir / outfile1.name)) in caplog.text
job0.proc.exhow = 'move'
job0.export()
assert fs.exists(job0.proc.exdir / outfile1.name)
assert fs.islink(outfile1)
assert fs.samefile(outfile1, job0.proc.exdir / outfile1.name)
assert ('Exported: %s' % (job0.proc.exdir / outfile1.name)) in caplog.text
# outfile is a link, then copy the file
job0.export()
assert fs.exists(job0.proc.exdir / outfile1.name)
assert not fs.islink(job0.proc.exdir / outfile1.name)
assert fs.islink(outfile1)
assert fs.samefile(outfile1, job0.proc.exdir / outfile1.name)
job0.proc.exhow = 'link'
job0.export()
assert fs.exists(job0.proc.exdir / outfile1.name)
assert fs.islink(job0.proc.exdir / outfile1.name)
assert not fs.islink(outfile1)
assert fs.samefile(outfile1, job0.proc.exdir / outfile1.name)
job0.proc.exhow = 'gzip'
job0.export()
assert fs.exists(job0.proc.exdir / (outfile1.name + '.gz'))
job0.proc.expart = [TemplateLiquid('outfile')]
fs.remove(job0.proc.exdir / (outfile1.name + '.gz'))
job0.export()
assert fs.exists(job0.proc.exdir / (outfile1.name + '.gz'))
job0.proc.expart = [TemplateLiquid('*.txt')]
fs.remove(job0.proc.exdir / (outfile1.name + '.gz'))
job0.export()
assert fs.exists(job0.proc.exdir / (outfile1.name + '.gz'))
def test_export(job0, tmp_path, caplog):
job0.proc.config.export_dir = ''
job_done(job0, 'succeeded')
assert 'Exported' not in caplog.text
job0.proc.config.export_dir = tmp_path / 'test_export'
proc_prerun(job0.proc)
job0.proc.config.export_part = []
job_done(job0, 'succeeded')
assert 'Exported' not in caplog.text
# export everything
outfile1 = job0.dir / 'output' / 'test_export_outfile.txt'
outfile1.parent.mkdir(exist_ok=True)
outfile1.write_text('')
job0.__attrs_property_cached__['output'] = OrderedDiot(outfile=('file',
outfile1))
job0.proc.config.export_how = 'copy'
job0.proc.config.export_ow = True
job_done(job0, 'succeeded')
assert fs.exists(job0.proc.config.export_dir / outfile1.name)
assert not fs.islink(outfile1)
assert not fs.samefile(outfile1,
job0.proc.config.export_dir / outfile1.name)
assert ('Exported: %s' %
(job0.proc.config.export_dir / outfile1.name)) in caplog.text
job0.proc.config.export_how = 'move'
job_done(job0, 'succeeded')
assert fs.exists(job0.proc.config.export_dir / outfile1.name)
assert fs.islink(outfile1)
assert fs.samefile(outfile1, job0.proc.config.export_dir / outfile1.name)
assert ('Exported: %s' %
(job0.proc.config.export_dir / outfile1.name)) in caplog.text
# outfile is a link, then copy the file
job_done(job0, 'succeeded')
assert fs.exists(job0.proc.config.export_dir / outfile1.name)
assert not fs.islink(job0.proc.config.export_dir / outfile1.name)
assert fs.islink(outfile1)
assert fs.samefile(outfile1, job0.proc.config.export_dir / outfile1.name)
job0.proc.config.export_how = 'link'
job_done(job0, 'succeeded')
assert fs.exists(job0.proc.config.export_dir / outfile1.name)
assert fs.islink(job0.proc.config.export_dir / outfile1.name)
assert not fs.islink(outfile1)
assert fs.samefile(outfile1, job0.proc.config.export_dir / outfile1.name)
job0.proc.config.export_how = 'gzip'
job_done(job0, 'succeeded')
assert fs.exists(job0.proc.config.export_dir / (outfile1.name + '.gz'))
job0.proc.config.export_part = ['outfile']
fs.remove(job0.proc.config.export_dir / (outfile1.name + '.gz'))
job_done(job0, 'succeeded')
assert fs.exists(job0.proc.config.export_dir / (outfile1.name + '.gz'))
job0.proc.config.export_part = ['*.txt']
fs.remove(job0.proc.config.export_dir / (outfile1.name + '.gz'))
job_done(job0, 'succeeded')
assert fs.exists(job0.proc.config.export_dir / (outfile1.name + '.gz'))