def test_putfile(self):
tmpoutdir = None
tmpindir = None
try:
tmpindir = tempfile.mkdtemp(prefix='isolateserver_test')
infile = os.path.join(tmpindir, u'in')
with fs.open(infile, 'wb') as f:
f.write('data')
tmpoutdir = tempfile.mkdtemp(prefix='isolateserver_test')
# Copy as fileobj
fo = os.path.join(tmpoutdir, u'fo')
isolateserver.putfile(io.BytesIO('data'), fo)
self.assertEqual(True, fs.exists(fo))
self.assertEqual(False, fs.islink(fo))
self.assertFile(fo, 'data')
# Copy with partial fileobj
pfo = os.path.join(tmpoutdir, u'pfo')
fobj = io.BytesIO('adatab')
fobj.read(1) # Read the 'a'
isolateserver.putfile(fobj, pfo, size=4)
self.assertEqual(True, fs.exists(pfo))
self.assertEqual(False, fs.islink(pfo))
self.assertEqual('b', fobj.read())
self.assertFile(pfo, 'data')
# Copy as not readonly
cp = os.path.join(tmpoutdir, u'cp')
with fs.open(infile, 'rb') as f:
isolateserver.putfile(f, cp, file_mode=0o755)
self.assertEqual(True, fs.exists(cp))
self.assertEqual(False, fs.islink(cp))
self.assertFile(cp, 'data')
# Use hardlink
hl = os.path.join(tmpoutdir, u'hl')
with fs.open(infile, 'rb') as f:
isolateserver.putfile(f, hl, use_symlink=False)
self.assertEqual(True, fs.exists(hl))
self.assertEqual(False, fs.islink(hl))
self.assertFile(hl, 'data')
# Use symlink
sl = os.path.join(tmpoutdir, u'sl')
with fs.open(infile, 'rb') as f:
isolateserver.putfile(f, sl, use_symlink=True)
self.assertEqual(True, fs.exists(sl))
self.assertEqual(True, fs.islink(sl))
self.assertEqual('data', fs.open(sl, 'rb').read())
self.assertFile(sl, 'data')
finally:
if tmpindir:
file_path.rmtree(tmpindir)
if tmpoutdir:
file_path.rmtree(tmpoutdir)
def trim(self, min_free_space):
"""Purges cache.
Removes cache directories that were not accessed for a long time
until there is enough free space and the number of caches is sane.
If min_free_space is None, disk free space is not checked.
Requires NamedCache to be open.
"""
self._lock.assert_locked()
if not os.path.isdir(self.root_dir):
return
free_space = 0
if min_free_space is not None:
file_path.get_free_space(self.root_dir)
while ((min_free_space is not None and free_space < min_free_space)
or len(self._lru) > MAX_CACHE_SIZE):
try:
name, (path, _) = self._lru.get_oldest()
except KeyError:
return
named_dir = self._get_named_path(name)
if fs.islink(named_dir):
fs.unlink(named_dir)
path_abs = os.path.join(self.root_dir, path)
if os.path.isdir(path_abs):
file_path.rmtree(path_abs)
if min_free_space is not None:
free_space = file_path.get_free_space(self.root_dir)
self._lru.pop(name)
def test_symlink_absolute(self):
# A symlink to an absolute path is valid.
# /dir
# /dir/file
# /ld -> /dir
# /lf -> /ld/file
dirpath = os.path.join(self.tempdir, 'dir')
filepath = os.path.join(dirpath, 'file')
fs.mkdir(dirpath)
write_content(filepath, b'hello')
linkfile = os.path.join(self.tempdir, 'lf')
linkdir = os.path.join(self.tempdir, 'ld')
dstfile = os.path.join(linkdir, 'file')
fs.symlink(dstfile, linkfile)
fs.symlink(dirpath, linkdir)
self.assertEqual(True, fs.islink(linkfile))
self.assertEqual(True, fs.islink(linkdir))
self.assertEqual(dstfile, fs.readlink(linkfile))
self.assertEqual(dirpath, fs.readlink(linkdir))
self.assertEqual(['file'], fs.listdir(linkdir))
# /lf resolves to /dir/file.
with fs.open(linkfile) as f:
self.assertEqual('hello', f.read())
# Ensures that followlinks is respected in walk().
expected = [
(self.tempdir, ['dir', 'ld'], ['lf']),
(dirpath, [], ['file']),
]
actual = [
(r, sorted(d), sorted(f))
for r, d, f in sorted(fs.walk(self.tempdir, followlinks=False))
]
self.assertEqual(expected, actual)
expected = [
(self.tempdir, ['dir', 'ld'], ['lf']),
(dirpath, [], ['file']),
(linkdir, [], ['file']),
]
actual = [
(r, sorted(d), sorted(f))
for r, d, f in sorted(fs.walk(self.tempdir, followlinks=True))
]
self.assertEqual(expected, actual)
def test_symlink_missing_destination_abs(self):
# A symlink to a missing destination is valid and can be read back.
filepath = os.path.join(self.tempdir, 'file')
linkfile = os.path.join(self.tempdir, 'lf')
fs.symlink(filepath, linkfile)
self.assertEqual(True, fs.islink(linkfile))
self.assertEqual(filepath, fs.readlink(linkfile))
def test_corrupted(self):
with open(os.path.join(self.tempdir, u'state.json'), 'w') as f:
f.write('}}}}')
fs.makedirs(os.path.join(self.tempdir, 'a'), 0777)
with self.manager.open():
self.assertFalse(os.path.isdir(self.tempdir))
self.make_caches(['a'])
self.assertTrue(fs.islink(os.path.join(self.tempdir, 'named', 'a')))
def test_corrupted(self):
os.mkdir(self.cache_dir)
with open(os.path.join(self.cache_dir, u'state.json'), 'w') as f:
f.write('}}}}')
fs.makedirs(os.path.join(self.cache_dir, 'a'), 0777)
with local_caching.NamedCache(self.cache_dir, self.policies) as cache:
self.assertFalse(os.path.isdir(cache.cache_dir))
self.make_caches(cache, ['a'])
self.assertTrue(fs.islink(os.path.join(cache.cache_dir, 'named', 'a')))
def test_load_corrupted_state(self):
os.mkdir(self.cache_dir)
c = local_caching.NamedCache
with open(os.path.join(self.cache_dir, c.STATE_FILE), 'w') as f:
f.write('}}}}')
fs.makedirs(os.path.join(self.cache_dir, '1'), 0777)
cache = self.get_cache(_get_policies())
self._add_one_item(cache, 1)
self.assertTrue(
fs.exists(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1')))
self.assertTrue(
fs.islink(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1')))
self.assertEqual([], cache.trim())
self.assertTrue(
fs.exists(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1')))
self.assertTrue(
fs.islink(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1')))
self.assertEqual(True, cache.cleanup())
self.assertEqual(
sorted([cache.NAMED_DIR, cache.STATE_FILE, cache._lru[u'1'][0]]),
sorted(fs.listdir(cache.cache_dir)))
def assertExpectedTree(self, expected):
# Return True is the entries in out_dir are exactly the same as entries in
# expected. Return False otherwise.
count = 0
for path in expected:
content = expected[path]
# Assume expected path are always relative to root.
root_dir = os.path.join(self.tempdir, 'io')
full_path = os.path.join(root_dir, path)
self.assertTrue(os.path.exists(full_path))
while fs.islink(full_path):
full_path = os.readlink(full_path)
# If we expect a non-empty directory, check the entries in dir.
# If we expect an empty dir, its existence (checked above) is sufficient.
if not os.path.isdir(full_path):
with open(full_path, 'r') as f:
self.assertEqual(f.read(), content)
count += 1
self.assertEqual(count, len(expected))
def link_outputs_to_outdir(run_dir, out_dir, outputs):
"""Links any named outputs to out_dir so they can be uploaded.
Raises an error if the file already exists in that directory.
"""
if not outputs:
return
isolateserver.create_directories(out_dir, outputs)
for o in outputs:
try:
infile = os.path.join(run_dir, o)
outfile = os.path.join(out_dir, o)
if fs.islink(infile):
# TODO(aludwin): handle directories
fs.copy2(infile, outfile)
else:
file_path.link_file(outfile, infile, file_path.HARDLINK_WITH_FALLBACK)
except OSError as e:
logging.info("Couldn't collect output file %s: %s", o, e)
def trim(self, min_free_space):
"""Purges cache.
Removes cache directories that were not accessed for a long time
until there is enough free space and the number of caches is sane.
If min_free_space is None, disk free space is not checked.
Requires NamedCache to be open.
Returns:
Number of caches deleted.
"""
self._lock.assert_locked()
if not os.path.isdir(self.root_dir):
return 0
total = 0
free_space = 0
if min_free_space:
free_space = file_path.get_free_space(self.root_dir)
while ((min_free_space and free_space < min_free_space)
or len(self._lru) > MAX_CACHE_SIZE):
logging.info('Making space for named cache %s > %s or %s > %s',
free_space, min_free_space, len(self._lru),
MAX_CACHE_SIZE)
try:
name, (path, _) = self._lru.get_oldest()
except KeyError:
return total
named_dir = self._get_named_path(name)
if fs.islink(named_dir):
fs.unlink(named_dir)
path_abs = os.path.join(self.root_dir, path)
if os.path.isdir(path_abs):
logging.info('Removing named cache %s', path_abs)
file_path.rmtree(path_abs)
if min_free_space:
free_space = file_path.get_free_space(self.root_dir)
self._lru.pop(name)
total += 1
return total
def _remove(self, name):
"""Removes a cache directory and entry.
Returns:
Number of caches deleted.
"""
self._lock.assert_locked()
# First try to remove the alias if it exists.
named_dir = self._get_named_path(name)
if fs.islink(named_dir):
fs.unlink(named_dir)
# Then remove the actual data.
if name not in self._lru:
return
rel_path, _size = self._lru.get(name)
abs_path = os.path.join(self.cache_dir, rel_path)
if os.path.isdir(abs_path):
file_path.rmtree(abs_path)
self._lru.pop(name)
def _remove(self, name):
"""Removes a cache directory and entry.
NamedCache must be open.
Returns:
Number of caches deleted.
"""
self._lock.assert_locked()
rel_path = self._lru.get(name)
if not rel_path:
return
named_dir = self._get_named_path(name)
if fs.islink(named_dir):
fs.unlink(named_dir)
abs_path = os.path.join(self.root_dir, rel_path)
if os.path.isdir(abs_path):
file_path.rmtree(abs_path)
self._lru.pop(name)
def split_at_symlink(base_dir, relfile):
"""Scans each component of relfile and cut the string at the symlink if
there is any.
Returns a tuple (base_path, symlink, rest), with symlink == rest == None if
not symlink was found.
"""
if base_dir:
assert relfile
assert os.path.isabs(base_dir)
index = 0
else:
assert os.path.isabs(relfile)
index = 1
def at_root(rest):
if base_dir:
return safe_join(base_dir, rest)
return rest
while True:
try:
index = relfile.index(os.path.sep, index)
except ValueError:
index = len(relfile)
full = at_root(relfile[:index])
if fs.islink(full):
# A symlink!
base = os.path.dirname(relfile[:index])
symlink = os.path.basename(relfile[:index])
rest = relfile[index:]
logging.debug(
'split_at_symlink(%s, %s) -> (%s, %s, %s)' %
(base_dir, relfile, base, symlink, rest))
return base, symlink, rest
if index == len(relfile):
break
index += 1
return relfile, None, None
def split_at_symlink(base_dir, relfile):
"""Scans each component of relfile and cut the string at the symlink if
there is any.
Returns a tuple (base_path, symlink, rest), with symlink == rest == None if
not symlink was found.
"""
if base_dir:
assert relfile
assert os.path.isabs(base_dir)
index = 0
else:
assert os.path.isabs(relfile)
index = 1
def at_root(rest):
if base_dir:
return safe_join(base_dir, rest)
return rest
while True:
try:
index = relfile.index(os.path.sep, index)
except ValueError:
index = len(relfile)
full = at_root(relfile[:index])
if fs.islink(full):
# A symlink!
base = os.path.dirname(relfile[:index])
symlink = os.path.basename(relfile[:index])
rest = relfile[index:]
logging.debug(
'split_at_symlink(%s, %s) -> (%s, %s, %s)' %
(base_dir, relfile, base, symlink, rest))
return base, symlink, rest
if index == len(relfile):
break
index += 1
return relfile, None, None
def copy_recursively(src, dst):
"""Efficiently copies a file or directory from src_dir to dst_dir.
`item` may be a file, directory, or a symlink to a file or directory.
All symlinks are replaced with their targets, so the resulting
directory structure in dst_dir will never have any symlinks.
To increase speed, copy_recursively hardlinks individual files into the
(newly created) directory structure if possible, unlike Python's
shutil.copytree().
"""
orig_src = src
try:
# Replace symlinks with their final target.
while fs.islink(src):
res = fs.readlink(src)
src = os.path.join(os.path.dirname(src), res)
# TODO(sadafm): Explicitly handle cyclic symlinks.
# Note that fs.isfile (which is a wrapper around os.path.isfile) throws
# an exception if src does not exist. A warning will be logged in that case.
if fs.isfile(src):
file_path.link_file(dst, src, file_path.HARDLINK_WITH_FALLBACK)
return
if not fs.exists(dst):
os.makedirs(dst)
for child in fs.listdir(src):
copy_recursively(os.path.join(src, child), os.path.join(dst, child))
except OSError as e:
if e.errno == errno.ENOENT:
logging.warning('Path %s does not exist or %s is a broken symlink',
src, orig_src)
else:
logging.info("Couldn't collect output file %s: %s", src, e)
def cleanup(self):
"""Removes unknown directories.
Does not recalculate the cache size since it's surprisingly slow on some
OSes.
"""
success = True
with self._lock:
try:
actual = set(fs.listdir(self.cache_dir))
actual.discard(self.NAMED_DIR)
actual.discard(self.STATE_FILE)
expected = {v[0]: k for k, v in self._lru.iteritems()}
# First, handle the actual cache content.
# Remove missing entries.
for missing in (set(expected) - actual):
self._lru.pop(expected[missing])
# Remove unexpected items.
for unexpected in (actual - set(expected)):
try:
p = os.path.join(self.cache_dir, unexpected)
if fs.isdir(p) and not fs.islink(p):
file_path.rmtree(p)
else:
fs.remove(p)
except (IOError, OSError) as e:
logging.error('Failed to remove %s: %s', unexpected, e)
success = False
# Second, fix named cache links.
named = os.path.join(self.cache_dir, self.NAMED_DIR)
if os.path.isdir(named):
actual = set(fs.listdir(named))
expected = set(self._lru)
# Confirm entries. Do not add missing ones for now.
for name in expected.intersection(actual):
p = os.path.join(self.cache_dir, self.NAMED_DIR, name)
expected_link = os.path.join(self.cache_dir,
self._lru[name][0])
if fs.islink(p):
if sys.platform == 'win32':
# TODO(maruel): Implement readlink() on Windows in fs.py, then
# remove this condition.
# https://crbug.com/853721
continue
link = fs.readlink(p)
if expected_link == link:
continue
logging.warning(
'Unexpected symlink for cache %s: %s, expected %s',
name, link, expected_link)
else:
logging.warning(
'Unexpected non symlink for cache %s', name)
if fs.isdir(p) and not fs.islink(p):
file_path.rmtree(p)
else:
fs.remove(p)
# Remove unexpected items.
for unexpected in (actual - expected):
try:
p = os.path.join(self.cache_dir, self.NAMED_DIR,
unexpected)
if fs.isdir(p):
file_path.rmtree(p)
else:
fs.remove(p)
except (IOError, OSError) as e:
logging.error('Failed to remove %s: %s',
unexpected, e)
success = False
finally:
self._save()
return success
