def test_putfile(self): tmpoutdir = None tmpindir = None try: tmpindir = tempfile.mkdtemp(prefix='isolateserver_test') infile = os.path.join(tmpindir, u'in') with fs.open(infile, 'wb') as f: f.write('data') tmpoutdir = tempfile.mkdtemp(prefix='isolateserver_test') # Copy as fileobj fo = os.path.join(tmpoutdir, u'fo') isolateserver.putfile(io.BytesIO('data'), fo) self.assertEqual(True, fs.exists(fo)) self.assertEqual(False, fs.islink(fo)) self.assertFile(fo, 'data') # Copy with partial fileobj pfo = os.path.join(tmpoutdir, u'pfo') fobj = io.BytesIO('adatab') fobj.read(1) # Read the 'a' isolateserver.putfile(fobj, pfo, size=4) self.assertEqual(True, fs.exists(pfo)) self.assertEqual(False, fs.islink(pfo)) self.assertEqual('b', fobj.read()) self.assertFile(pfo, 'data') # Copy as not readonly cp = os.path.join(tmpoutdir, u'cp') with fs.open(infile, 'rb') as f: isolateserver.putfile(f, cp, file_mode=0o755) self.assertEqual(True, fs.exists(cp)) self.assertEqual(False, fs.islink(cp)) self.assertFile(cp, 'data') # Use hardlink hl = os.path.join(tmpoutdir, u'hl') with fs.open(infile, 'rb') as f: isolateserver.putfile(f, hl, use_symlink=False) self.assertEqual(True, fs.exists(hl)) self.assertEqual(False, fs.islink(hl)) self.assertFile(hl, 'data') # Use symlink sl = os.path.join(tmpoutdir, u'sl') with fs.open(infile, 'rb') as f: isolateserver.putfile(f, sl, use_symlink=True) self.assertEqual(True, fs.exists(sl)) self.assertEqual(True, fs.islink(sl)) self.assertEqual('data', fs.open(sl, 'rb').read()) self.assertFile(sl, 'data') finally: if tmpindir: file_path.rmtree(tmpindir) if tmpoutdir: file_path.rmtree(tmpoutdir)
def trim(self, min_free_space): """Purges cache. Removes cache directories that were not accessed for a long time until there is enough free space and the number of caches is sane. If min_free_space is None, disk free space is not checked. Requires NamedCache to be open. """ self._lock.assert_locked() if not os.path.isdir(self.root_dir): return free_space = 0 if min_free_space is not None: file_path.get_free_space(self.root_dir) while ((min_free_space is not None and free_space < min_free_space) or len(self._lru) > MAX_CACHE_SIZE): try: name, (path, _) = self._lru.get_oldest() except KeyError: return named_dir = self._get_named_path(name) if fs.islink(named_dir): fs.unlink(named_dir) path_abs = os.path.join(self.root_dir, path) if os.path.isdir(path_abs): file_path.rmtree(path_abs) if min_free_space is not None: free_space = file_path.get_free_space(self.root_dir) self._lru.pop(name)
def test_symlink_absolute(self): # A symlink to an absolute path is valid. # /dir # /dir/file # /ld -> /dir # /lf -> /ld/file dirpath = os.path.join(self.tempdir, 'dir') filepath = os.path.join(dirpath, 'file') fs.mkdir(dirpath) write_content(filepath, b'hello') linkfile = os.path.join(self.tempdir, 'lf') linkdir = os.path.join(self.tempdir, 'ld') dstfile = os.path.join(linkdir, 'file') fs.symlink(dstfile, linkfile) fs.symlink(dirpath, linkdir) self.assertEqual(True, fs.islink(linkfile)) self.assertEqual(True, fs.islink(linkdir)) self.assertEqual(dstfile, fs.readlink(linkfile)) self.assertEqual(dirpath, fs.readlink(linkdir)) self.assertEqual(['file'], fs.listdir(linkdir)) # /lf resolves to /dir/file. with fs.open(linkfile) as f: self.assertEqual('hello', f.read()) # Ensures that followlinks is respected in walk(). expected = [ (self.tempdir, ['dir', 'ld'], ['lf']), (dirpath, [], ['file']), ] actual = [ (r, sorted(d), sorted(f)) for r, d, f in sorted(fs.walk(self.tempdir, followlinks=False)) ] self.assertEqual(expected, actual) expected = [ (self.tempdir, ['dir', 'ld'], ['lf']), (dirpath, [], ['file']), (linkdir, [], ['file']), ] actual = [ (r, sorted(d), sorted(f)) for r, d, f in sorted(fs.walk(self.tempdir, followlinks=True)) ] self.assertEqual(expected, actual)
def test_symlink_missing_destination_abs(self): # A symlink to a missing destination is valid and can be read back. filepath = os.path.join(self.tempdir, 'file') linkfile = os.path.join(self.tempdir, 'lf') fs.symlink(filepath, linkfile) self.assertEqual(True, fs.islink(linkfile)) self.assertEqual(filepath, fs.readlink(linkfile))
def test_corrupted(self): with open(os.path.join(self.tempdir, u'state.json'), 'w') as f: f.write('}}}}') fs.makedirs(os.path.join(self.tempdir, 'a'), 0777) with self.manager.open(): self.assertFalse(os.path.isdir(self.tempdir)) self.make_caches(['a']) self.assertTrue(fs.islink(os.path.join(self.tempdir, 'named', 'a')))
def test_corrupted(self): os.mkdir(self.cache_dir) with open(os.path.join(self.cache_dir, u'state.json'), 'w') as f: f.write('}}}}') fs.makedirs(os.path.join(self.cache_dir, 'a'), 0777) with local_caching.NamedCache(self.cache_dir, self.policies) as cache: self.assertFalse(os.path.isdir(cache.cache_dir)) self.make_caches(cache, ['a']) self.assertTrue(fs.islink(os.path.join(cache.cache_dir, 'named', 'a')))
def test_load_corrupted_state(self): os.mkdir(self.cache_dir) c = local_caching.NamedCache with open(os.path.join(self.cache_dir, c.STATE_FILE), 'w') as f: f.write('}}}}') fs.makedirs(os.path.join(self.cache_dir, '1'), 0777) cache = self.get_cache(_get_policies()) self._add_one_item(cache, 1) self.assertTrue( fs.exists(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1'))) self.assertTrue( fs.islink(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1'))) self.assertEqual([], cache.trim()) self.assertTrue( fs.exists(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1'))) self.assertTrue( fs.islink(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1'))) self.assertEqual(True, cache.cleanup()) self.assertEqual( sorted([cache.NAMED_DIR, cache.STATE_FILE, cache._lru[u'1'][0]]), sorted(fs.listdir(cache.cache_dir)))
def assertExpectedTree(self, expected): # Return True is the entries in out_dir are exactly the same as entries in # expected. Return False otherwise. count = 0 for path in expected: content = expected[path] # Assume expected path are always relative to root. root_dir = os.path.join(self.tempdir, 'io') full_path = os.path.join(root_dir, path) self.assertTrue(os.path.exists(full_path)) while fs.islink(full_path): full_path = os.readlink(full_path) # If we expect a non-empty directory, check the entries in dir. # If we expect an empty dir, its existence (checked above) is sufficient. if not os.path.isdir(full_path): with open(full_path, 'r') as f: self.assertEqual(f.read(), content) count += 1 self.assertEqual(count, len(expected))
def link_outputs_to_outdir(run_dir, out_dir, outputs): """Links any named outputs to out_dir so they can be uploaded. Raises an error if the file already exists in that directory. """ if not outputs: return isolateserver.create_directories(out_dir, outputs) for o in outputs: try: infile = os.path.join(run_dir, o) outfile = os.path.join(out_dir, o) if fs.islink(infile): # TODO(aludwin): handle directories fs.copy2(infile, outfile) else: file_path.link_file(outfile, infile, file_path.HARDLINK_WITH_FALLBACK) except OSError as e: logging.info("Couldn't collect output file %s: %s", o, e)
def trim(self, min_free_space): """Purges cache. Removes cache directories that were not accessed for a long time until there is enough free space and the number of caches is sane. If min_free_space is None, disk free space is not checked. Requires NamedCache to be open. Returns: Number of caches deleted. """ self._lock.assert_locked() if not os.path.isdir(self.root_dir): return 0 total = 0 free_space = 0 if min_free_space: free_space = file_path.get_free_space(self.root_dir) while ((min_free_space and free_space < min_free_space) or len(self._lru) > MAX_CACHE_SIZE): logging.info('Making space for named cache %s > %s or %s > %s', free_space, min_free_space, len(self._lru), MAX_CACHE_SIZE) try: name, (path, _) = self._lru.get_oldest() except KeyError: return total named_dir = self._get_named_path(name) if fs.islink(named_dir): fs.unlink(named_dir) path_abs = os.path.join(self.root_dir, path) if os.path.isdir(path_abs): logging.info('Removing named cache %s', path_abs) file_path.rmtree(path_abs) if min_free_space: free_space = file_path.get_free_space(self.root_dir) self._lru.pop(name) total += 1 return total
def _remove(self, name): """Removes a cache directory and entry. Returns: Number of caches deleted. """ self._lock.assert_locked() # First try to remove the alias if it exists. named_dir = self._get_named_path(name) if fs.islink(named_dir): fs.unlink(named_dir) # Then remove the actual data. if name not in self._lru: return rel_path, _size = self._lru.get(name) abs_path = os.path.join(self.cache_dir, rel_path) if os.path.isdir(abs_path): file_path.rmtree(abs_path) self._lru.pop(name)
def _remove(self, name): """Removes a cache directory and entry. NamedCache must be open. Returns: Number of caches deleted. """ self._lock.assert_locked() rel_path = self._lru.get(name) if not rel_path: return named_dir = self._get_named_path(name) if fs.islink(named_dir): fs.unlink(named_dir) abs_path = os.path.join(self.root_dir, rel_path) if os.path.isdir(abs_path): file_path.rmtree(abs_path) self._lru.pop(name)
def split_at_symlink(base_dir, relfile): """Scans each component of relfile and cut the string at the symlink if there is any. Returns a tuple (base_path, symlink, rest), with symlink == rest == None if not symlink was found. """ if base_dir: assert relfile assert os.path.isabs(base_dir) index = 0 else: assert os.path.isabs(relfile) index = 1 def at_root(rest): if base_dir: return safe_join(base_dir, rest) return rest while True: try: index = relfile.index(os.path.sep, index) except ValueError: index = len(relfile) full = at_root(relfile[:index]) if fs.islink(full): # A symlink! base = os.path.dirname(relfile[:index]) symlink = os.path.basename(relfile[:index]) rest = relfile[index:] logging.debug( 'split_at_symlink(%s, %s) -> (%s, %s, %s)' % (base_dir, relfile, base, symlink, rest)) return base, symlink, rest if index == len(relfile): break index += 1 return relfile, None, None
def copy_recursively(src, dst): """Efficiently copies a file or directory from src_dir to dst_dir. `item` may be a file, directory, or a symlink to a file or directory. All symlinks are replaced with their targets, so the resulting directory structure in dst_dir will never have any symlinks. To increase speed, copy_recursively hardlinks individual files into the (newly created) directory structure if possible, unlike Python's shutil.copytree(). """ orig_src = src try: # Replace symlinks with their final target. while fs.islink(src): res = fs.readlink(src) src = os.path.join(os.path.dirname(src), res) # TODO(sadafm): Explicitly handle cyclic symlinks. # Note that fs.isfile (which is a wrapper around os.path.isfile) throws # an exception if src does not exist. A warning will be logged in that case. if fs.isfile(src): file_path.link_file(dst, src, file_path.HARDLINK_WITH_FALLBACK) return if not fs.exists(dst): os.makedirs(dst) for child in fs.listdir(src): copy_recursively(os.path.join(src, child), os.path.join(dst, child)) except OSError as e: if e.errno == errno.ENOENT: logging.warning('Path %s does not exist or %s is a broken symlink', src, orig_src) else: logging.info("Couldn't collect output file %s: %s", src, e)
def cleanup(self): """Removes unknown directories. Does not recalculate the cache size since it's surprisingly slow on some OSes. """ success = True with self._lock: try: actual = set(fs.listdir(self.cache_dir)) actual.discard(self.NAMED_DIR) actual.discard(self.STATE_FILE) expected = {v[0]: k for k, v in self._lru.iteritems()} # First, handle the actual cache content. # Remove missing entries. for missing in (set(expected) - actual): self._lru.pop(expected[missing]) # Remove unexpected items. for unexpected in (actual - set(expected)): try: p = os.path.join(self.cache_dir, unexpected) if fs.isdir(p) and not fs.islink(p): file_path.rmtree(p) else: fs.remove(p) except (IOError, OSError) as e: logging.error('Failed to remove %s: %s', unexpected, e) success = False # Second, fix named cache links. named = os.path.join(self.cache_dir, self.NAMED_DIR) if os.path.isdir(named): actual = set(fs.listdir(named)) expected = set(self._lru) # Confirm entries. Do not add missing ones for now. for name in expected.intersection(actual): p = os.path.join(self.cache_dir, self.NAMED_DIR, name) expected_link = os.path.join(self.cache_dir, self._lru[name][0]) if fs.islink(p): if sys.platform == 'win32': # TODO(maruel): Implement readlink() on Windows in fs.py, then # remove this condition. # https://crbug.com/853721 continue link = fs.readlink(p) if expected_link == link: continue logging.warning( 'Unexpected symlink for cache %s: %s, expected %s', name, link, expected_link) else: logging.warning( 'Unexpected non symlink for cache %s', name) if fs.isdir(p) and not fs.islink(p): file_path.rmtree(p) else: fs.remove(p) # Remove unexpected items. for unexpected in (actual - expected): try: p = os.path.join(self.cache_dir, self.NAMED_DIR, unexpected) if fs.isdir(p): file_path.rmtree(p) else: fs.remove(p) except (IOError, OSError) as e: logging.error('Failed to remove %s: %s', unexpected, e) success = False finally: self._save() return success