class TestCacheFS(unittest.TestCase,FSTestCases,ThreadingTestCases): """Test simple operation of CacheFS""" def setUp(self): self._check_interval = sys.getcheckinterval() sys.setcheckinterval(10) self.wrapped_fs = TempFS() self.fs = CacheFS(self.wrapped_fs,cache_timeout=0.01) def tearDown(self): self.fs.close() sys.setcheckinterval(self._check_interval) def test_values_are_used_from_cache(self): old_timeout = self.fs.cache_timeout self.fs.cache_timeout = None try: self.assertFalse(self.fs.isfile("hello")) self.wrapped_fs.setcontents("hello","world") self.assertTrue(self.fs.isfile("hello")) self.wrapped_fs.remove("hello") self.assertTrue(self.fs.isfile("hello")) self.fs.clear_cache() self.assertFalse(self.fs.isfile("hello")) finally: self.fs.cache_timeout = old_timeout def test_values_are_updated_in_cache(self): old_timeout = self.fs.cache_timeout self.fs.cache_timeout = None try: self.assertFalse(self.fs.isfile("hello")) self.wrapped_fs.setcontents("hello","world") self.assertTrue(self.fs.isfile("hello")) self.wrapped_fs.remove("hello") self.assertTrue(self.fs.isfile("hello")) self.wrapped_fs.setcontents("hello","world") self.assertTrue(self.fs.isfile("hello")) self.fs.remove("hello") self.assertFalse(self.fs.isfile("hello")) finally: self.fs.cache_timeout = old_timeout
class TestCacheFS(unittest.TestCase,FSTestCases,ThreadingTestCases): """Test simple operation of CacheFS""" def setUp(self): self._check_interval = sys.getcheckinterval() sys.setcheckinterval(10) self.wrapped_fs = TempFS() self.fs = CacheFS(self.wrapped_fs,cache_timeout=0.01) def tearDown(self): self.fs.close() sys.setcheckinterval(self._check_interval) def test_values_are_used_from_cache(self): old_timeout = self.fs.cache_timeout self.fs.cache_timeout = None try: self.assertFalse(self.fs.isfile("hello")) self.wrapped_fs.setcontents("hello",b("world")) self.assertTrue(self.fs.isfile("hello")) self.wrapped_fs.remove("hello") self.assertTrue(self.fs.isfile("hello")) self.fs.clear_cache() self.assertFalse(self.fs.isfile("hello")) finally: self.fs.cache_timeout = old_timeout def test_values_are_updated_in_cache(self): old_timeout = self.fs.cache_timeout self.fs.cache_timeout = None try: self.assertFalse(self.fs.isfile("hello")) self.wrapped_fs.setcontents("hello",b("world")) self.assertTrue(self.fs.isfile("hello")) self.wrapped_fs.remove("hello") self.assertTrue(self.fs.isfile("hello")) self.wrapped_fs.setcontents("hello",b("world")) self.assertTrue(self.fs.isfile("hello")) self.fs.remove("hello") self.assertFalse(self.fs.isfile("hello")) finally: self.fs.cache_timeout = old_timeout
class RegistryFileOpener(WinRegistryFileReader): """ This is a callback class used by dfwinreg to open registry hive files. We are using dfvfs as the backend to open files within our image. To resolve system variables, we make use of the variable database of the WindowsSystem instance. """ # pylint: disable=too-few-public-methods def __init__(self, dfvfs, partition, windows_system): super(RegistryFileOpener, self).__init__() self.dfvfs = dfvfs self.partition = partition self.not_present = set() self.open_handles = [] self.tmpfs = TempFS() self.windows_system = windows_system # callbacks.register_on_job_end(self._cleanup_open_files) def _cleanup_open_files(self, __): for path, handle in self.open_handles: try: handle.close() self.tmpfs.remove(path) except (OSError, FSError) as err: LOGGER.warning("Error cleaning up %s: %s", path, err) self.tmpfs.close() def Open(self, path, ascii_codepage='cp1252'): LOGGER.info("open registry %s", path) """ Opens a path within the dfVFS volume """ realpath = path.replace('\\', '/') if path in self.not_present: return None # check for variables and if we know them realpath = path for match in re.finditer('%[a-zA-Z0-9_]+%', path): key = match.group(0) val = self.windows_system.get_var(key) if val: realpath = realpath.replace(key, val) else: LOGGER.warning("Could not resolve variable %s", key) return None realpath = realpath.replace('\\', '/') if realpath.lower().startswith('c:/'): # catch absolute paths realpath = '/' + realpath[3:] if not realpath[0] == '/': realpath = '/' + realpath if realpath in self.not_present: return None path_specs = list( self.dfvfs.find_paths([realpath], partitions=[self.partition])) if not path_specs: LOGGER.warning("Could not find requested registry hive %s [%s]", path, realpath) self.not_present.add(path) self.not_present.add(realpath) return None if len(path_specs) > 1: LOGGER.warning( "Found multiple registry hives for query %s, using %s", path, dfvfs_helper.reconstruct_full_path(path_specs[0])) # extract the file locally filename = realpath.replace('/', '_') dfvfs_helper.export_file(path_specs[0], self.tmpfs, filename) try: file_object = self.tmpfs.open(filename, 'rb') except ResourceNotFound: files = self.tmpfs.listdir("/") LOGGER.warning("Could not open registry hive %s [%s] (%s)", path, realpath, files) return None self.open_handles.append((filename, file_object)) reg_file = regfile_impl.REGFWinRegistryFile( ascii_codepage=ascii_codepage) reg_file.Open(file_object) return reg_file
class COWFS(FS): def __init__( self, base_fs: FS, additions_fs: Optional[FS] = None, deletions_fs: Optional[FS] = None, ) -> None: FS.__init__(self) if additions_fs: self.additions_fs = additions_fs else: self.additions_fs = TempFS() if deletions_fs: _deletions_invariant(deletions_fs) self.deletions_fs = deletions_fs else: self.deletions_fs = TempFS() self.original_base_fs = base_fs self.base_fs = fs.wrap.read_only(base_fs) self.invariant() @staticmethod def create_cowfs(base_fs: FS, read_write_layer: FS, recreate: bool = False) -> "COWFS": additions_fs = read_write_layer.makedir("/additions", recreate=recreate) deletions_fs = read_write_layer.makedir("/deletions", recreate=recreate) return COWFS(base_fs, additions_fs, deletions_fs) def __str__(self) -> str: return (f"COWFS({self.original_base_fs}, " f"{self.additions_fs}, " f"{self.deletions_fs})") def __repr__(self) -> str: return (f"COWFS({self.original_base_fs!r}, " f"{self.additions_fs!r}, " f"{self.deletions_fs!r})") ############################################################ def invariant(self) -> bool: if not self.additions_fs: raise ValueError(f"Invalid additions_fs: {self.additions_fs}.") if not self.deletions_fs: raise ValueError(f"Invalid deletions_fs: {self.additions_fs}.") if not self.base_fs: raise ValueError(f"Invalid base_fs: {self.base_fs}.") _deletions_invariant(self.deletions_fs) additions_paths = set(paths(self.additions_fs)) deletions_paths = { fs.path.dirname(file) for file in self.deletions_fs.walk.files() } if additions_paths > deletions_paths: raise ValueError(f"Additions_paths {additions_paths} " + "is not a subset of deletions_path " + f"{deletions_paths}. Extras are " + f"{additions_paths - deletions_paths}.") return True def is_deletion(self, path: str) -> bool: """ Is the path marked in the deletions_fs" """ return self.deletions_fs.exists(del_path(path)) def mark_deletion(self, path: str) -> None: """ Mark the path in the deletions_fs. """ self.deletions_fs.makedirs(path, None, True) self.deletions_fs.touch(del_path(path)) def makedirs_mark_deletion( self, path: str, permissions: Optional[Permissions] = None, recreate: bool = False, ) -> None: for p in fs.path.recursepath(path)[:-1]: self.additions_fs.makedirs(p, permissions=permissions, recreate=True) self.mark_deletion(p) self.additions_fs.makedir(path, permissions=permissions, recreate=recreate) self.mark_deletion(path) def layer(self, path: str) -> int: """ Get the layer on which the file lives, or ROOT_LAYER if it's the root path. """ if path == "/": return ROOT_LAYER if self.additions_fs.exists(path): return ADD_LAYER elif self.is_deletion(path): return NO_LAYER elif self.base_fs.exists(path): return BASE_LAYER else: return NO_LAYER def copy_up(self, path: str) -> None: """ Copy the file from the base_fs to additions_fs. """ self.makedirs_mark_deletion(fs.path.dirname(path)) self.mark_deletion(path) fs.copy.copy_file(self.base_fs, path, self.additions_fs, path) def triple_tree(self) -> None: print("base_fs ------------------------------") self.base_fs.tree() print("additions_fs ------------------------------") self.additions_fs.tree() print("deletions_fs ------------------------------") self.deletions_fs.tree() ############################################################ def getmeta(self, namespace: str = "standard") -> Mapping[str, object]: return self.base_fs.getmeta(namespace) def getinfo(self, path: str, namespaces: Optional[Collection[str]] = None) -> Info: self.check() self.validatepath(path) layer = self.layer(path) if layer == NO_LAYER: raise fs.errors.ResourceNotFound(path) elif layer == BASE_LAYER: return self.base_fs.getinfo(path, namespaces) elif layer == ADD_LAYER: return self.additions_fs.getinfo(path, namespaces) elif layer == ROOT_LAYER: # TODO implement this raw_info = {} if namespaces is None or "basic" in namespaces: raw_info["basic"] = {"name": "", "is_dir": True} return Info(raw_info) else: raise RuntimeError(f"Unknown layer {layer}.") def getsyspath(self, path: str) -> str: self.check() # self.validatepath(path) layer = self.layer(path) if layer == NO_LAYER: raise fs.errors.NoSysPath(path=path) elif layer == BASE_LAYER: return self.base_fs.getsyspath(path) elif layer == ADD_LAYER: return self.additions_fs.getsyspath(path) elif layer == ROOT_LAYER: raise fs.errors.NoSysPath(path=path) else: raise RuntimeError(f"Unknown layer {layer}.") def listdir(self, path: str) -> List[str]: self.check() self.validatepath(path) layer = self.layer(path) if layer == NO_LAYER: raise fs.errors.ResourceNotFound(path) elif layer == BASE_LAYER: return [ name for name in self.base_fs.listdir(path) if self.layer(fs.path.join(path, name)) != NO_LAYER ] elif layer == ADD_LAYER: # Get the listing on the additions layer names = set(self.additions_fs.listdir(path)) # Add in the listing on the base layer (if it exists) if self.base_fs.isdir(path): names |= set(self.base_fs.listdir(path)) # Return the entries that actually exist return [ name for name in list(names) if self.layer(fs.path.join(path, name)) != NO_LAYER ] elif layer == ROOT_LAYER: # Get the listing of the root on the additions layer and # the base layer. names = set(self.additions_fs.listdir("/")) names |= set(self.base_fs.listdir("/")) # Return the entries that actually exist. return [ name for name in list(names) if self.layer(name) != NO_LAYER ] else: raise RuntimeError(f"Unknown layer {layer}.") def makedir( self, path: str, permissions: Optional[Permissions] = None, recreate: bool = False, ) -> SubFS["COWFS"]: self.check() self.validatepath(path) # Check if it *can* be created. # get a normalized parent_dir path. parent_dir = fs.path.dirname(fs.path.forcedir(path)[:-1]) if not parent_dir: parent_dir = "/" if not self.isdir(parent_dir): raise fs.errors.ResourceNotFound(path) layer = self.layer(path) if layer == NO_LAYER: self.makedirs_mark_deletion(path, permissions=permissions, recreate=recreate) return SubFS(self, path) elif layer in [BASE_LAYER, ADD_LAYER, ROOT_LAYER]: if recreate: return SubFS(self, path) else: # I think this is wrong. What if it's a file? raise fs.errors.DirectoryExists(path) else: raise RuntimeError(f"Unknown layer {layer}.") def openbin(self, path: str, mode: str = "r", buffering: int = -1, **options: Any) -> BinaryIO: self.check() self.validatepath(path) parent_dir = fs.path.dirname(fs.path.forcedir(path)[:-1]) if not parent_dir: parent_dir = "/" if not self.isdir(parent_dir): raise fs.errors.ResourceNotFound(path) mode_obj = Mode(mode) layer = self.layer(path) if layer == NO_LAYER: if mode_obj.create: for p in fs.path.recursepath(path)[:-1]: self.additions_fs.makedirs(p, recreate=True) self.mark_deletion(p) self.mark_deletion(path) return self.additions_fs.openbin(path, mode, buffering, **options) else: raise fs.errors.ResourceNotFound(path) elif layer == ADD_LAYER: self.mark_deletion(path) return self.additions_fs.openbin(path, mode, buffering, **options) elif layer == BASE_LAYER: if mode_obj.writing: self.copy_up(path) return self.additions_fs.openbin(path, mode, buffering, **options) else: return self.base_fs.openbin(path, mode, buffering, **options) elif layer == ROOT_LAYER: raise fs.errors.FileExpected(path) else: raise RuntimeError(f"Unknown layer {layer}.") def remove(self, path: str) -> None: self.check() self.validatepath(path) layer = self.layer(path) if layer == NO_LAYER: raise fs.errors.ResourceNotFound(path) elif layer == BASE_LAYER: if self.base_fs.isfile(path): self.mark_deletion(path) else: raise fs.errors.FileExpected(path) elif layer == ADD_LAYER: self.additions_fs.remove(path) self.mark_deletion(path) elif layer == ROOT_LAYER: raise fs.errors.FileExpected(path) else: raise RuntimeError(f"Unknown layer {layer}.") def removedir(self, path: str) -> None: self.check() layer = self.layer(path) if layer == NO_LAYER: raise fs.errors.ResourceNotFound(path) elif layer == BASE_LAYER: if self.base_fs.isdir(path): self.mark_deletion(path) else: raise fs.errors.FileExpected(path) elif layer == ADD_LAYER: if self.additions_fs.isdir(path): self.additions_fs.removedir(path) self.mark_deletion(path) else: raise fs.errors.DirectoryExpected(path) elif layer == ROOT_LAYER: raise fs.errors.RemoveRootError(path) else: raise RuntimeError(f"Unknown layer {layer}.") def setinfo(self, path: str, info: _INFO_DICT) -> None: self.check() self.validatepath(path) layer = self.layer(path) if layer == NO_LAYER: raise fs.errors.ResourceNotFound(path) elif layer == BASE_LAYER: self.copy_up(path) self.additions_fs.setinfo(path, info) elif layer == ADD_LAYER: self.additions_fs.setinfo(path, info) elif layer == ROOT_LAYER: pass else: raise RuntimeError(f"Unknown layer {layer}.") ############################################################ def makedirs( self, path: str, permissions: Optional[Permissions] = None, recreate: bool = False, ) -> SubFS[FS]: return FS.makedirs(self, path, permissions=permissions, recreate=recreate)