else:
print_dir(fs, pathjoin(path, item),
levels[:] + [is_last_item])
else:
write('%s %s' %
(wrap_prefix(prefix + char_line), wrap_filename(item)))
return len(dir_listing)
print_dir(fs, path)
return dircount[0], filecount[0]
if __name__ == "__main__":
from fs.tempfs import TempFS
from six import b
t1 = TempFS()
t1.setcontents("foo", b("test"))
t1.makedir("bar")
t1.setcontents("bar/baz", b("another test"))
t1.tree()
t2 = TempFS()
print t2.listdir()
movedir(t1, t2)
print t2.listdir()
t1.tree()
t2.tree()
#write(wrap_prefix(prefix[:-1] + ' ') + wrap_error('max recursion levels reached'))
else:
print_dir(fs, pathjoin(path, item), levels[:] + [is_last_item])
else:
write('%s %s' % (wrap_prefix(prefix + char_line), wrap_filename(item)))
return len(dir_listing)
print_dir(fs, path)
return dircount[0], filecount[0]
if __name__ == "__main__":
from fs.tempfs import TempFS
from six import b
t1 = TempFS()
t1.setcontents("foo", b("test"))
t1.makedir("bar")
t1.setcontents("bar/baz", b("another test"))
t1.tree()
t2 = TempFS()
print t2.listdir()
movedir(t1, t2)
print t2.listdir()
t1.tree()
t2.tree()
class RegistryFileOpener(WinRegistryFileReader):
"""
This is a callback class used by dfwinreg to open registry hive files.
We are using dfvfs as the backend to open files within our image.
To resolve system variables, we make use of the variable database of
the WindowsSystem instance.
"""
# pylint: disable=too-few-public-methods
def __init__(self, dfvfs, partition, windows_system):
super(RegistryFileOpener, self).__init__()
self.dfvfs = dfvfs
self.partition = partition
self.not_present = set()
self.open_handles = []
self.tmpfs = TempFS()
self.windows_system = windows_system
# callbacks.register_on_job_end(self._cleanup_open_files)
def _cleanup_open_files(self, __):
for path, handle in self.open_handles:
try:
handle.close()
self.tmpfs.remove(path)
except (OSError, FSError) as err:
LOGGER.warning("Error cleaning up %s: %s", path, err)
self.tmpfs.close()
def Open(self, path, ascii_codepage='cp1252'):
LOGGER.info("open registry %s", path)
""" Opens a path within the dfVFS volume """
realpath = path.replace('\\', '/')
if path in self.not_present:
return None
# check for variables and if we know them
realpath = path
for match in re.finditer('%[a-zA-Z0-9_]+%', path):
key = match.group(0)
val = self.windows_system.get_var(key)
if val:
realpath = realpath.replace(key, val)
else:
LOGGER.warning("Could not resolve variable %s", key)
return None
realpath = realpath.replace('\\', '/')
if realpath.lower().startswith('c:/'): # catch absolute paths
realpath = '/' + realpath[3:]
if not realpath[0] == '/':
realpath = '/' + realpath
if realpath in self.not_present:
return None
path_specs = list(
self.dfvfs.find_paths([realpath], partitions=[self.partition]))
if not path_specs:
LOGGER.warning("Could not find requested registry hive %s [%s]",
path, realpath)
self.not_present.add(path)
self.not_present.add(realpath)
return None
if len(path_specs) > 1:
LOGGER.warning(
"Found multiple registry hives for query %s, using %s", path,
dfvfs_helper.reconstruct_full_path(path_specs[0]))
# extract the file locally
filename = realpath.replace('/', '_')
dfvfs_helper.export_file(path_specs[0], self.tmpfs, filename)
try:
file_object = self.tmpfs.open(filename, 'rb')
except ResourceNotFound:
files = self.tmpfs.listdir("/")
LOGGER.warning("Could not open registry hive %s [%s] (%s)", path,
realpath, files)
return None
self.open_handles.append((filename, file_object))
reg_file = regfile_impl.REGFWinRegistryFile(
ascii_codepage=ascii_codepage)
reg_file.Open(file_object)
return reg_file
class COWFS(FS):
def __init__(
self,
base_fs: FS,
additions_fs: Optional[FS] = None,
deletions_fs: Optional[FS] = None,
) -> None:
FS.__init__(self)
if additions_fs:
self.additions_fs = additions_fs
else:
self.additions_fs = TempFS()
if deletions_fs:
_deletions_invariant(deletions_fs)
self.deletions_fs = deletions_fs
else:
self.deletions_fs = TempFS()
self.original_base_fs = base_fs
self.base_fs = fs.wrap.read_only(base_fs)
self.invariant()
@staticmethod
def create_cowfs(base_fs: FS,
read_write_layer: FS,
recreate: bool = False) -> "COWFS":
additions_fs = read_write_layer.makedir("/additions",
recreate=recreate)
deletions_fs = read_write_layer.makedir("/deletions",
recreate=recreate)
return COWFS(base_fs, additions_fs, deletions_fs)
def __str__(self) -> str:
return (f"COWFS({self.original_base_fs}, "
f"{self.additions_fs}, "
f"{self.deletions_fs})")
def __repr__(self) -> str:
return (f"COWFS({self.original_base_fs!r}, "
f"{self.additions_fs!r}, "
f"{self.deletions_fs!r})")
############################################################
def invariant(self) -> bool:
if not self.additions_fs:
raise ValueError(f"Invalid additions_fs: {self.additions_fs}.")
if not self.deletions_fs:
raise ValueError(f"Invalid deletions_fs: {self.additions_fs}.")
if not self.base_fs:
raise ValueError(f"Invalid base_fs: {self.base_fs}.")
_deletions_invariant(self.deletions_fs)
additions_paths = set(paths(self.additions_fs))
deletions_paths = {
fs.path.dirname(file)
for file in self.deletions_fs.walk.files()
}
if additions_paths > deletions_paths:
raise ValueError(f"Additions_paths {additions_paths} " +
"is not a subset of deletions_path " +
f"{deletions_paths}. Extras are " +
f"{additions_paths - deletions_paths}.")
return True
def is_deletion(self, path: str) -> bool:
"""
Is the path marked in the deletions_fs"
"""
return self.deletions_fs.exists(del_path(path))
def mark_deletion(self, path: str) -> None:
"""
Mark the path in the deletions_fs.
"""
self.deletions_fs.makedirs(path, None, True)
self.deletions_fs.touch(del_path(path))
def makedirs_mark_deletion(
self,
path: str,
permissions: Optional[Permissions] = None,
recreate: bool = False,
) -> None:
for p in fs.path.recursepath(path)[:-1]:
self.additions_fs.makedirs(p,
permissions=permissions,
recreate=True)
self.mark_deletion(p)
self.additions_fs.makedir(path,
permissions=permissions,
recreate=recreate)
self.mark_deletion(path)
def layer(self, path: str) -> int:
"""
Get the layer on which the file lives, or ROOT_LAYER if it's the
root path.
"""
if path == "/":
return ROOT_LAYER
if self.additions_fs.exists(path):
return ADD_LAYER
elif self.is_deletion(path):
return NO_LAYER
elif self.base_fs.exists(path):
return BASE_LAYER
else:
return NO_LAYER
def copy_up(self, path: str) -> None:
"""
Copy the file from the base_fs to additions_fs.
"""
self.makedirs_mark_deletion(fs.path.dirname(path))
self.mark_deletion(path)
fs.copy.copy_file(self.base_fs, path, self.additions_fs, path)
def triple_tree(self) -> None:
print("base_fs ------------------------------")
self.base_fs.tree()
print("additions_fs ------------------------------")
self.additions_fs.tree()
print("deletions_fs ------------------------------")
self.deletions_fs.tree()
############################################################
def getmeta(self, namespace: str = "standard") -> Mapping[str, object]:
return self.base_fs.getmeta(namespace)
def getinfo(self,
path: str,
namespaces: Optional[Collection[str]] = None) -> Info:
self.check()
self.validatepath(path)
layer = self.layer(path)
if layer == NO_LAYER:
raise fs.errors.ResourceNotFound(path)
elif layer == BASE_LAYER:
return self.base_fs.getinfo(path, namespaces)
elif layer == ADD_LAYER:
return self.additions_fs.getinfo(path, namespaces)
elif layer == ROOT_LAYER:
# TODO implement this
raw_info = {}
if namespaces is None or "basic" in namespaces:
raw_info["basic"] = {"name": "", "is_dir": True}
return Info(raw_info)
else:
raise RuntimeError(f"Unknown layer {layer}.")
def getsyspath(self, path: str) -> str:
self.check()
# self.validatepath(path)
layer = self.layer(path)
if layer == NO_LAYER:
raise fs.errors.NoSysPath(path=path)
elif layer == BASE_LAYER:
return self.base_fs.getsyspath(path)
elif layer == ADD_LAYER:
return self.additions_fs.getsyspath(path)
elif layer == ROOT_LAYER:
raise fs.errors.NoSysPath(path=path)
else:
raise RuntimeError(f"Unknown layer {layer}.")
def listdir(self, path: str) -> List[str]:
self.check()
self.validatepath(path)
layer = self.layer(path)
if layer == NO_LAYER:
raise fs.errors.ResourceNotFound(path)
elif layer == BASE_LAYER:
return [
name for name in self.base_fs.listdir(path)
if self.layer(fs.path.join(path, name)) != NO_LAYER
]
elif layer == ADD_LAYER:
# Get the listing on the additions layer
names = set(self.additions_fs.listdir(path))
# Add in the listing on the base layer (if it exists)
if self.base_fs.isdir(path):
names |= set(self.base_fs.listdir(path))
# Return the entries that actually exist
return [
name for name in list(names)
if self.layer(fs.path.join(path, name)) != NO_LAYER
]
elif layer == ROOT_LAYER:
# Get the listing of the root on the additions layer and
# the base layer.
names = set(self.additions_fs.listdir("/"))
names |= set(self.base_fs.listdir("/"))
# Return the entries that actually exist.
return [
name for name in list(names) if self.layer(name) != NO_LAYER
]
else:
raise RuntimeError(f"Unknown layer {layer}.")
def makedir(
self,
path: str,
permissions: Optional[Permissions] = None,
recreate: bool = False,
) -> SubFS["COWFS"]:
self.check()
self.validatepath(path)
# Check if it *can* be created.
# get a normalized parent_dir path.
parent_dir = fs.path.dirname(fs.path.forcedir(path)[:-1])
if not parent_dir:
parent_dir = "/"
if not self.isdir(parent_dir):
raise fs.errors.ResourceNotFound(path)
layer = self.layer(path)
if layer == NO_LAYER:
self.makedirs_mark_deletion(path,
permissions=permissions,
recreate=recreate)
return SubFS(self, path)
elif layer in [BASE_LAYER, ADD_LAYER, ROOT_LAYER]:
if recreate:
return SubFS(self, path)
else:
# I think this is wrong. What if it's a file?
raise fs.errors.DirectoryExists(path)
else:
raise RuntimeError(f"Unknown layer {layer}.")
def openbin(self,
path: str,
mode: str = "r",
buffering: int = -1,
**options: Any) -> BinaryIO:
self.check()
self.validatepath(path)
parent_dir = fs.path.dirname(fs.path.forcedir(path)[:-1])
if not parent_dir:
parent_dir = "/"
if not self.isdir(parent_dir):
raise fs.errors.ResourceNotFound(path)
mode_obj = Mode(mode)
layer = self.layer(path)
if layer == NO_LAYER:
if mode_obj.create:
for p in fs.path.recursepath(path)[:-1]:
self.additions_fs.makedirs(p, recreate=True)
self.mark_deletion(p)
self.mark_deletion(path)
return self.additions_fs.openbin(path, mode, buffering,
**options)
else:
raise fs.errors.ResourceNotFound(path)
elif layer == ADD_LAYER:
self.mark_deletion(path)
return self.additions_fs.openbin(path, mode, buffering, **options)
elif layer == BASE_LAYER:
if mode_obj.writing:
self.copy_up(path)
return self.additions_fs.openbin(path, mode, buffering,
**options)
else:
return self.base_fs.openbin(path, mode, buffering, **options)
elif layer == ROOT_LAYER:
raise fs.errors.FileExpected(path)
else:
raise RuntimeError(f"Unknown layer {layer}.")
def remove(self, path: str) -> None:
self.check()
self.validatepath(path)
layer = self.layer(path)
if layer == NO_LAYER:
raise fs.errors.ResourceNotFound(path)
elif layer == BASE_LAYER:
if self.base_fs.isfile(path):
self.mark_deletion(path)
else:
raise fs.errors.FileExpected(path)
elif layer == ADD_LAYER:
self.additions_fs.remove(path)
self.mark_deletion(path)
elif layer == ROOT_LAYER:
raise fs.errors.FileExpected(path)
else:
raise RuntimeError(f"Unknown layer {layer}.")
def removedir(self, path: str) -> None:
self.check()
layer = self.layer(path)
if layer == NO_LAYER:
raise fs.errors.ResourceNotFound(path)
elif layer == BASE_LAYER:
if self.base_fs.isdir(path):
self.mark_deletion(path)
else:
raise fs.errors.FileExpected(path)
elif layer == ADD_LAYER:
if self.additions_fs.isdir(path):
self.additions_fs.removedir(path)
self.mark_deletion(path)
else:
raise fs.errors.DirectoryExpected(path)
elif layer == ROOT_LAYER:
raise fs.errors.RemoveRootError(path)
else:
raise RuntimeError(f"Unknown layer {layer}.")
def setinfo(self, path: str, info: _INFO_DICT) -> None:
self.check()
self.validatepath(path)
layer = self.layer(path)
if layer == NO_LAYER:
raise fs.errors.ResourceNotFound(path)
elif layer == BASE_LAYER:
self.copy_up(path)
self.additions_fs.setinfo(path, info)
elif layer == ADD_LAYER:
self.additions_fs.setinfo(path, info)
elif layer == ROOT_LAYER:
pass
else:
raise RuntimeError(f"Unknown layer {layer}.")
############################################################
def makedirs(
self,
path: str,
permissions: Optional[Permissions] = None,
recreate: bool = False,
) -> SubFS[FS]:
return FS.makedirs(self,
path,
permissions=permissions,
recreate=recreate)
if self._f is not None:
self._f.close()
def open_atomic_write(fs, path, mode='w'):
"""Open a file for 'atomic' writing
This returns a context manager which ensures that a file is written in its entirety or not at all.
"""
return AtomicWriter(fs, path, mode=mode)
if __name__ == "__main__":
from fs.tempfs import TempFS
from six import b
t1 = TempFS()
t1.setcontents("foo", b("test"))
t1.makedir("bar")
t1.setcontents("bar/baz", b("another test"))
t1.tree()
t2 = TempFS()
print(t2.listdir())
movedir(t1, t2)
print(t2.listdir())
t1.tree()
t2.tree()
def open_atomic_write(fs, path, mode='w'):
"""Open a file for 'atomic' writing
This returns a context manager which ensures that a file is written in its entirety or not at all.
"""
return AtomicWriter(fs, path, mode=mode)
if __name__ == "__main__":
from fs.tempfs import TempFS
from six import b
t1 = TempFS()
t1.setcontents("foo", b("test"))
t1.makedir("bar")
t1.setcontents("bar/baz", b("another test"))
t1.tree()
t2 = TempFS()
print(t2.listdir())
movedir(t1, t2)
print(t2.listdir())
t1.tree()
t2.tree()
tempfs = TempFS()
for num in range(0, 100):
with tempfs.open('file' + str(num), 'w') as fi:
fi.write("I AM A FILE\n" * num)
# Connect to S3
# s3fs = S3FS(
# 'default',
# aws_access_key_id="minio",
# aws_secret_access_key="minio123",
# endpoint_url="http://*****:*****@default/?endpoint_url=http://localhost:9009')
# Print some info for us to use.
count = len(s3fs.listdir('/'))
print("File Count in S3: {}".format(count))
count = len(tempfs.listdir('/'))
print("File Count in tempfs: {}".format(count))
# Upload the files to S3
files = tempfs.listdir('/')
for fi in files:
with s3fs.open(fi, 'w') as s3fi:
s3fi.write(tempfs.open(fi, 'r').read())
count = len(s3fs.listdir('/'))
print("File Count in S3: {}".format(count))
