def makedir(self, path, permissions=None, recreate=False):
path = self.fix_path(path)
if self.exists(path) and not recreate:
raise errors.DirectoryExists(path)
if path == "/":
return SubFS(self, path)
if self.exists(path):
meta = self.getinfo(path)
if meta.is_dir:
if recreate == False:
raise errors.DirectoryExists(path)
else:
return SubFS(self, path)
if meta.is_file:
raise errors.DirectoryExpected(path)
ppath = self.get_parent(path)
if not self.exists(ppath):
raise errors.ResourceNotFound(ppath)
try:
folderMetadata = self.dropbox.files_create_folder_v2(path)
except ApiError as e:
raise errors.DirectoryExpected(path=path)
return SubFS(self, path)
def makedir(
self,
path: str,
permissions: Optional[Permissions] = None,
recreate: bool = False,
) -> SubFS["COWFS"]:
self.check()
self.validatepath(path)
# Check if it *can* be created.
# get a normalized parent_dir path.
parent_dir = fs.path.dirname(fs.path.forcedir(path)[:-1])
if not parent_dir:
parent_dir = "/"
if not self.isdir(parent_dir):
raise fs.errors.ResourceNotFound(path)
layer = self.layer(path)
if layer == NO_LAYER:
self.makedirs_mark_deletion(path,
permissions=permissions,
recreate=recreate)
return SubFS(self, path)
elif layer in [BASE_LAYER, ADD_LAYER, ROOT_LAYER]:
if recreate:
return SubFS(self, path)
else:
# I think this is wrong. What if it's a file?
raise fs.errors.DirectoryExists(path)
else:
raise RuntimeError(f"Unknown layer {layer}.")
def dirs_match(dirpath: str) -> bool:
primary_dirs = filter_to_primary_dirs(
dirpath,
(
relpath(dir)
for dir in SubFS(primary_fs, dirpath).walk.dirs()
if "$" in dir
),
)
latest_dirs = filter_to_primary_dirs(
dirpath,
(
relpath(dir)
for dir in SubFS(latest_version_fs, dirpath).walk.dirs()
if "$" in dir
),
)
PDS_LOGGER.open("Directory changes detected")
if primary_dirs == latest_dirs:
for dir in primary_dirs:
full_dirpath = join(dirpath, relpath(dir))
lid = dirpath_to_lid(full_dirpath)
if lid not in result.changes_dict:
raise KeyError(f"{lid} not in changes_dict.")
if result.changed(lid):
PDS_LOGGER.log(
"info", f"CHANGE DETECTED in {dirpath}: {lid} changed"
)
PDS_LOGGER.close()
return False
PDS_LOGGER.close()
return True
else:
# list of dirs does not match
added = primary_dirs - latest_dirs
removed = latest_dirs - primary_dirs
if added and removed:
PDS_LOGGER.log(
"info",
f"CHANGE DETECTED IN {dirpath}: added {added}; removed {removed}",
)
elif added:
PDS_LOGGER.log("info", f"CHANGE DETECTED IN {dirpath}: added {added}")
else: # removed
PDS_LOGGER.log(
"info", f"CHANGE DETECTED IN {dirpath}: removed {removed}"
)
PDS_LOGGER.close()
return False
def create_from_lidvids_and_dirpath(
subcomponents: Set[LIDVID], fs: FS, dirpath: str
) -> "VersionContents[LIDVID]":
filepaths = component_files(fs, dirpath)
return VersionContents[LIDVID](
True, subcomponents, SubFS(fs, dirpath), set(filepaths)
)
def makedir(self,
path: str,
permissions: Optional[Permissions] = None,
recreate: bool = False) -> SubFS[FS]:
"""Make a directory.
Note:
As GCS is not a real filesystem but a key-value store that does not have any concept of directories, we write empty blobs as a work around.
See: https://fs-s3fs.readthedocs.io/en/latest/#limitations
This implementation currently ignores the `permissions` argument, the empty blobs are written with default permissions.
"""
self.check()
_path = self.validatepath(path)
_key = self._path_to_dir_key(_path)
if not self.isdir(dirname(_path)):
raise errors.ResourceNotFound(path)
try:
self.getinfo(path)
except errors.ResourceNotFound:
pass
else:
if recreate:
return self.opendir(_path)
else:
raise errors.DirectoryExists(path)
blob = self.bucket.blob(_key)
blob.upload_from_string(b"")
return SubFS(self, path)
def makedir(self, path, permissions=None, recreate=False):
_log.info(f'makedir({path}, {permissions}, {recreate})')
path = self.validatepath(path)
with self._lock:
parentDir = dirname(path)
# parentDir here is expected to have a leading slash
assert parentDir[0] == '/'
response = self.session.get_path(parentDir)
if response.status_code == 404:
raise ResourceNotFound(parentDir)
response.raise_for_status()
if recreate is False:
response = self.session.get_path(path)
if response.status_code != 404:
raise DirectoryExists(path)
response = self.session.post_path(
parentDir,
'/children',
json={
'name': basename(path),
'folder': {},
'@microsoft.graph.conflictBehavior': 'replace'
})
# TODO - will need to deal with these errors locally but don't know what they are yet
response.raise_for_status()
# don't need to close this filesystem so we return the non-closing version
return SubFS(self, path)
def makedir(self, path, permissions=None, recreate=False):
_CheckPath(path)
with self._lock:
parentDir = dirname(path)
# parentDir here is expected to have a leading slash
assert parentDir[0] == "/"
response = self.session.get(_PathUrl(parentDir, ""))
if response.status_code == 404:
raise ResourceNotFound(parentDir)
response.raise_for_status()
if recreate is False:
response = self.session.get(_PathUrl(path, ""))
if response.status_code != 404:
raise DirectoryExists(path)
response = self.session.post(_PathUrl(parentDir, ":/children"),
json={
"name": basename(path),
"folder": {}
})
# TODO - will need to deal with these errors locally but don't know what they are yet
response.raise_for_status()
# don't need to close this filesystem so we return the non-closing version
return SubFS(self, path)
def makedir(self, path, permissions=None, recreate=False):
"""
Create a directory under `path`.
:param str path: Path pointing to a file.
:param Permissions permissions: PyFilesystem permission instance
:param bool recreate: Not supported
"""
# type: (Text, Permissions, bool) -> SubFS
path = ensureUnicode(path)
self.check()
_path = toAscii(self.validatepath(path))
if not self.isdir(dirname(_path)):
raise errors.ResourceNotFound(path)
if permissions is None:
permissions = Permissions(user='******', group='r-x', other='r-x')
try:
self.getinfo(path)
except errors.ResourceNotFound:
self._odfs.mkdir(_path, permissions.mode)
return SubFS(self, path)
def files_match(dirpath: str) -> bool:
# All files in subcomponents will have a "$" in their path (it
# comes after the name of the subcomponent), so by filtering
# them out, we get only the files for this component. PDS4
# *does* allow directories in a component (that aren't part of
# a subcomponent), so we use walk instead of listdir() to get
# *all* the files, not just the top-level ones.
primary_files = filter_to_primary_files(
dirpath,
(
relpath(filepath)
for filepath in SubFS(primary_fs, dirpath).walk.files()
if "$" not in filepath
),
)
latest_files = filter_to_primary_files(
dirpath,
(
relpath(filepath)
for filepath in SubFS(latest_version_fs, dirpath).walk.files()
if "$" not in filepath
),
)
try:
PDS_LOGGER.open("File changes detected")
if primary_files != latest_files:
PDS_LOGGER.log(
"info",
f"CHANGE DETECTED IN {dirpath}: {primary_files} != {latest_files}",
)
PDS_LOGGER.close()
return False
for filename in primary_files:
filepath = join(dirpath, relpath(filename))
if primary_fs.getbytes(filepath) != latest_version_fs.getbytes(
filepath
):
PDS_LOGGER.log(
"info", f"CHANGE DETECTED IN {filepath}; DIRPATH = {dirpath}"
)
PDS_LOGGER.close()
return False
except Exception as e:
PDS_LOGGER.exception(e)
finally:
PDS_LOGGER.close()
return True
def component_files(fs: FS, dirpath: str) -> Iterator[str]:
"""
Returns the relative filepaths for files that are contained within
the directory for a PDS4 component. Files in PDS4 subcomponents
will have "$" in their paths (in the subcomponent directory name).
"""
return (abspath(filepath) for filepath in SubFS(fs, dirpath).walk.files()
if "$" not in filepath)
def makedir(self, path, permissions=None, recreate=False):
try:
folderMetadata = self.dropbox.files_create_folder(path)
except ApiError as e:
assert isinstance(e.reason, CreateFolderError)
# TODO - there are other possibilities
raise DirectoryExpected(path=path)
# don't need to close this filesystem so we return the non-closing version
return SubFS(self, path)
def makedir(self, path, permissions=None, recreate=False):
parentMetadata = self._itemFromPath(dirname(path))
if parentMetadata is None:
raise DirectoryExpected(path=path)
childMetadata = self._childByName(parentMetadata["id"], basename(path))
if childMetadata is not None:
if recreate is False:
raise DirectoryExists(path=path)
else:
return SubFS(self, path)
newMetadata = {
"name": basename(path),
"parents": [parentMetadata["id"]],
"mimeType": _folderMimeType
}
newMetadataId = self.drive.files().create(body=newMetadata,
fields="id").execute()
return SubFS(self, path)
def _create_subdirectory(self, name, path, parents):
newMetadata = {
"name": basename(name),
"parents": parents,
"mimeType": _folderMimeType
}
self.drive.files().create(
body=newMetadata, fields="id").execute(num_retries=self.retryCount)
return SubFS(self, path)
def makedir(self, path, permissions=None, recreate=False):
with self._lock:
abs_path = self.validatepath(path)
try:
self._saver.saveBytes(b'',
abs_path,
compound_type=Compound.DIR_TYPE)
except CompoundAlreadyExistsException:
if not recreate:
raise fs.errors.DirectoryExists(abs_path)
return SubFS(self, abs_path)
return self.opendir(abs_path)
def makedir(self, path, permissions=None, recreate=False):
_CheckPath(path)
with self._lock:
info(f"makedir: {path}, {permissions}, {recreate}")
parentMetadata = self._itemFromPath(dirname(path))
if parentMetadata is None:
raise ResourceNotFound(path=path)
childMetadata = self._childByName(parentMetadata["id"],
basename(path))
if childMetadata is not None:
if recreate is False:
raise DirectoryExists(path=path)
else:
return SubFS(self, path)
newMetadata = {
"name": basename(path),
"parents": [parentMetadata["id"]],
"mimeType": _folderMimeType
}
_ = self.drive.files().create(body=newMetadata,
fields="id").execute()
return SubFS(self, path)
def makedir(self, path, permissions=None, recreate=False):
self.check()
if not self.strict:
return SubFS(self, path)
_path = self.validatepath(path)
_key = self._path_to_dir_key(_path)
if self.strict and not self.isdir(dirname(_path)):
raise errors.ResourceNotFound(path)
try:
self._getinfo(path)
except errors.ResourceNotFound:
pass
else:
if recreate:
return self.opendir(_path)
else:
raise errors.DirectoryExists(path)
with s3errors(path):
_obj = self.s3.Object(self._bucket_name, _key)
_obj.put(**self._get_upload_args(_key))
return SubFS(self, path)
def makedir(self, path: Text, permissions: Optional[Permissions] = None, recreate: bool = False) -> SubFS[FS]:
""" Make a directory. """
npath = self.normalize_path(path)
if self.exists(npath):
if recreate:
return SubFS(self, npath)
raise errors.DirectoryExists(path)
if npath == "/":
return SubFS(self, path)
if not self.exists(dirname(npath)):
raise errors.ResourceNotFound(dirname(path))
perm = 0o750
if permissions is not None:
perm = permissions.mode
cursor = self.connection.cursor()
cursor.execute(
"INSERT INTO sqlar (name, mode, mtime, sz, data) VALUES (?, ?, ?, ?, ?)",
(npath, perm, datetime.utcnow().timestamp(), 0, None)
)
cursor.close()
return SubFS(self, npath)
def makedir(self, path, permissions=None, recreate=False): parentDir = dirname(path) itemRequest = self.client.item(path=parentDir) try: item = itemRequest.get() except OneDriveError as e: raise ResourceNotFound(path=parentDir, exc=e) if item.folder is None: raise DirectoryExpected(path=parentDir) newItem = Item() newItem.name = basename(path) newItem.folder = Folder() itemRequest.children.add(entity=newItem) # don't need to close this filesystem so we return the non-closing version return SubFS(self, path)
def makedir(self, path, permissions=None, recreate=False):
_CheckPath(path)
with self._lock:
_log.info(f"makedir: {path}, {permissions}, {recreate}")
parentMetadata = self._itemFromPath(dirname(path))
if parentMetadata is None:
raise ResourceNotFound(path=path)
childMetadata = self._childByName(parentMetadata["id"], basename(path))
if childMetadata is not None:
if recreate is False:
raise DirectoryExists(path=path)
return SubFS(self, path)
return self._createSubdirectory(basename(path), path, [parentMetadata["id"]])
def __getitem__(self, lidvid: LIDVID) -> VersionContents:
dirpath = lidvid_path(lidvid)
if not self.fs.isdir(dirpath):
raise KeyError(lidvid)
d = read_subdir_versions_from_directory(self.fs, dirpath)
def make_sub_lidvid(seg: str, vid_part: str) -> LIDVID:
lid_parts = lidvid.lid().parts()
lid_parts.append(seg)
return LIDVID.create_from_lid_and_vid(
LID.create_from_parts(lid_parts), VID(vid_part))
lidvids = {
make_sub_lidvid(segment, vid)
for segment, vid in list(d.items())
}
sub_fs = SubFS(self.fs, dirpath)
filepaths = set(sub_fs.walk.files(exclude=[SUBDIR_VERSIONS_FILENAME]))
return VersionContents.create_from_lidvids(lidvids, sub_fs, filepaths)
def makedir(self, path, permissions=None, recreate=False):
self.check()
_path = self.validatepath(path)
_key = self._path_to_dir_key(_path)
if not self.isdir(dirname(_path.rstrip("/"))):
raise errors.ResourceNotFound(path)
try:
self.getinfo(path)
except errors.ResourceNotFound:
pass
else:
if recreate:
return self.opendir(_path)
raise errors.DirectoryExists(path)
with dlkerrors(path):
self.dlk.mkdir(_key)
return SubFS(self, _path)
def makedir(self, path, permissions=None, recreate=False):
_CheckPath(path)
with self._lock:
info(f"makedir: {path}, {permissions}, {recreate}")
parentMetadata = self._itemFromPath(dirname(path))
if isinstance(parentMetadata, list): # adding new folder to root folder
if not self._childByName(None, path):
return self._create_subdirectory(path, path, None)
raise DirectoryExists(path=path)
if parentMetadata is None:
raise ResourceNotFound(path=path)
childMetadata = self._childByName(parentMetadata["id"], basename(path))
if childMetadata is not None:
if recreate is False:
raise DirectoryExists(path=path)
return SubFS(self, path)
return self._create_subdirectory(basename(path), path, [parentMetadata["id"]])
def _merge_primaries(changes_dict: ChangesDict, src_fs: FS,
dst_fs: FS) -> None:
# TODO Not sure that this hits all cases, including removal of
# files and directories. Think about it.
for dirpath in src_fs.walk.dirs(search="depth"):
if _is_component_path(dirpath):
lid = dirpath_to_lid(dirpath)
changed = changes_dict.changed(lid)
if changed:
if not dst_fs.isdir(dirpath):
dst_fs.makedirs(dirpath)
src_sub_fs = SubFS(src_fs, dirpath)
dst_sub_fs = SubFS(dst_fs, dirpath)
# delete directories in dst that don't exist in src
for subdirpath in dst_sub_fs.walk.dirs(search="depth"):
if not src_sub_fs.isdir(subdirpath):
dst_sub_fs.removetree(subdirpath)
# delete the files in the destination (if any)
for filepath in component_files(dst_fs, dirpath):
dst_sub_fs.remove(filepath)
# copy the new files across
src_sub_fs = SubFS(src_fs, dirpath)
for filepath in component_files(src_fs, dirpath):
fs.copy.copy_file(src_sub_fs, filepath, dst_sub_fs,
filepath)
def makedirs(self, path, permissions=None, recreate=False):
if not self.strict:
self.check()
return SubFS(self, path)
return super(S3FS, self).makedirs(path, permissions, recreate)
def makedir(self, path, permissions=None, recreate=False):
self.check()
self._c.create_directory_by_path(self._project_id, path)
return SubFS(self, path)
def makedir(self, path: str, permissions: Permissions = None,
recreate: bool = False):
"""Create directory on filesystem.
:param path: Path of new directory on filesystem
:param permissions: Currently not implemented
:param recreate: Ignore if directory already exists
"""
path = normpath(path)
base = split(path)[0]
dirname = split(path)[1]
# Plausability checks
try:
self.opendir(base)
except DirectoryExpected:
raise ResourceNotFound(path)
base = self._get_dir_entry(base)
try:
self._get_dir_entry(path)
except ResourceNotFound:
pass
else:
if not recreate:
raise DirectoryExists(path)
else:
# TODO: Update mtime
return SubFS(self, path)
parent_is_root = base == self.fs.root_dir
# Determine 8DOT3 file name + LFN
short_name = EightDotThree()
n = short_name.make_8dot3_name(dirname, base)
short_name.set_str_name(n)
dt = DosDateTime.now(tz=self.tz)
newdir = FATDirectoryEntry(DIR_Name=short_name,
DIR_Attr=FATDirectoryEntry.ATTR_DIRECTORY,
DIR_NTRes=0,
DIR_CrtTimeTenth=0,
DIR_CrtTime=dt.serialize_time(),
DIR_CrtDate=dt.serialize_date(),
DIR_LstAccessDate=dt.serialize_date(),
DIR_FstClusHI=0x00,
DIR_WrtTime=dt.serialize_time(),
DIR_WrtDate=dt.serialize_date(),
DIR_FstClusLO=0x00,
DIR_FileSize=0,
encoding=self.fs.encoding)
# Create LFN entry if required
_sfn = short_name.get_unpadded_filename()
if _sfn != dirname.upper() or (_sfn != dirname and self.preserve_case):
lfn_entry = make_lfn_entry(dirname, short_name)
newdir.set_lfn_entry(lfn_entry)
# Create . and .. directory entries
first_cluster = self.fs.allocate_bytes(
FATDirectoryEntry.FAT_DIRECTORY_HEADER_SIZE * 2,
erase=True)[0]
newdir.set_cluster(first_cluster)
dot_sn = EightDotThree()
dot_sn.set_byte_name(b". ")
dot = FATDirectoryEntry(DIR_Name=dot_sn,
DIR_Attr=FATDirectoryEntry.ATTR_DIRECTORY,
DIR_NTRes=newdir.ntres,
DIR_CrtTimeTenth=newdir.crttimetenth,
DIR_CrtTime=newdir.crttime,
DIR_CrtDate=newdir.crtdate,
DIR_LstAccessDate=newdir.lstaccessdate,
DIR_FstClusHI=newdir.fstclushi,
DIR_WrtTime=newdir.wrttime,
DIR_WrtDate=newdir.wrtdate,
DIR_FstClusLO=newdir.fstcluslo,
DIR_FileSize=newdir.filesize,
encoding=self.fs.encoding)
dotdot_sn = EightDotThree()
dotdot_sn.set_byte_name(b".. ")
base_fstclushi = base.fstclushi if not parent_is_root else 0x0
base_fstcluslo = base.fstcluslo if not parent_is_root else 0x0
dotdot = FATDirectoryEntry(DIR_Name=dotdot_sn,
DIR_Attr=FATDirectoryEntry.ATTR_DIRECTORY,
DIR_NTRes=base.ntres,
DIR_CrtTimeTenth=base.crttimetenth,
DIR_CrtTime=base.crttime,
DIR_CrtDate=base.crtdate,
DIR_LstAccessDate=base.lstaccessdate,
DIR_FstClusHI=base_fstclushi,
DIR_WrtTime=base.wrttime,
DIR_WrtDate=base.wrtdate,
DIR_FstClusLO=base_fstcluslo,
DIR_FileSize=base.filesize,
encoding=self.fs.encoding)
newdir.add_subdirectory(dot)
newdir.add_subdirectory(dotdot)
# Write new directory contents
self.fs.update_directory_entry(newdir)
# Write parent directory
base.add_subdirectory(newdir)
self.fs.update_directory_entry(base)
# Flush FAT(s) to disk
self.fs.flush_fat()
return SubFS(self, path)