def _list_dir(self, dir_drive: str, dir_tail: str) -> List[str]:
response: Dict[str, List[Dict[str, str]]]
try:
response = self._s3_client.list_objects(
Bucket=dir_drive,
Prefix=dir_tail
)
except Exception:
raise FileSystemOperationError(traceback.format_exc())
items: List[str] = []
for item in response.get("Contents", []):
items.append(item["Key"])
while response.get("IsTruncated", False):
marker = response.get("NextMarker") or items[-1]
try:
response = self._s3_client.list_objects(
Bucket=dir_drive,
Prefix=dir_tail,
Marker=marker
)
except Exception:
raise FileSystemOperationError(traceback.format_exc())
for item in response.get("Contents", []):
items.append(item["Key"])
return items
def file_read(self, path: Path) -> bytes:
parent_dir: MemoryDirectory
try:
parent_dir = self._get_dir(path.parent)
return parent_dir.get_file(path.basename).contents
except (NotADirectoryError, NotAFileError):
raise FileSystemOperationError(traceback.format_exc())
def _remove(self, path: Path, is_dir: bool) -> None:
tail_str: str = self._separator.join(path.tail)
if is_dir and tail_str:
tail_str += self._separator
response: Dict[str, Union[bool, List[Dict[str, str]]]] = {
"IsTruncated": True
}
while response.get("IsTruncated", False):
contents = cast(List[Dict[str, str]], response.get("Contents", []))
marker = cast(str, response.get("NextMarker"))
if not marker and contents:
marker = contents[-1]["Key"]
if not marker:
marker = ""
try:
response = self._s3_client.list_objects(
Bucket=path.drive,
Prefix=tail_str,
Marker=marker,
)
contents = cast(List[Dict[str, str]],
response.get('Contents', []))
if contents:
self._s3_client.delete_objects(
Bucket=path.drive,
Delete={
'Objects': [{
'Key': f['Key']
} for f in contents
if is_dir or f['Key'] == tail_str]
})
except Exception:
raise FileSystemOperationError(traceback.format_exc())
def file_write(self, path: Path, content: bytes) -> None:
try:
parent_path: str = self.path_to_string(path.parent)
if not os.path.exists(parent_path):
os.makedirs(parent_path)
with open(self.path_to_string(path), "wb") as f:
f.write(content)
except Exception:
raise FileSystemOperationError(traceback.format_exc())
def file_read(self, path: Path) -> bytes:
prefix: str = self._separator.join(path.tail)
try:
data = BytesIO()
self._s3_client.download_fileobj(path.drive, prefix, data)
data.seek(0)
return data.read()
except Exception:
raise FileSystemOperationError(traceback.format_exc())
def dir_list(self, path: Path) -> Iterable[FSObjectPath]:
tail_str: str = self._separator.join(path.tail)
if tail_str:
tail_str += self._separator
response: Dict[str, Union[bool, List[Dict[str, str]], str]] = {
"IsTruncated": True
}
marker = ""
while response.get("IsTruncated", False):
try:
response = self._s3_client.list_objects(
Bucket=path.drive,
Prefix=tail_str,
Delimiter=self._separator,
Marker=marker)
except Exception:
raise FileSystemOperationError(traceback.format_exc())
contents = cast(List[Dict[str, str]], response.get("Contents", []))
for item in contents:
file_name: str = item["Key"]
if file_name != tail_str:
if file_name.endswith(self._separator):
yield FSObjectPath(
FSObjectType.DIR,
Path(path.drive,
*(file_name.split(self._separator)[:-1])))
else:
yield FSObjectPath(
FSObjectType.FILE,
Path(path.drive,
*file_name.split(self._separator)))
for item in cast(List[Dict[str, str]],
response.get("CommonPrefixes", [])):
dir_name: str = item["Prefix"]
yield FSObjectPath(
FSObjectType.DIR,
Path(path.drive, *(dir_name.split(self._separator)[:-1])))
marker = cast(str, response.get("NextMarker"))
if not marker and contents:
marker = contents[-1]["Key"]
if not marker:
marker = ""
def file_write(self, path: Path, content: bytes) -> None:
try:
dirs: Tuple[str, ...] = path.parent.tail
acc_prefix: str = ""
for dir_ in dirs:
if acc_prefix:
acc_prefix = self._separator.join((acc_prefix, dir_))
else:
acc_prefix = dir_
self._s3_client.upload_fileobj(BytesIO(b""), path.drive,
acc_prefix + self._separator)
path_str: str = self.path_to_string(path)
tail: str = path_str[path_str.find("/") + 1:]
self._s3_client.upload_fileobj(BytesIO(content), path.drive, tail)
except Exception:
raise FileSystemOperationError(traceback.format_exc())
def file_set_perms(self, path: Path, policies: List[Policy]) -> None:
""" Set ACL policies for the object.
Check `norfs.fs.s3.s3_scopes` and `norfs.fs.s3.s3_perms` to understand how :class:`norfs.permissions.Scope` and
:class:`norfs.permissions.Perm` map to S3 Grantees and Permissions.
"""
try:
key = self._separator.join(path.tail)
acl = self._s3_client.get_object_acl(Bucket=path.drive, Key=key)
grants = []
for policy in policies:
if policy.scope == Scope.OWNER:
grantee = acl['Owner'].copy()
grantee['Type'] = 'CanonicalUser'
else:
grantee = s3_scopes.get(policy.scope)
if all((p in policy.perms for p in _ALL_PERMS)):
grants.append({
'Grantee': grantee,
'Permission': 'FULL_CONTROL',
})
else:
for perm in policy.perms:
permission = s3_perms.get(perm)
if permission:
grants.append({
'Grantee': grantee,
'Permission': permission,
})
self._s3_client.put_object_acl(
AccessControlPolicy={
'Grants': grants,
'Owner': acl['Owner'],
},
Bucket=path.drive,
Key=key,
)
except Exception:
raise FileSystemOperationError(traceback.format_exc())
def file_set_properties(self,
path: Path,
content_type: Optional[str] = None,
tags: Optional[Dict[str, str]] = None,
metadata: Optional[Dict[str, str]] = None) -> None:
""" Set properties for the object.
"""
kwargs: Dict[str, Any] = {}
if content_type:
kwargs['ContentType'] = content_type
kwargs['MetadataDirective'] = 'REPLACE'
if tags:
kwargs['Tagging'] = urlencode(tags)
kwargs['TaggingDirective'] = 'REPLACE'
if metadata:
kwargs['Metadata'] = metadata
kwargs['MetadataDirective'] = 'REPLACE'
key: str = self._separator.join(path.tail)
try:
acl = self._s3_client.get_object_acl(Bucket=path.drive, Key=key)
self._s3_client.copy_object(Key=key,
Bucket=path.drive,
CopySource={
"Bucket": path.drive,
"Key": key
},
**kwargs)
self._s3_client.put_object_acl(
AccessControlPolicy={
'Grants': acl['Grants'],
'Owner': acl['Owner'],
},
Bucket=path.drive,
Key=key,
)
except Exception:
raise FileSystemOperationError(traceback.format_exc())
def remove(self) -> None:
""" Tries to remove self from the file system.
On failure it raises a :class:`norfs.fs.base.FileSystemOperationError`
"""
raise FileSystemOperationError(f"Cannot remove {str(self)}")
def file_remove(self, path: Path) -> None:
try:
os.remove(self.path_to_string(path))
except Exception:
raise FileSystemOperationError(traceback.format_exc())
def file_read(self, path: Path) -> bytes:
try:
with open(self.path_to_string(path), "rb") as f:
return f.read()
except Exception:
raise FileSystemOperationError(traceback.format_exc())
def dir_remove(self, path: Path) -> None:
try:
shutil.rmtree(self.path_to_string(path))
except Exception:
raise FileSystemOperationError(traceback.format_exc())
def dir_remove(self, path: Path) -> None:
try:
current_dir: MemoryDirectory = self._get_dir(path.parent)
current_dir.remove_dir(path.basename)
except NotADirectoryError:
raise FileSystemOperationError(traceback.format_exc())
def file_remove(self, path: Path) -> None:
try:
parent_dir: MemoryDirectory = self._get_dir(path.parent)
parent_dir.remove_file(path.basename)
except (NotADirectoryError, NotAFileError):
raise FileSystemOperationError(traceback.format_exc())