def test_file_ops(tmpdir):
tmpdir = str(tmpdir)
fs = LocalFileSystem()
with pytest.raises(FileNotFoundError):
fs.info(tmpdir + '/nofile')
fs.touch(tmpdir + '/afile')
i1 = fs.ukey(tmpdir + '/afile')
assert tmpdir + '/afile' in fs.ls(tmpdir)
with fs.open(tmpdir + '/afile', 'wb') as f:
f.write(b'data')
i2 = fs.ukey(tmpdir + '/afile')
assert i1 != i2 # because file changed
fs.copy(tmpdir + '/afile', tmpdir + '/afile2')
assert tmpdir + '/afile2' in fs.ls(tmpdir)
fs.move(tmpdir + '/afile', tmpdir + '/afile3')
assert not fs.exists(tmpdir + '/afile')
fs.rm(tmpdir + '/afile3', recursive=True)
assert not fs.exists(tmpdir + '/afile3')
fs.rm(tmpdir, recursive=True)
assert not fs.exists(tmpdir)
def test_file_ops(tmpdir):
tmpdir = str(tmpdir)
fs = LocalFileSystem()
with pytest.raises(FileNotFoundError):
fs.info(tmpdir + "/nofile")
fs.touch(tmpdir + "/afile")
i1 = fs.ukey(tmpdir + "/afile")
assert tmpdir + "/afile" in fs.ls(tmpdir)
with fs.open(tmpdir + "/afile", "wb") as f:
f.write(b"data")
i2 = fs.ukey(tmpdir + "/afile")
assert i1 != i2 # because file changed
fs.copy(tmpdir + "/afile", tmpdir + "/afile2")
assert tmpdir + "/afile2" in fs.ls(tmpdir)
fs.move(tmpdir + "/afile", tmpdir + "/afile3")
assert not fs.exists(tmpdir + "/afile")
fs.rm(tmpdir + "/afile3", recursive=True)
assert not fs.exists(tmpdir + "/afile3")
fs.rm(tmpdir, recursive=True)
assert not fs.exists(tmpdir)
def test_put_file(storage):
fs = AzureBlobFileSystem(account_name=storage.account_name,
connection_string=CONN_STR)
lfs = LocalFileSystem()
fs.mkdir("putdir")
# Check that put on an empty file works
with open("sample.txt", "wb") as f:
f.write(b"")
fs.put("sample.txt", "putdir/sample.txt")
fs.get("putdir/sample.txt", "sample2.txt")
with open("sample.txt", "rb") as f:
f1 = f.read()
with open("sample2.txt", "rb") as f:
f2 = f.read()
assert f1 == f2
lfs.rm("sample.txt")
lfs.rm("sample2.txt")
# Check that put on a file with data works
with open("sample3.txt", "wb") as f:
f.write(b"01234567890")
fs.put("sample3.txt", "putdir/sample3.txt")
fs.get("putdir/sample3.txt", "sample4.txt")
with open("sample3.txt", "rb") as f:
f3 = f.read()
with open("sample4.txt", "rb") as f:
f4 = f.read()
assert f3 == f4
fs.rm("putdir", recursive=True)
def test_delete_cwd(tmpdir):
cwd = os.getcwd()
fs = LocalFileSystem()
try:
os.chdir(tmpdir)
with pytest.raises(ValueError):
fs.rm(".", recursive=True)
finally:
os.chdir(cwd)
def remove_local_file(filepath: Union[str, Path]) -> None:
"""
Deletes a file from the local file system.
Parameters
----------
filepath: str
The filepath of the local file to delete.
"""
fs = LocalFileSystem()
fs.rm(filepath)
log.info(f"Removed {filepath} from local file system.")
def convert_dicom(sourcedata_dir: PathLike, bids_filename: PathLike) -> None:
import subprocess
from pathlib import Path
from fsspec.implementations.local import LocalFileSystem
output_fmt = str(Path(bids_filename).name).replace(".nii.gz", "")
output_dir = str(Path(bids_filename).parent)
# Ensure output directory is empty.
fs = LocalFileSystem()
if fs.exists(output_dir):
fs.rm(output_dir, recursive=True)
fs.makedirs(output_dir)
# Run conversion with dcm2niix with anonymization and maximum compression.
subprocess.run(
f"dcm2niix -9 -b y -ba y -f {output_fmt} -o {output_dir} -z i {sourcedata_dir}",
shell=True,
)
def test_file_ops(tmpdir):
tmpdir = make_path_posix(str(tmpdir))
fs = LocalFileSystem(auto_mkdir=True)
with pytest.raises(FileNotFoundError):
fs.info(tmpdir + "/nofile")
fs.touch(tmpdir + "/afile")
i1 = fs.ukey(tmpdir + "/afile")
assert tmpdir + "/afile" in fs.ls(tmpdir)
with fs.open(tmpdir + "/afile", "wb") as f:
f.write(b"data")
i2 = fs.ukey(tmpdir + "/afile")
assert i1 != i2 # because file changed
fs.copy(tmpdir + "/afile", tmpdir + "/afile2")
assert tmpdir + "/afile2" in fs.ls(tmpdir)
fs.move(tmpdir + "/afile", tmpdir + "/afile3")
assert not fs.exists(tmpdir + "/afile")
fs.cp(tmpdir + "/afile3", tmpdir + "/deeply/nested/file")
assert fs.exists(tmpdir + "/deeply/nested/file")
fs.rm(tmpdir + "/afile3", recursive=True)
assert not fs.exists(tmpdir + "/afile3")
files = [tmpdir + "/afile4", tmpdir + "/afile5"]
[fs.touch(f) for f in files]
with pytest.raises(TypeError):
fs.rm_file(files)
fs.rm(files)
assert all(not fs.exists(f) for f in files)
fs.touch(tmpdir + "/afile6")
fs.rm_file(tmpdir + "/afile6")
assert not fs.exists(tmpdir + "/afile6")
# IsADirectoryError raised on Linux, PermissionError on Windows
with pytest.raises((IsADirectoryError, PermissionError)):
fs.rm_file(tmpdir)
fs.rm(tmpdir, recursive=True)
assert not fs.exists(tmpdir)
def __exit__(self, *_):
# cleanup!
fs = LocalFileSystem()
if fs.exists(self.tmp_path):
fs.rm(self.tmp_path, recursive=True)
class FileSystem(AbstractFileSystem):
"""Wrapper for easier initialization of various file-system classes"""
def __init__(self, name="local", assumed_role=None, endpoint_url=None):
super().__init__()
self.name = name
self.assume_client = None
self.assume_role = assumed_role
self.endpoint_url = endpoint_url
if self.name == "local":
self.filesystem = LocalFileSystem()
elif self.name == "s3":
session = botocore.session.get_session()
if self.assume_role:
self.assume_client = session.create_client("sts")
session_credentials = (
botocore.credentials.RefreshableCredentials.create_from_metadata(
metadata=self._sts_refresh(),
refresh_using=self._sts_refresh,
method="sts-assume-role",
)
)
session._credentials = session_credentials
client_kwargs = {"endpoint_url": endpoint_url} if endpoint_url else None
self.filesystem = s3fs.S3FileSystem(
session=session, client_kwargs=client_kwargs
)
else:
raise ValueError(f"Unsupported FileReader type: {type}")
for method_name, method in inspect.getmembers(
self.filesystem, predicate=inspect.ismethod
):
if method_name not in (
"__init__",
"_rm",
"cp_file",
"created",
"ls",
"modified",
"sign",
):
setattr(self, method_name, method)
def _sts_refresh(self):
"""Refresh tokens by calling assume_role again"""
response = self.assume_client.assume_role(
RoleArn=self.assume_role,
RoleSessionName=f"data-toolz-filesystem-s3-{uuid4()}",
DurationSeconds=3600,
).get("Credentials")
return {
"access_key": response.get("AccessKeyId"),
"secret_key": response.get("SecretAccessKey"),
"token": response.get("SessionToken"),
"expiry_time": response.get("Expiration").isoformat(),
}
def _rm(self, path):
return self.filesystem.rm(path=path)
def cp_file(self, path1, path2, **kwargs):
return self.filesystem.copy(path1=path1, path2=path2, **kwargs)
def created(self, path):
return self.filesystem.created(path=path)
def ls(self, path, detail=True, **kwargs):
return self.filesystem.ls(path=path, detail=detail, **kwargs)
def modified(self, path):
return self.filesystem.modified(path=path)
def sign(self, path, expiration=100, **kwargs):
return self.filesystem.sign(path=path, expiration=expiration, **kwargs)
