def test_multiple_backend_paths(local_testdir, s3, hdfs):
anon, s3so = s3
path = f"s3:{local_testdir}"
s3_path = UPath(path, anon=anon, **s3so)
assert s3_path.joinpath("text.txt")._url.scheme == "s3"
host, user, port = hdfs
path = f"hdfs:{local_testdir}"
UPath(path, host=host, user=user, port=port)
assert s3_path.joinpath("text1.txt")._url.scheme == "s3"
def _add_foreign_mag(
self,
foreign_mag_view_or_path: Union[PathLike, str, MagView],
symlink: bool,
make_relative: bool,
extend_layer_bounding_box: bool = True,
) -> MagView:
"""
The foreign mag is (shallow) copied and the existing mag is added to the datasource-properties.json.
If extend_layer_bounding_box is true, the self.bounding_box will be extended
by the bounding box of the layer the foreign mag belongs to.
Symlinked mags can only be added to layers on local file systems.
"""
self.dataset._ensure_writable()
if isinstance(foreign_mag_view_or_path, MagView):
foreign_mag_view = foreign_mag_view_or_path
else:
# local import to prevent circular dependency
from .dataset import Dataset
foreign_mag_view_path = UPath(foreign_mag_view_or_path)
foreign_mag_view = (Dataset.open(
foreign_mag_view_path.parent.parent).get_layer(
foreign_mag_view_path.parent.name).get_mag(
foreign_mag_view_path.name))
self._assert_mag_does_not_exist_yet(foreign_mag_view.mag)
if symlink:
assert is_fs_path(
self.path
), f"Cannot create symlinks in remote layer {self.path}"
assert is_fs_path(
foreign_mag_view.path
), f"Cannot create symlink to remote mag {foreign_mag_view.path}"
foreign_normalized_mag_path = (
Path(relpath(foreign_mag_view.path, self.path))
if make_relative else foreign_mag_view.path.resolve())
(self.path /
str(foreign_mag_view.mag)).symlink_to(foreign_normalized_mag_path)
else:
copytree(
foreign_mag_view.path,
self.path / str(foreign_mag_view.mag),
)
self.add_mag_for_existing_files(foreign_mag_view.mag)
if extend_layer_bounding_box:
self.bounding_box = self.bounding_box.extended_by(
foreign_mag_view.layer.bounding_box)
self.dataset._export_as_json()
return self._mags[foreign_mag_view.mag]
class TestUPathS3(TestUpath):
@pytest.fixture(autouse=True)
def path(self, local_testdir, s3):
anon, s3so = s3
path = f"s3:/{local_testdir}"
print(path)
self.path = UPath(path, anon=anon, **s3so)
def test_is_S3Path(self):
assert isinstance(self.path, S3Path)
def test_chmod(self):
# todo
pass
def test_mkdir(self):
new_dir = self.path.joinpath("new_dir")
# new_dir.mkdir()
# mkdir doesnt really do anything. A directory only exists in s3
# if some file or something is written to it
new_dir.joinpath("test.txt").touch()
assert new_dir.exists()
def test_rmdir(self, local_testdir):
dirname = "rmdir_test"
mock_dir = self.path.joinpath(dirname)
mock_dir.joinpath("test.txt").touch()
mock_dir.rmdir()
assert not mock_dir.exists()
with pytest.raises(NotDirectoryError):
self.path.joinpath("file1.txt").rmdir()
def test_touch_unlink(self):
path = self.path.joinpath("test_touch.txt")
path.touch()
assert path.exists()
path.unlink()
assert not path.exists()
# should raise FileNotFoundError since file is missing
with pytest.raises(FileNotFoundError):
path.unlink()
# file doesn't exists, but missing_ok is True
path.unlink(missing_ok=True)
def test_glob(self, pathlib_base):
mock_glob = list(self.path.glob("**.txt"))
path_glob = list(pathlib_base.glob("**/*.txt"))
assert len(mock_glob) == len(path_glob)
assert all(
map(lambda m: m.path in [str(p)[4:] for p in path_glob],
mock_glob))
class TestMemoryPath(TestUpath):
@pytest.fixture(autouse=True)
def path(self, local_testdir):
path = f"memory:/{local_testdir}"
self.path = UPath(path)
self.prepare_file_system()
def test_is_MemoryPath(self):
assert isinstance(self.path, MemoryPath)
def test_glob(self, pathlib_base):
mock_glob = list(self.path.glob("**.txt"))
path_glob = list(pathlib_base.glob("**/*.txt"))
assert len(mock_glob) == len(path_glob)
if not sys.platform.startswith("win"): # need to fix windows tests here
assert all(
map(
lambda m: m.path in [str(p)[4:] for p in path_glob],
mock_glob,
)
)
def remote_testoutput_path() -> UPath:
"""Minio is an S3 clone and is used as local test server"""
container_name = "minio"
cmd = ("docker run"
f" -p {MINIO_PORT}:9000"
f" -e MINIO_ROOT_USER={MINIO_ROOT_USER}"
f" -e MINIO_ROOT_PASSWORD={MINIO_ROOT_PASSWORD}"
f" --name {container_name}"
" --rm"
" -d"
" minio/minio server /data")
subprocess.check_output(shlex.split(cmd))
remote_path = UPath(
"s3://testoutput",
key=MINIO_ROOT_USER,
secret=MINIO_ROOT_PASSWORD,
client_kwargs={"endpoint_url": f"http://localhost:{MINIO_PORT}"},
)
remote_path.fs.mkdirs("testoutput", exist_ok=True)
try:
yield remote_path
finally:
subprocess.check_output(["docker", "stop", container_name])
def path(self, local_testdir, hdfs):
host, user, port = hdfs
path = f"hdfs:{local_testdir}"
self.path = UPath(path, host=host, user=user, port=port)
def test_httppath():
path = UPath("http://example.com")
assert isinstance(path, HTTPPath)
assert path.exists()
def path(self, local_testdir):
path = f"memory:/{local_testdir}"
self.path = UPath(path)
self.prepare_file_system()
def path(self, local_testdir):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
self.path = UPath(f"mock:{local_testdir}")
class TestUpath:
@pytest.fixture(autouse=True)
def path(self, local_testdir):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
self.path = UPath(f"mock:{local_testdir}")
def test_cwd(self):
with pytest.raises(NotImplementedError):
self.path.cwd()
def test_home(self):
with pytest.raises(NotImplementedError):
self.path.home()
def test_stat(self):
stat = self.path.stat()
assert stat
def test_chmod(self):
with pytest.raises(NotImplementedError):
self.path.joinpath("file1.txt").chmod(777)
@pytest.mark.parametrize("url, expected", [("file1.txt", True),
("fakefile.txt", False)])
def test_exists(self, url, expected):
path = self.path.joinpath(url)
assert path.exists() == expected
def test_expanduser(self):
with pytest.raises(NotImplementedError):
self.path.expanduser()
def test_glob(self, pathlib_base):
mock_glob = list(self.path.glob("**.txt"))
path_glob = list(pathlib_base.glob("**/*.txt"))
assert len(mock_glob) == len(path_glob)
assert all(
map(
lambda m: m.path in
[str(p).replace("\\", "/") for p in path_glob],
mock_glob,
))
def test_group(self):
with pytest.raises(NotImplementedError):
self.path.group()
def test_is_dir(self):
assert self.path.is_dir()
path = self.path.joinpath("file1.txt")
assert not path.is_dir()
def test_is_file(self):
path = self.path.joinpath("file1.txt")
assert path.is_file()
assert not self.path.is_file()
def test_is_mount(self):
with pytest.raises(NotImplementedError):
self.path.is_mount()
def test_is_symlink(self):
with pytest.raises(NotImplementedError):
self.path.is_symlink()
def test_is_socket(self):
with pytest.raises(NotImplementedError):
self.path.is_socket()
def test_is_fifo(self):
with pytest.raises(NotImplementedError):
self.path.is_fifo()
def test_is_block_device(self):
with pytest.raises(NotImplementedError):
self.path.is_block_device()
def test_is_char_device(self):
with pytest.raises(NotImplementedError):
self.path.is_char_device()
def test_iterdir(self, local_testdir):
pl_path = Path(local_testdir)
up_iter = list(self.path.iterdir())
pl_iter = list(pl_path.iterdir())
assert len(up_iter) == len(pl_iter)
pnames = [p.name for p in pl_iter]
assert all(map(lambda x: x.name in pnames, up_iter))
def test_lchmod(self):
with pytest.raises(NotImplementedError):
self.path.lchmod(mode=77)
def test_lstat(self):
with pytest.raises(NotImplementedError):
self.path.lstat()
def test_mkdir(self):
new_dir = self.path.joinpath("new_dir")
new_dir.mkdir()
assert new_dir.exists()
def test_open(self):
pass
def test_owner(self):
with pytest.raises(NotImplementedError):
self.path.owner()
def test_read_bytes(self, pathlib_base):
mock = self.path.joinpath("file2.txt")
pl = pathlib_base.joinpath("file2.txt")
assert mock.read_bytes() == pl.read_bytes()
def test_read_text(self, local_testdir):
upath = self.path.joinpath("file1.txt")
assert (upath.read_text() == Path(local_testdir).joinpath(
"file1.txt").read_text())
def test_readlink(self):
with pytest.raises(NotImplementedError):
self.path.readlink()
@pytest.mark.xfail
def test_rename(self):
# need to impliment
raise False
def test_replace(self):
pass
def test_resolve(self):
pass
def test_rglob(self):
pass
def test_samefile(self):
pass
def test_symlink_to(self):
pass
def test_touch_unlink(self):
path = self.path.joinpath("test_touch.txt")
path.touch()
assert path.exists()
path.unlink()
assert not path.exists()
# should raise FileNotFoundError since file is missing
with pytest.raises(FileNotFoundError):
path.unlink()
# file doesn't exists, but missing_ok is True
path.unlink(missing_ok=True)
def test_link_to(self):
pass
def test_write_bytes(self, pathlib_base):
fn = "test_write_bytes.txt"
s = b"hello_world"
path = self.path.joinpath(fn)
path.write_bytes(s)
assert path.read_bytes() == s
def test_write_text(self, pathlib_base):
fn = "test_write_text.txt"
s = "hello_world"
path = self.path.joinpath(fn)
path.write_text(s)
assert path.read_text() == s
def prepare_file_system(self):
self.make_top_folder()
self.make_test_files()
def make_top_folder(self):
self.path.mkdir(parents=True, exist_ok=True)
def make_test_files(self):
folder1 = self.path.joinpath("folder1")
folder1.mkdir(exist_ok=True)
folder1_files = ["file1.txt", "file2.txt"]
for f in folder1_files:
p = folder1.joinpath(f)
p.touch()
p.write_text(f)
file1 = self.path.joinpath("file1.txt")
file1.touch()
file1.write_text("hello world")
file2 = self.path.joinpath("file2.txt")
file2.touch()
file2.write_bytes(b"hello world")
def test_UPath_warning():
with warnings.catch_warnings(record=True) as w:
path = UPath("mock:/") # noqa: F841
assert len(w) == 1
assert issubclass(w[-1].category, UserWarning)
assert "mock" in str(w[-1].message)
def test_windows_path(local_testdir):
assert isinstance(UPath(local_testdir), pathlib.WindowsPath)
def test_posix_path(local_testdir):
assert isinstance(UPath(local_testdir), pathlib.PosixPath)
def path(self, local_testdir, s3):
anon, s3so = s3
path = f"s3:/{local_testdir}"
print(path)
self.path = UPath(path, anon=anon, **s3so)
def compress(
self,
target_path: Optional[Union[str, Path]] = None,
args: Optional[Namespace] = None,
) -> None:
"""
Compresses the files on disk. This has consequences for writing data (see `write`).
The data gets compressed inplace, if target_path is None.
Otherwise it is written to target_path/layer_name/mag.
Compressing mags on remote file systems requires a `target_path`.
"""
from webknossos.dataset.dataset import Dataset
if target_path is None:
if self._is_compressed():
logging.info(f"Mag {self.name} is already compressed")
return
else:
assert is_fs_path(
self.path
), "Cannot compress a remote mag without `target_path`."
else:
target_path = UPath(target_path)
uncompressed_full_path = self.path
compressed_dataset_path = (
self.layer.dataset.path / f".compress-{uuid4()}"
if target_path is None
else target_path
)
compressed_dataset = Dataset(
compressed_dataset_path,
voxel_size=self.layer.dataset.voxel_size,
exist_ok=True,
)
compressed_mag = compressed_dataset.get_or_add_layer(
layer_name=self.layer.name,
category=self.layer.category,
dtype_per_channel=self.layer.dtype_per_channel,
num_channels=self.layer.num_channels,
data_format=self.layer.data_format,
largest_segment_id=self.layer._get_largest_segment_id_maybe(),
).get_or_add_mag(
mag=self.mag,
chunk_size=self.info.chunk_size,
chunks_per_shard=self.info.chunks_per_shard,
compress=True,
)
compressed_mag.layer.bounding_box = self.layer.bounding_box
logging.info(
"Compressing mag {0} in '{1}'".format(
self.name, str(uncompressed_full_path)
)
)
with get_executor_for_args(args) as executor:
job_args = []
for bbox in self.get_bounding_boxes_on_disk():
bbox = bbox.intersected_with(self.layer.bounding_box, dont_assert=True)
if not bbox.is_empty():
bbox = bbox.align_with_mag(self.mag, ceil=True)
source_view = self.get_view(
absolute_offset=bbox.topleft, size=bbox.size
)
target_view = compressed_mag.get_view(
absolute_offset=bbox.topleft, size=bbox.size
)
job_args.append((source_view, target_view))
wait_and_ensure_success(
executor.map_to_futures(_compress_cube_job, job_args), "Compressing"
)
if target_path is None:
rmtree(self.path)
compressed_mag.path.rename(self.path)
rmtree(compressed_dataset.path)
# update the handle to the new dataset
MagView.__init__(
self,
self.layer,
self._mag,
self.info.chunk_size,
self.info.chunks_per_shard,
True,
)