def test_multiple_backend_paths(local_testdir, s3, hdfs): anon, s3so = s3 path = f"s3:{local_testdir}" s3_path = UPath(path, anon=anon, **s3so) assert s3_path.joinpath("text.txt")._url.scheme == "s3" host, user, port = hdfs path = f"hdfs:{local_testdir}" UPath(path, host=host, user=user, port=port) assert s3_path.joinpath("text1.txt")._url.scheme == "s3"
def _add_foreign_mag( self, foreign_mag_view_or_path: Union[PathLike, str, MagView], symlink: bool, make_relative: bool, extend_layer_bounding_box: bool = True, ) -> MagView: """ The foreign mag is (shallow) copied and the existing mag is added to the datasource-properties.json. If extend_layer_bounding_box is true, the self.bounding_box will be extended by the bounding box of the layer the foreign mag belongs to. Symlinked mags can only be added to layers on local file systems. """ self.dataset._ensure_writable() if isinstance(foreign_mag_view_or_path, MagView): foreign_mag_view = foreign_mag_view_or_path else: # local import to prevent circular dependency from .dataset import Dataset foreign_mag_view_path = UPath(foreign_mag_view_or_path) foreign_mag_view = (Dataset.open( foreign_mag_view_path.parent.parent).get_layer( foreign_mag_view_path.parent.name).get_mag( foreign_mag_view_path.name)) self._assert_mag_does_not_exist_yet(foreign_mag_view.mag) if symlink: assert is_fs_path( self.path ), f"Cannot create symlinks in remote layer {self.path}" assert is_fs_path( foreign_mag_view.path ), f"Cannot create symlink to remote mag {foreign_mag_view.path}" foreign_normalized_mag_path = ( Path(relpath(foreign_mag_view.path, self.path)) if make_relative else foreign_mag_view.path.resolve()) (self.path / str(foreign_mag_view.mag)).symlink_to(foreign_normalized_mag_path) else: copytree( foreign_mag_view.path, self.path / str(foreign_mag_view.mag), ) self.add_mag_for_existing_files(foreign_mag_view.mag) if extend_layer_bounding_box: self.bounding_box = self.bounding_box.extended_by( foreign_mag_view.layer.bounding_box) self.dataset._export_as_json() return self._mags[foreign_mag_view.mag]
class TestUPathS3(TestUpath): @pytest.fixture(autouse=True) def path(self, local_testdir, s3): anon, s3so = s3 path = f"s3:/{local_testdir}" print(path) self.path = UPath(path, anon=anon, **s3so) def test_is_S3Path(self): assert isinstance(self.path, S3Path) def test_chmod(self): # todo pass def test_mkdir(self): new_dir = self.path.joinpath("new_dir") # new_dir.mkdir() # mkdir doesnt really do anything. A directory only exists in s3 # if some file or something is written to it new_dir.joinpath("test.txt").touch() assert new_dir.exists() def test_rmdir(self, local_testdir): dirname = "rmdir_test" mock_dir = self.path.joinpath(dirname) mock_dir.joinpath("test.txt").touch() mock_dir.rmdir() assert not mock_dir.exists() with pytest.raises(NotDirectoryError): self.path.joinpath("file1.txt").rmdir() def test_touch_unlink(self): path = self.path.joinpath("test_touch.txt") path.touch() assert path.exists() path.unlink() assert not path.exists() # should raise FileNotFoundError since file is missing with pytest.raises(FileNotFoundError): path.unlink() # file doesn't exists, but missing_ok is True path.unlink(missing_ok=True) def test_glob(self, pathlib_base): mock_glob = list(self.path.glob("**.txt")) path_glob = list(pathlib_base.glob("**/*.txt")) assert len(mock_glob) == len(path_glob) assert all( map(lambda m: m.path in [str(p)[4:] for p in path_glob], mock_glob))
class TestMemoryPath(TestUpath): @pytest.fixture(autouse=True) def path(self, local_testdir): path = f"memory:/{local_testdir}" self.path = UPath(path) self.prepare_file_system() def test_is_MemoryPath(self): assert isinstance(self.path, MemoryPath) def test_glob(self, pathlib_base): mock_glob = list(self.path.glob("**.txt")) path_glob = list(pathlib_base.glob("**/*.txt")) assert len(mock_glob) == len(path_glob) if not sys.platform.startswith("win"): # need to fix windows tests here assert all( map( lambda m: m.path in [str(p)[4:] for p in path_glob], mock_glob, ) )
def remote_testoutput_path() -> UPath: """Minio is an S3 clone and is used as local test server""" container_name = "minio" cmd = ("docker run" f" -p {MINIO_PORT}:9000" f" -e MINIO_ROOT_USER={MINIO_ROOT_USER}" f" -e MINIO_ROOT_PASSWORD={MINIO_ROOT_PASSWORD}" f" --name {container_name}" " --rm" " -d" " minio/minio server /data") subprocess.check_output(shlex.split(cmd)) remote_path = UPath( "s3://testoutput", key=MINIO_ROOT_USER, secret=MINIO_ROOT_PASSWORD, client_kwargs={"endpoint_url": f"http://localhost:{MINIO_PORT}"}, ) remote_path.fs.mkdirs("testoutput", exist_ok=True) try: yield remote_path finally: subprocess.check_output(["docker", "stop", container_name])
def path(self, local_testdir, hdfs): host, user, port = hdfs path = f"hdfs:{local_testdir}" self.path = UPath(path, host=host, user=user, port=port)
def test_httppath(): path = UPath("http://example.com") assert isinstance(path, HTTPPath) assert path.exists()
def path(self, local_testdir): path = f"memory:/{local_testdir}" self.path = UPath(path) self.prepare_file_system()
def path(self, local_testdir): with warnings.catch_warnings(): warnings.simplefilter("ignore") self.path = UPath(f"mock:{local_testdir}")
class TestUpath: @pytest.fixture(autouse=True) def path(self, local_testdir): with warnings.catch_warnings(): warnings.simplefilter("ignore") self.path = UPath(f"mock:{local_testdir}") def test_cwd(self): with pytest.raises(NotImplementedError): self.path.cwd() def test_home(self): with pytest.raises(NotImplementedError): self.path.home() def test_stat(self): stat = self.path.stat() assert stat def test_chmod(self): with pytest.raises(NotImplementedError): self.path.joinpath("file1.txt").chmod(777) @pytest.mark.parametrize("url, expected", [("file1.txt", True), ("fakefile.txt", False)]) def test_exists(self, url, expected): path = self.path.joinpath(url) assert path.exists() == expected def test_expanduser(self): with pytest.raises(NotImplementedError): self.path.expanduser() def test_glob(self, pathlib_base): mock_glob = list(self.path.glob("**.txt")) path_glob = list(pathlib_base.glob("**/*.txt")) assert len(mock_glob) == len(path_glob) assert all( map( lambda m: m.path in [str(p).replace("\\", "/") for p in path_glob], mock_glob, )) def test_group(self): with pytest.raises(NotImplementedError): self.path.group() def test_is_dir(self): assert self.path.is_dir() path = self.path.joinpath("file1.txt") assert not path.is_dir() def test_is_file(self): path = self.path.joinpath("file1.txt") assert path.is_file() assert not self.path.is_file() def test_is_mount(self): with pytest.raises(NotImplementedError): self.path.is_mount() def test_is_symlink(self): with pytest.raises(NotImplementedError): self.path.is_symlink() def test_is_socket(self): with pytest.raises(NotImplementedError): self.path.is_socket() def test_is_fifo(self): with pytest.raises(NotImplementedError): self.path.is_fifo() def test_is_block_device(self): with pytest.raises(NotImplementedError): self.path.is_block_device() def test_is_char_device(self): with pytest.raises(NotImplementedError): self.path.is_char_device() def test_iterdir(self, local_testdir): pl_path = Path(local_testdir) up_iter = list(self.path.iterdir()) pl_iter = list(pl_path.iterdir()) assert len(up_iter) == len(pl_iter) pnames = [p.name for p in pl_iter] assert all(map(lambda x: x.name in pnames, up_iter)) def test_lchmod(self): with pytest.raises(NotImplementedError): self.path.lchmod(mode=77) def test_lstat(self): with pytest.raises(NotImplementedError): self.path.lstat() def test_mkdir(self): new_dir = self.path.joinpath("new_dir") new_dir.mkdir() assert new_dir.exists() def test_open(self): pass def test_owner(self): with pytest.raises(NotImplementedError): self.path.owner() def test_read_bytes(self, pathlib_base): mock = self.path.joinpath("file2.txt") pl = pathlib_base.joinpath("file2.txt") assert mock.read_bytes() == pl.read_bytes() def test_read_text(self, local_testdir): upath = self.path.joinpath("file1.txt") assert (upath.read_text() == Path(local_testdir).joinpath( "file1.txt").read_text()) def test_readlink(self): with pytest.raises(NotImplementedError): self.path.readlink() @pytest.mark.xfail def test_rename(self): # need to impliment raise False def test_replace(self): pass def test_resolve(self): pass def test_rglob(self): pass def test_samefile(self): pass def test_symlink_to(self): pass def test_touch_unlink(self): path = self.path.joinpath("test_touch.txt") path.touch() assert path.exists() path.unlink() assert not path.exists() # should raise FileNotFoundError since file is missing with pytest.raises(FileNotFoundError): path.unlink() # file doesn't exists, but missing_ok is True path.unlink(missing_ok=True) def test_link_to(self): pass def test_write_bytes(self, pathlib_base): fn = "test_write_bytes.txt" s = b"hello_world" path = self.path.joinpath(fn) path.write_bytes(s) assert path.read_bytes() == s def test_write_text(self, pathlib_base): fn = "test_write_text.txt" s = "hello_world" path = self.path.joinpath(fn) path.write_text(s) assert path.read_text() == s def prepare_file_system(self): self.make_top_folder() self.make_test_files() def make_top_folder(self): self.path.mkdir(parents=True, exist_ok=True) def make_test_files(self): folder1 = self.path.joinpath("folder1") folder1.mkdir(exist_ok=True) folder1_files = ["file1.txt", "file2.txt"] for f in folder1_files: p = folder1.joinpath(f) p.touch() p.write_text(f) file1 = self.path.joinpath("file1.txt") file1.touch() file1.write_text("hello world") file2 = self.path.joinpath("file2.txt") file2.touch() file2.write_bytes(b"hello world")
def test_UPath_warning(): with warnings.catch_warnings(record=True) as w: path = UPath("mock:/") # noqa: F841 assert len(w) == 1 assert issubclass(w[-1].category, UserWarning) assert "mock" in str(w[-1].message)
def test_windows_path(local_testdir): assert isinstance(UPath(local_testdir), pathlib.WindowsPath)
def test_posix_path(local_testdir): assert isinstance(UPath(local_testdir), pathlib.PosixPath)
def path(self, local_testdir, s3): anon, s3so = s3 path = f"s3:/{local_testdir}" print(path) self.path = UPath(path, anon=anon, **s3so)
def compress( self, target_path: Optional[Union[str, Path]] = None, args: Optional[Namespace] = None, ) -> None: """ Compresses the files on disk. This has consequences for writing data (see `write`). The data gets compressed inplace, if target_path is None. Otherwise it is written to target_path/layer_name/mag. Compressing mags on remote file systems requires a `target_path`. """ from webknossos.dataset.dataset import Dataset if target_path is None: if self._is_compressed(): logging.info(f"Mag {self.name} is already compressed") return else: assert is_fs_path( self.path ), "Cannot compress a remote mag without `target_path`." else: target_path = UPath(target_path) uncompressed_full_path = self.path compressed_dataset_path = ( self.layer.dataset.path / f".compress-{uuid4()}" if target_path is None else target_path ) compressed_dataset = Dataset( compressed_dataset_path, voxel_size=self.layer.dataset.voxel_size, exist_ok=True, ) compressed_mag = compressed_dataset.get_or_add_layer( layer_name=self.layer.name, category=self.layer.category, dtype_per_channel=self.layer.dtype_per_channel, num_channels=self.layer.num_channels, data_format=self.layer.data_format, largest_segment_id=self.layer._get_largest_segment_id_maybe(), ).get_or_add_mag( mag=self.mag, chunk_size=self.info.chunk_size, chunks_per_shard=self.info.chunks_per_shard, compress=True, ) compressed_mag.layer.bounding_box = self.layer.bounding_box logging.info( "Compressing mag {0} in '{1}'".format( self.name, str(uncompressed_full_path) ) ) with get_executor_for_args(args) as executor: job_args = [] for bbox in self.get_bounding_boxes_on_disk(): bbox = bbox.intersected_with(self.layer.bounding_box, dont_assert=True) if not bbox.is_empty(): bbox = bbox.align_with_mag(self.mag, ceil=True) source_view = self.get_view( absolute_offset=bbox.topleft, size=bbox.size ) target_view = compressed_mag.get_view( absolute_offset=bbox.topleft, size=bbox.size ) job_args.append((source_view, target_view)) wait_and_ensure_success( executor.map_to_futures(_compress_cube_job, job_args), "Compressing" ) if target_path is None: rmtree(self.path) compressed_mag.path.rename(self.path) rmtree(compressed_dataset.path) # update the handle to the new dataset MagView.__init__( self, self.layer, self._mag, self.info.chunk_size, self.info.chunks_per_shard, True, )