def testPathComponents(self): components = PathComponents("/some/external/path/to/file.h5/with/internal/path/to/data") assert components.externalPath == "/some/external/path/to/file.h5" assert components.extension == ".h5" assert components.internalPath == "/with/internal/path/to/data" components = PathComponents("/some/external/path/to/file.h5_crazy_ext.h5/with/internal/path/to/data") assert components.externalPath == "/some/external/path/to/file.h5_crazy_ext.h5" assert components.extension == ".h5" assert components.internalPath == "/with/internal/path/to/data" # Everything should work for URLs, too. components = PathComponents("http://somehost:8000/path/to/data/with.ext") assert components.externalPath == "http://somehost:8000/path/to/data/with.ext" assert components.extension == ".ext" assert components.internalPath is None assert components.externalDirectory == "http://somehost:8000/path/to/data" assert components.filenameBase == "with" # Asterisk should be treated like an ordinary character for component purposes. assert PathComponents("/tmp/hello*.png").totalPath() == "/tmp/hello*.png" # Try modifying the properties and verify that the total path is updated. components = PathComponents("/some/external/path/to/file.h5/with/internal/path/to/data") components.extension = ".hdf5" assert components.externalPath == "/some/external/path/to/file.hdf5" assert components.totalPath() == "/some/external/path/to/file.hdf5/with/internal/path/to/data" components.filenameBase = "newbase" assert components.totalPath() == "/some/external/path/to/newbase.hdf5/with/internal/path/to/data" components.internalDirectory = "new/internal/dir" assert components.totalPath() == "/some/external/path/to/newbase.hdf5/new/internal/dir/data" components.internalDatasetName = "newdata" assert components.totalPath() == "/some/external/path/to/newbase.hdf5/new/internal/dir/newdata" components.externalDirectory = "/new/extern/dir/" assert components.totalPath() == "/new/extern/dir/newbase.hdf5/new/internal/dir/newdata" components.externalDirectory = "/new/extern/dir" assert components.totalPath() == "/new/extern/dir/newbase.hdf5/new/internal/dir/newdata" components.externalPath = "/new/externalpath/somefile.h5" assert components.totalPath() == "/new/externalpath/somefile.h5/new/internal/dir/newdata" components.filename = "newfilename.h5" assert components.totalPath() == "/new/externalpath/newfilename.h5/new/internal/dir/newdata" components.internalPath = "/new/internal/path/dataset" assert components.totalPath() == "/new/externalpath/newfilename.h5/new/internal/path/dataset"
def testPathComponents(self): components = PathComponents('/some/external/path/to/file.h5/with/internal/path/to/data') assert components.externalPath == '/some/external/path/to/file.h5' assert components.extension == '.h5' assert components.internalPath == '/with/internal/path/to/data' components = PathComponents('/some/external/path/to/file.h5_crazy_ext.h5/with/internal/path/to/data') assert components.externalPath == '/some/external/path/to/file.h5_crazy_ext.h5' assert components.extension == '.h5' assert components.internalPath == '/with/internal/path/to/data' # Everything should work for URLs, too. components = PathComponents('http://somehost:8000/path/to/data/with.ext') assert components.externalPath == 'http://somehost:8000/path/to/data/with.ext' assert components.extension == '.ext' assert components.internalPath is None assert components.externalDirectory == 'http://somehost:8000/path/to/data' assert components.filenameBase == 'with' # Try modifying the properties and verify that the total path is updated. components = PathComponents('/some/external/path/to/file.h5/with/internal/path/to/data') components.extension = '.hdf5' assert components.externalPath == '/some/external/path/to/file.hdf5' assert components.totalPath() == '/some/external/path/to/file.hdf5/with/internal/path/to/data' components.filenameBase = 'newbase' assert components.totalPath() == '/some/external/path/to/newbase.hdf5/with/internal/path/to/data' components.internalDirectory = 'new/internal/dir' assert components.totalPath() == '/some/external/path/to/newbase.hdf5/new/internal/dir/data' components.internalDatasetName = 'newdata' assert components.totalPath() == '/some/external/path/to/newbase.hdf5/new/internal/dir/newdata' components.externalDirectory = '/new/extern/dir/' assert components.totalPath() == '/new/extern/dir/newbase.hdf5/new/internal/dir/newdata' components.externalDirectory = '/new/extern/dir' assert components.totalPath() == '/new/extern/dir/newbase.hdf5/new/internal/dir/newdata' components.externalPath = '/new/externalpath/somefile.h5' assert components.totalPath() == '/new/externalpath/somefile.h5/new/internal/dir/newdata' components.filename = 'newfilename.h5' assert components.totalPath() == '/new/externalpath/newfilename.h5/new/internal/dir/newdata' components.internalPath = '/new/internal/path/dataset' assert components.totalPath() == '/new/externalpath/newfilename.h5/new/internal/path/dataset'
def pathIsHdf5(cls, path: Path) -> bool: return PathComponents( Path(path).as_posix()).extension in [".ilp", ".h5", ".hdf5"]
def pathIsN5(cls, path: Path) -> bool: return PathComponents(Path(path).as_posix()).extension in [".n5"]
def create_nickname(self, fileName: str): comps = PathComponents(fileName) expected_nickname = Path(comps.externalPath).stem if comps.internalPath: expected_nickname += comps.internalPath.replace("/", "-") return expected_nickname
def _attemptOpenAsHdf5(self, filePath): # Check for an hdf5 extension pathComponents = PathComponents(filePath) ext = pathComponents.extension if ext not in (".%s" % x for x in OpInputDataReader.h5Exts): return ([], None) externalPath = pathComponents.externalPath internalPath = pathComponents.internalPath if not os.path.exists(externalPath): raise OpInputDataReader.DatasetReadError( "Input file does not exist: " + externalPath) # Open the h5 file in read-only mode try: h5File = h5py.File(externalPath, 'r') except OpInputDataReader.DatasetReadError: raise except Exception as e: msg = "Unable to open HDF5 File: {}\n{}".format( externalPath, str(e)) raise OpInputDataReader.DatasetReadError(msg) else: if not internalPath: possible_internal_paths = self._get_hdf5_dataset_names(h5File) if len(possible_internal_paths) == 1: internalPath = possible_internal_paths[0] elif len(possible_internal_paths) == 0: h5File.close() msg = "HDF5 file contains no datasets: {}".format( externalPath) raise OpInputDataReader.DatasetReadError(msg) else: h5File.close() msg = "When using hdf5, you must append the hdf5 internal path to the "\ "data set to your filename, e.g. myfile.h5/volume/data "\ "No internal path provided for dataset in file: {}".format( externalPath) raise OpInputDataReader.DatasetReadError(msg) try: compression_setting = h5File[internalPath].compression except Exception as e: h5File.close() msg = "Error reading HDF5 File: {}\n{}".format(externalPath, e) raise OpInputDataReader.DatasetReadError(msg) # If the h5 dataset is compressed, we'll have better performance # with a multi-process hdf5 access object. # (Otherwise, single-process is faster.) allow_multiprocess_hdf5 = "LAZYFLOW_MULTIPROCESS_HDF5" in os.environ and os.environ[ "LAZYFLOW_MULTIPROCESS_HDF5"] != "" if compression_setting is not None and allow_multiprocess_hdf5: h5File.close() h5File = MultiProcessHdf5File(externalPath, 'r') self._file = h5File h5Reader = OpStreamingHdf5Reader(parent=self) h5Reader.Hdf5File.setValue(h5File) try: h5Reader.InternalPath.setValue(internalPath) except OpStreamingHdf5Reader.DatasetReadError as e: msg = "Error reading HDF5 File: {}\n{}".format(externalPath, e.msg) raise OpInputDataReader.DatasetReadError(msg) return ([h5Reader], h5Reader.OutputImage)
def getPossibleInternalPaths(self): possible_internal_paths = set() for expanded_path in self.expanded_paths: external_path = PathComponents(expanded_path).externalPath possible_internal_paths |= set(self.getPossibleInternalPathsFor(external_path)) return possible_internal_paths
def file_extensions(self) -> List[str]: return [PathComponents(ep).extension for ep in self.expanded_paths]
def internal_paths(self) -> List[str]: return [PathComponents(ep).internalPath for ep in self.expanded_paths]
def get_relative_paths(self) -> List[str]: external_paths = [Path(PathComponents(path).externalPath) for path in self.expanded_paths] try: return sorted([str(ep.absolute().relative_to(self.base_dir)) for ep in external_paths]) except ValueError: raise CantSaveAsRelativePathsException(self.filePath, self.base_dir)
def default_output_dir(self) -> Path: first_external_path = PathComponents(self.filePath.split(os.path.pathsep)[0]).externalPath return Path(first_external_path).parent