def testPathComponents(self):
components = PathComponents("/some/external/path/to/file.h5/with/internal/path/to/data")
assert components.externalPath == "/some/external/path/to/file.h5"
assert components.extension == ".h5"
assert components.internalPath == "/with/internal/path/to/data"
components = PathComponents("/some/external/path/to/file.h5_crazy_ext.h5/with/internal/path/to/data")
assert components.externalPath == "/some/external/path/to/file.h5_crazy_ext.h5"
assert components.extension == ".h5"
assert components.internalPath == "/with/internal/path/to/data"
# Everything should work for URLs, too.
components = PathComponents("http://somehost:8000/path/to/data/with.ext")
assert components.externalPath == "http://somehost:8000/path/to/data/with.ext"
assert components.extension == ".ext"
assert components.internalPath is None
assert components.externalDirectory == "http://somehost:8000/path/to/data"
assert components.filenameBase == "with"
# Asterisk should be treated like an ordinary character for component purposes.
assert PathComponents("/tmp/hello*.png").totalPath() == "/tmp/hello*.png"
# Try modifying the properties and verify that the total path is updated.
components = PathComponents("/some/external/path/to/file.h5/with/internal/path/to/data")
components.extension = ".hdf5"
assert components.externalPath == "/some/external/path/to/file.hdf5"
assert components.totalPath() == "/some/external/path/to/file.hdf5/with/internal/path/to/data"
components.filenameBase = "newbase"
assert components.totalPath() == "/some/external/path/to/newbase.hdf5/with/internal/path/to/data"
components.internalDirectory = "new/internal/dir"
assert components.totalPath() == "/some/external/path/to/newbase.hdf5/new/internal/dir/data"
components.internalDatasetName = "newdata"
assert components.totalPath() == "/some/external/path/to/newbase.hdf5/new/internal/dir/newdata"
components.externalDirectory = "/new/extern/dir/"
assert components.totalPath() == "/new/extern/dir/newbase.hdf5/new/internal/dir/newdata"
components.externalDirectory = "/new/extern/dir"
assert components.totalPath() == "/new/extern/dir/newbase.hdf5/new/internal/dir/newdata"
components.externalPath = "/new/externalpath/somefile.h5"
assert components.totalPath() == "/new/externalpath/somefile.h5/new/internal/dir/newdata"
components.filename = "newfilename.h5"
assert components.totalPath() == "/new/externalpath/newfilename.h5/new/internal/dir/newdata"
components.internalPath = "/new/internal/path/dataset"
assert components.totalPath() == "/new/externalpath/newfilename.h5/new/internal/path/dataset"
def testPathComponents(self):
components = PathComponents('/some/external/path/to/file.h5/with/internal/path/to/data')
assert components.externalPath == '/some/external/path/to/file.h5'
assert components.extension == '.h5'
assert components.internalPath == '/with/internal/path/to/data'
components = PathComponents('/some/external/path/to/file.h5_crazy_ext.h5/with/internal/path/to/data')
assert components.externalPath == '/some/external/path/to/file.h5_crazy_ext.h5'
assert components.extension == '.h5'
assert components.internalPath == '/with/internal/path/to/data'
# Everything should work for URLs, too.
components = PathComponents('http://somehost:8000/path/to/data/with.ext')
assert components.externalPath == 'http://somehost:8000/path/to/data/with.ext'
assert components.extension == '.ext'
assert components.internalPath is None
assert components.externalDirectory == 'http://somehost:8000/path/to/data'
assert components.filenameBase == 'with'
# Try modifying the properties and verify that the total path is updated.
components = PathComponents('/some/external/path/to/file.h5/with/internal/path/to/data')
components.extension = '.hdf5'
assert components.externalPath == '/some/external/path/to/file.hdf5'
assert components.totalPath() == '/some/external/path/to/file.hdf5/with/internal/path/to/data'
components.filenameBase = 'newbase'
assert components.totalPath() == '/some/external/path/to/newbase.hdf5/with/internal/path/to/data'
components.internalDirectory = 'new/internal/dir'
assert components.totalPath() == '/some/external/path/to/newbase.hdf5/new/internal/dir/data'
components.internalDatasetName = 'newdata'
assert components.totalPath() == '/some/external/path/to/newbase.hdf5/new/internal/dir/newdata'
components.externalDirectory = '/new/extern/dir/'
assert components.totalPath() == '/new/extern/dir/newbase.hdf5/new/internal/dir/newdata'
components.externalDirectory = '/new/extern/dir'
assert components.totalPath() == '/new/extern/dir/newbase.hdf5/new/internal/dir/newdata'
components.externalPath = '/new/externalpath/somefile.h5'
assert components.totalPath() == '/new/externalpath/somefile.h5/new/internal/dir/newdata'
components.filename = 'newfilename.h5'
assert components.totalPath() == '/new/externalpath/newfilename.h5/new/internal/dir/newdata'
components.internalPath = '/new/internal/path/dataset'
assert components.totalPath() == '/new/externalpath/newfilename.h5/new/internal/path/dataset'
def pathIsHdf5(cls, path: Path) -> bool:
return PathComponents(
Path(path).as_posix()).extension in [".ilp", ".h5", ".hdf5"]
def pathIsN5(cls, path: Path) -> bool:
return PathComponents(Path(path).as_posix()).extension in [".n5"]
def create_nickname(self, fileName: str):
comps = PathComponents(fileName)
expected_nickname = Path(comps.externalPath).stem
if comps.internalPath:
expected_nickname += comps.internalPath.replace("/", "-")
return expected_nickname
def _attemptOpenAsHdf5(self, filePath):
# Check for an hdf5 extension
pathComponents = PathComponents(filePath)
ext = pathComponents.extension
if ext not in (".%s" % x for x in OpInputDataReader.h5Exts):
return ([], None)
externalPath = pathComponents.externalPath
internalPath = pathComponents.internalPath
if not os.path.exists(externalPath):
raise OpInputDataReader.DatasetReadError(
"Input file does not exist: " + externalPath)
# Open the h5 file in read-only mode
try:
h5File = h5py.File(externalPath, 'r')
except OpInputDataReader.DatasetReadError:
raise
except Exception as e:
msg = "Unable to open HDF5 File: {}\n{}".format(
externalPath, str(e))
raise OpInputDataReader.DatasetReadError(msg)
else:
if not internalPath:
possible_internal_paths = self._get_hdf5_dataset_names(h5File)
if len(possible_internal_paths) == 1:
internalPath = possible_internal_paths[0]
elif len(possible_internal_paths) == 0:
h5File.close()
msg = "HDF5 file contains no datasets: {}".format(
externalPath)
raise OpInputDataReader.DatasetReadError(msg)
else:
h5File.close()
msg = "When using hdf5, you must append the hdf5 internal path to the "\
"data set to your filename, e.g. myfile.h5/volume/data "\
"No internal path provided for dataset in file: {}".format(
externalPath)
raise OpInputDataReader.DatasetReadError(msg)
try:
compression_setting = h5File[internalPath].compression
except Exception as e:
h5File.close()
msg = "Error reading HDF5 File: {}\n{}".format(externalPath, e)
raise OpInputDataReader.DatasetReadError(msg)
# If the h5 dataset is compressed, we'll have better performance
# with a multi-process hdf5 access object.
# (Otherwise, single-process is faster.)
allow_multiprocess_hdf5 = "LAZYFLOW_MULTIPROCESS_HDF5" in os.environ and os.environ[
"LAZYFLOW_MULTIPROCESS_HDF5"] != ""
if compression_setting is not None and allow_multiprocess_hdf5:
h5File.close()
h5File = MultiProcessHdf5File(externalPath, 'r')
self._file = h5File
h5Reader = OpStreamingHdf5Reader(parent=self)
h5Reader.Hdf5File.setValue(h5File)
try:
h5Reader.InternalPath.setValue(internalPath)
except OpStreamingHdf5Reader.DatasetReadError as e:
msg = "Error reading HDF5 File: {}\n{}".format(externalPath, e.msg)
raise OpInputDataReader.DatasetReadError(msg)
return ([h5Reader], h5Reader.OutputImage)
def getPossibleInternalPaths(self):
possible_internal_paths = set()
for expanded_path in self.expanded_paths:
external_path = PathComponents(expanded_path).externalPath
possible_internal_paths |= set(self.getPossibleInternalPathsFor(external_path))
return possible_internal_paths
def file_extensions(self) -> List[str]:
return [PathComponents(ep).extension for ep in self.expanded_paths]
def internal_paths(self) -> List[str]:
return [PathComponents(ep).internalPath for ep in self.expanded_paths]
def get_relative_paths(self) -> List[str]:
external_paths = [Path(PathComponents(path).externalPath) for path in self.expanded_paths]
try:
return sorted([str(ep.absolute().relative_to(self.base_dir)) for ep in external_paths])
except ValueError:
raise CantSaveAsRelativePathsException(self.filePath, self.base_dir)
def default_output_dir(self) -> Path:
first_external_path = PathComponents(self.filePath.split(os.path.pathsep)[0]).externalPath
return Path(first_external_path).parent
