def test_3d_vigra_along_t(self):
"""Test if 3d volumes generated through vigra are recognized correctly"""
# Prepare some data set for this case
data = numpy.random.randint(0, 255,
(10, 15, 50, 100, 3)).astype(numpy.uint8)
axistags = vigra.defaultAxistags("zyxc")
expected_axistags = vigra.defaultAxistags("tzyxc")
h5_op = OpStreamingH5N5SequenceReaderS(graph=self.graph)
n5_op = OpStreamingH5N5SequenceReaderS(graph=self.graph)
try:
testDataH5FileName = f"{self.tempdir_normalized_name}/test.h5"
testDataN5FileName = f"{self.tempdir_normalized_name}/test.n5"
# Write the dataset to an hdf5 file
# (Note: Don't use vigra to do this, which may reorder the axes)
h5File = h5py.File(testDataH5FileName)
n5File = z5py.N5File(testDataN5FileName)
try:
h5File.create_group("volumes")
n5File.create_group("volumes")
internalPathString = "subvolume-{sliceIndex:02d}"
for sliceIndex, tSlice in enumerate(data):
subpath = internalPathString.format(sliceIndex=sliceIndex)
h5File["volumes"].create_dataset(subpath, data=tSlice)
n5File["volumes"].create_dataset(subpath, data=tSlice)
# Write the axistags attribute
current_path = "volumes/{}".format(subpath)
h5File[current_path].attrs["axistags"] = axistags.toJSON()
n5File[current_path].attrs["axistags"] = axistags.toJSON()
finally:
h5File.close()
n5File.close()
# Read the data with an operator
hdf5GlobString = f"{testDataH5FileName}/volumes/subvolume-*"
n5GlobString = f"{testDataN5FileName}/volumes/subvolume-*"
h5_op.SequenceAxis.setValue("t")
n5_op.SequenceAxis.setValue("t")
h5_op.GlobString.setValue(hdf5GlobString)
n5_op.GlobString.setValue(n5GlobString)
assert h5_op.OutputImage.ready()
assert n5_op.OutputImage.ready()
assert h5_op.OutputImage.meta.axistags == expected_axistags
assert n5_op.OutputImage.meta.axistags == expected_axistags
numpy.testing.assert_array_equal(h5_op.OutputImage.value, data)
numpy.testing.assert_array_equal(n5_op.OutputImage.value, data)
finally:
h5_op.cleanUp()
n5_op.cleanUp()
def _applyPattern(self):
globStrings = self.patternEdit.text()
H5EXTS = OpStreamingH5N5SequenceReaderM.H5EXTS
N5EXTS = OpStreamingH5N5SequenceReaderM.N5EXTS
filenames = []
# see if some glob strings include HDF5 and/or N5 files
globStrings = globStrings.split(os.path.pathsep)
pcs = [PathComponents(x) for x in globStrings]
is_h5_n5 = [x.extension in (H5EXTS + N5EXTS) for x in pcs]
h5GlobStrings = os.path.pathsep.join(
[x for x, y in zip(globStrings, is_h5_n5) if y is True])
globStrings = os.path.pathsep.join(
[x for x, y in zip(globStrings, is_h5_n5) if y is False])
filenames.extend(OpStackLoader.expandGlobStrings(globStrings))
try:
OpStreamingH5N5SequenceReaderS.checkGlobString(h5GlobStrings)
# OK, if nothing raised there is a single h5 file in h5GlobStrings:
pathComponents = PathComponents(
h5GlobStrings.split(os.path.pathsep)[0])
h5file = OpStreamingH5N5Reader.get_h5_n5_file(
pathComponents.externalPath, mode="r")
filenames.extend(
"{}/{}".format(pathComponents.externalPath, internal)
for internal in OpStreamingH5N5SequenceReaderS.
expandGlobStrings(h5file, h5GlobStrings))
except (
OpStreamingH5N5SequenceReaderS.WrongFileTypeError,
OpStreamingH5N5SequenceReaderS.NotTheSameFileError,
OpStreamingH5N5SequenceReaderS.NoInternalPlaceholderError,
OpStreamingH5N5SequenceReaderS.ExternalPlaceholderError,
):
pass
try:
OpStreamingH5N5SequenceReaderM.checkGlobString(h5GlobStrings)
filenames.extend(
"{}/{}".format(external, internal)
for external, internal in zip(
*OpStreamingH5N5SequenceReaderM.expandGlobStrings(
h5GlobStrings)))
except (
OpStreamingH5N5SequenceReaderM.WrongFileTypeError,
OpStreamingH5N5SequenceReaderM.SameFileError,
OpStreamingH5N5SequenceReaderM.NoExternalPlaceholderError,
OpStreamingH5N5SequenceReaderM.InternalPlaceholderError,
):
pass
self._updateFileList(filenames)
def _applyPattern(self):
globStrings = self.patternEdit.text()
H5EXTS = OpStreamingH5N5SequenceReaderM.H5EXTS
N5EXTS = OpStreamingH5N5SequenceReaderM.N5EXTS
filenames = []
# see if some glob strings include HDF5 and/or N5 files
globStrings = globStrings.split(os.path.pathsep)
pcs = [PathComponents(x) for x in globStrings]
is_h5_n5 = [x.extension in (H5EXTS + N5EXTS) for x in pcs]
h5GlobStrings = os.path.pathsep.join([x for x, y in zip(globStrings, is_h5_n5) if y is True])
globStrings = os.path.pathsep.join([x for x, y in zip(globStrings, is_h5_n5) if y is False])
filenames.extend(OpStackLoader.expandGlobStrings(globStrings))
try:
OpStreamingH5N5SequenceReaderS.checkGlobString(h5GlobStrings)
# OK, if nothing raised there is a single h5 file in h5GlobStrings:
pathComponents = PathComponents(h5GlobStrings.split(os.path.pathsep)[0])
h5file = OpStreamingH5N5Reader.get_h5_n5_file(pathComponents.externalPath, mode='r')
filenames.extend(
"{}/{}".format(pathComponents.externalPath, internal)
for internal in OpStreamingH5N5SequenceReaderS.expandGlobStrings(h5file, h5GlobStrings))
except (
OpStreamingH5N5SequenceReaderS.WrongFileTypeError,
OpStreamingH5N5SequenceReaderS.NotTheSameFileError,
OpStreamingH5N5SequenceReaderS.NoInternalPlaceholderError,
OpStreamingH5N5SequenceReaderS.ExternalPlaceholderError):
pass
try:
OpStreamingH5N5SequenceReaderM.checkGlobString(h5GlobStrings)
filenames.extend(
"{}/{}".format(external, internal)
for external, internal
in zip(*OpStreamingH5N5SequenceReaderM.expandGlobStrings(h5GlobStrings))
)
except (
OpStreamingH5N5SequenceReaderM.WrongFileTypeError,
OpStreamingH5N5SequenceReaderM.SameFileError,
OpStreamingH5N5SequenceReaderM.NoExternalPlaceholderError,
OpStreamingH5N5SequenceReaderM.InternalPlaceholderError):
pass
self._updateFileList(filenames)
def test_expandGlobStrings(self):
expected_datasets = ["g1/g2/data2", "g1/g2/data3"]
h5_file_name = f"{self.tempdir_normalized_name}/test.h5"
n5_file_name = f"{self.tempdir_normalized_name}/test.n5"
try:
h5_file = h5py.File(h5_file_name, mode="w")
n5_file = z5py.N5File(n5_file_name, mode="w")
h5_g1 = h5_file.create_group("g1")
n5_g1 = n5_file.create_group("g1")
h5_g2 = h5_g1.create_group("g2")
n5_g2 = n5_g1.create_group("g2")
h5_g3 = h5_file.create_group("g3")
n5_g3 = n5_file.create_group("g3")
h5_g1.create_dataset("data1", data=numpy.ones((10, 10)))
n5_g1.create_dataset("data1", data=numpy.ones((10, 10)))
h5_g2.create_dataset("data2", data=numpy.ones((10, 10)))
n5_g2.create_dataset("data2", data=numpy.ones((10, 10)))
h5_g2.create_dataset("data3", data=numpy.ones((10, 10)))
n5_g2.create_dataset("data3", data=numpy.ones((10, 10)))
h5_g3.create_dataset("data4", data=numpy.ones((10, 10)))
n5_g3.create_dataset("data4", data=numpy.ones((10, 10)))
h5_file.flush()
h5_glob_res1 = OpStreamingH5N5SequenceReaderS.expandGlobStrings(
h5_file, f"{h5_file_name}/g1/g2/data*")
n5_glob_res1 = OpStreamingH5N5SequenceReaderS.expandGlobStrings(
n5_file, f"{n5_file_name}/g1/g2/data*")
self.assertEqual(h5_glob_res1, expected_datasets)
self.assertEqual(n5_glob_res1, expected_datasets)
finally:
h5_file.close()
n5_file.close()
h5_glob_res2 = OpStreamingH5N5SequenceReaderS.expandGlobStrings(
h5_file_name, f"{h5_file_name}/g1/g2/data*")
n5_glob_res2 = OpStreamingH5N5SequenceReaderS.expandGlobStrings(
n5_file_name, f"{n5_file_name}/g1/g2/data*")
self.assertEqual(h5_glob_res2, expected_datasets)
self.assertEqual(n5_glob_res2, expected_datasets)
def _attemptOpenAsH5N5Stack(self, filePath):
if not ("*" in filePath or os.path.pathsep in filePath):
return ([], None)
# Now use the .checkGlobString method of the stack readers
isSingleFile = True
try:
OpStreamingH5N5SequenceReaderS.checkGlobString(filePath)
except OpStreamingH5N5SequenceReaderS.WrongFileTypeError:
return ([], None)
except (
OpStreamingH5N5SequenceReaderS.NoInternalPlaceholderError,
OpStreamingH5N5SequenceReaderS.NotTheSameFileError,
OpStreamingH5N5SequenceReaderS.ExternalPlaceholderError,
):
isSingleFile = False
isMultiFile = True
try:
OpStreamingH5N5SequenceReaderM.checkGlobString(filePath)
except (
OpStreamingH5N5SequenceReaderM.NoExternalPlaceholderError,
OpStreamingH5N5SequenceReaderM.SameFileError,
OpStreamingH5N5SequenceReaderM.InternalPlaceholderError,
):
isMultiFile = False
assert not (isMultiFile and isSingleFile)
if isSingleFile is True:
opReader = OpStreamingH5N5SequenceReaderS(parent=self)
elif isMultiFile is True:
opReader = OpStreamingH5N5SequenceReaderM(parent=self)
try:
opReader.SequenceAxis.connect(self.SequenceAxis)
opReader.GlobString.setValue(filePath)
return ([opReader], opReader.OutputImage)
except (OpStreamingH5N5SequenceReaderM.WrongFileTypeError,
OpStreamingH5N5SequenceReaderS.WrongFileTypeError):
return ([], None)
def test_expandGlobStrings(self):
expected_datasets = ["g1/g2/data2", "g1/g2/data3"]
h5_file_name = f"{self.tempdir_normalized_name}/test.h5"
n5_file_name = f"{self.tempdir_normalized_name}/test.n5"
try:
h5_file = h5py.File(h5_file_name, mode="w")
n5_file = z5py.N5File(n5_file_name, mode="w")
h5_g1 = h5_file.create_group("g1")
n5_g1 = n5_file.create_group("g1")
h5_g2 = h5_g1.create_group("g2")
n5_g2 = n5_g1.create_group("g2")
h5_g3 = h5_file.create_group("g3")
n5_g3 = n5_file.create_group("g3")
h5_g1.create_dataset("data1", data=numpy.ones((10, 10)))
n5_g1.create_dataset("data1", data=numpy.ones((10, 10)))
h5_g2.create_dataset("data2", data=numpy.ones((10, 10)))
n5_g2.create_dataset("data2", data=numpy.ones((10, 10)))
h5_g2.create_dataset("data3", data=numpy.ones((10, 10)))
n5_g2.create_dataset("data3", data=numpy.ones((10, 10)))
h5_g3.create_dataset("data4", data=numpy.ones((10, 10)))
n5_g3.create_dataset("data4", data=numpy.ones((10, 10)))
h5_file.flush()
h5_glob_res1 = OpStreamingH5N5SequenceReaderS.expandGlobStrings(h5_file, f"{h5_file_name}/g1/g2/data*")
n5_glob_res1 = OpStreamingH5N5SequenceReaderS.expandGlobStrings(n5_file, f"{n5_file_name}/g1/g2/data*")
self.assertEqual(h5_glob_res1, expected_datasets)
self.assertEqual(n5_glob_res1, expected_datasets)
finally:
h5_file.close()
n5_file.close()
h5_glob_res2 = OpStreamingH5N5SequenceReaderS.expandGlobStrings(h5_file_name, f"{h5_file_name}/g1/g2/data*")
n5_glob_res2 = OpStreamingH5N5SequenceReaderS.expandGlobStrings(n5_file_name, f"{n5_file_name}/g1/g2/data*")
self.assertEqual(h5_glob_res2, expected_datasets)
self.assertEqual(n5_glob_res2, expected_datasets)
def __init__(self,
filepath=None,
jsonNamespace=None,
cwd=None,
preloaded_array=None,
sequence_axis=None):
"""
filepath: may be a globstring or a full hdf5 path+dataset
jsonNamespace: If provided, overrides default settings after filepath is applied
cwd: The working directory for interpeting relative paths. If not provided, os.getcwd() is used.
preloaded_array: Instead of providing a filePath to read from, a pre-loaded array can be directly provided.
In that case, you'll probably want to configure the axistags member, or provide a tagged
vigra.VigraArray.
sequence_axis: Axis along which to stack (only applicable for stacks).
"""
assert preloaded_array is None or not filepath, "You can't provide filepath and a preloaded_array"
cwd = cwd or os.getcwd()
self.preloaded_array = preloaded_array # See description above.
Location = DatasetInfo.Location
# The original path to the data (also used as a fallback if the data isn't in the project yet)
self._filePath = ""
self._datasetId = "" # The name of the data within the project file (if it is stored locally)
# OBSOLETE: Whether or not this dataset should be used for training a classifier.
self.allowLabels = True
self.drange = None
self.normalizeDisplay = True
self.sequenceAxis = None
self.fromstack = False
self.nickname = ""
self.axistags = None
self.original_axistags = None
# Necessary in headless mode in order to recover the shape of the raw data
self.laneShape = None
self.laneDtype = None
# A flag indicating whether the dataset is backed by a real source (e.g. file)
# or by the fake provided (e.g. in headless mode when raw data are not necessary)
self.realDataSource = True
self.subvolume_roi = None
self.location = Location.FileSystem
self.display_mode = 'default' # choices: default, grayscale, rgba, random-colortable, binary-mask.
if self.preloaded_array is not None:
self.filePath = "" # set property to ensure unique _datasetId
self.location = Location.PreloadedArray
self.nickname = "preloaded-{}-array".format(
self.preloaded_array.dtype.name)
if hasattr(self.preloaded_array, 'axistags'):
self.axistags = self.preloaded_array.axistags
# Set defaults for location, nickname, filepath, and fromstack
if filepath:
# Check for sequences (either globstring or separated paths),
file_list = None
# To support h5 sequences, filepath may contain external and
# internal path components
if not isUrl(filepath):
file_list = filepath.split(os.path.pathsep)
pathComponents = [PathComponents(x) for x in file_list]
externalPaths = [pc.externalPath for pc in pathComponents]
internalPaths = [pc.internalPath for pc in pathComponents]
if len(file_list) > 0:
if len(externalPaths) == 1:
if '*' in externalPaths[0]:
if internalPaths[0] is not None:
assert ('*' not in internalPaths[0]), (
"Only internal OR external glob placeholder supported"
)
file_list = sorted(glob.glob(filepath))
else:
file_list = [externalPaths[0]]
if internalPaths[0] is not None:
if '*' in internalPaths[0]:
# overwrite internalPaths, will be assembled further down
glob_string = "{}{}".format(
externalPaths[0], internalPaths[0])
internalPaths = \
OpStreamingH5N5SequenceReaderS.expandGlobStrings(
externalPaths[0], glob_string)
if internalPaths:
file_list = [externalPaths[0]
] * len(internalPaths)
else:
file_list = None
else:
assert (not any('*' in ep for ep in externalPaths)), (
"Multiple glob paths shouldn't be happening")
file_list = [ex for ex in externalPaths]
assert all(
pc.extension == pathComponents[0].extension
for pc in pathComponents[1::]), (
"Supplied multiple files with multiple extensions")
# The following is necessary for h5 as well as npz-files
internalPathExts = (OpInputDataReader.h5_n5_Exts +
OpInputDataReader.npzExts)
internalPathExts = [
".{}".format(ipx) for ipx in internalPathExts
]
if pathComponents[
0].extension in internalPathExts and internalPaths:
if len(file_list) == len(internalPaths):
# assuming a matching internal paths to external paths
file_list_with_internal = []
for external, internal in zip(
file_list, internalPaths):
if internal:
file_list_with_internal.append(
'{}/{}'.format(external, internal))
else:
file_list_with_internal.append(external)
file_list = file_list_with_internal
else:
# sort of fallback, in case of a mismatch in lengths
for i in range(len(file_list)):
file_list[i] += '/' + internalPaths[0]
# For stacks, choose nickname based on a common prefix
if file_list:
fromstack = True
# Convert all paths to absolute
file_list = [make_absolute(f, cwd) for f in file_list]
if '*' in filepath:
filepath = make_absolute(filepath, cwd)
else:
filepath = os.path.pathsep.join(file_list)
# Add an underscore for each wildcard digit
prefix = os.path.commonprefix(file_list)
num_wildcards = len(file_list[-1]) - len(prefix) - len(
os.path.splitext(file_list[-1])[1])
nickname = PathComponents(prefix).filenameBase + (
"_" * num_wildcards)
else:
fromstack = False
if not isUrl(filepath):
# Convert all (non-url) paths to absolute
filepath = make_absolute(filepath, cwd)
nickname = PathComponents(filepath).filenameBase
self.location = DatasetInfo.Location.FileSystem
self.nickname = nickname
self.filePath = filepath
self.fromstack = fromstack
self.sequenceAxis = sequence_axis
if jsonNamespace is not None:
self.updateFromJson(jsonNamespace)
def test_globStringValidity(self):
"""Check whether globStrings are correctly verified"""
testGlobString = "/tmp/test.h5"
with self.assertRaises(
OpStreamingH5N5SequenceReaderS.NoInternalPlaceholderError):
OpStreamingH5N5SequenceReaderS.checkGlobString(testGlobString)
testGlobString = "/tmp/test.n5"
with self.assertRaises(
OpStreamingH5N5SequenceReaderS.NoInternalPlaceholderError):
OpStreamingH5N5SequenceReaderS.checkGlobString(testGlobString)
testGlobString = "/tmp/test.h5/a" + os.pathsep + "/tmp/test2.h5/a"
with self.assertRaises(
OpStreamingH5N5SequenceReaderS.NotTheSameFileError):
OpStreamingH5N5SequenceReaderS.checkGlobString(testGlobString)
testGlobString = "/tmp/test.n5/a" + os.pathsep + "/tmp/test2.n5/a"
with self.assertRaises(
OpStreamingH5N5SequenceReaderS.NotTheSameFileError):
OpStreamingH5N5SequenceReaderS.checkGlobString(testGlobString)
testGlobString = "/tmp/test*.h5/a" + os.pathsep + "/tmp/test*.h5/a"
with self.assertRaises(
OpStreamingH5N5SequenceReaderS.ExternalPlaceholderError):
OpStreamingH5N5SequenceReaderS.checkGlobString(testGlobString)
testGlobString = "/tmp/test*.n5/a" + os.pathsep + "/tmp/test*.n5/a"
with self.assertRaises(
OpStreamingH5N5SequenceReaderS.ExternalPlaceholderError):
OpStreamingH5N5SequenceReaderS.checkGlobString(testGlobString)
testGlobString = "/tmp/test.jpg/*"
with self.assertRaises(
OpStreamingH5N5SequenceReaderS.WrongFileTypeError):
OpStreamingH5N5SequenceReaderS.checkGlobString(testGlobString)
validGlobStrings = [
"/tmp/test.h5/*",
"/tmp/test.h5/data1" + os.pathsep + "/tmp/test.h5/data2",
"/tmp/test.h5/data*",
"/tmp/test.n5/*",
"/tmp/test.n5/data1" + os.pathsep + "/tmp/test.n5/data2",
"/tmp/test.n5/data*",
]
# Implicit test for validity; test fails if an exception is raised
for testGlobString in validGlobStrings:
OpStreamingH5N5SequenceReaderS.checkGlobString(testGlobString)
self.assertTrue(True)
def test_globStringValidity(self):
"""Check whether globStrings are correctly verified"""
testGlobString = "/tmp/test.h5"
with self.assertRaises(OpStreamingH5N5SequenceReaderS.NoInternalPlaceholderError):
OpStreamingH5N5SequenceReaderS.checkGlobString(testGlobString)
testGlobString = "/tmp/test.n5"
with self.assertRaises(OpStreamingH5N5SequenceReaderS.NoInternalPlaceholderError):
OpStreamingH5N5SequenceReaderS.checkGlobString(testGlobString)
testGlobString = "/tmp/test.h5/a" + os.pathsep + "/tmp/test2.h5/a"
with self.assertRaises(OpStreamingH5N5SequenceReaderS.NotTheSameFileError):
OpStreamingH5N5SequenceReaderS.checkGlobString(testGlobString)
testGlobString = "/tmp/test.n5/a" + os.pathsep + "/tmp/test2.n5/a"
with self.assertRaises(OpStreamingH5N5SequenceReaderS.NotTheSameFileError):
OpStreamingH5N5SequenceReaderS.checkGlobString(testGlobString)
testGlobString = "/tmp/test*.h5/a" + os.pathsep + "/tmp/test*.h5/a"
with self.assertRaises(OpStreamingH5N5SequenceReaderS.ExternalPlaceholderError):
OpStreamingH5N5SequenceReaderS.checkGlobString(testGlobString)
testGlobString = "/tmp/test*.n5/a" + os.pathsep + "/tmp/test*.n5/a"
with self.assertRaises(OpStreamingH5N5SequenceReaderS.ExternalPlaceholderError):
OpStreamingH5N5SequenceReaderS.checkGlobString(testGlobString)
testGlobString = "/tmp/test.jpg/*"
with self.assertRaises(OpStreamingH5N5SequenceReaderS.WrongFileTypeError):
OpStreamingH5N5SequenceReaderS.checkGlobString(testGlobString)
validGlobStrings = [
"/tmp/test.h5/*",
"/tmp/test.h5/data1" + os.pathsep + "/tmp/test.h5/data2",
"/tmp/test.h5/data*",
"/tmp/test.n5/*",
"/tmp/test.n5/data1" + os.pathsep + "/tmp/test.n5/data2",
"/tmp/test.n5/data*",
]
# Implicit test for validity; test fails if an exception is raised
for testGlobString in validGlobStrings:
OpStreamingH5N5SequenceReaderS.checkGlobString(testGlobString)
self.assertTrue(True)
def __init__(self, filepath=None, jsonNamespace=None, cwd=None,
preloaded_array=None, sequence_axis=None):
"""
filepath: may be a globstring or a full hdf5 path+dataset
jsonNamespace: If provided, overrides default settings after filepath is applied
cwd: The working directory for interpeting relative paths. If not provided, os.getcwd() is used.
preloaded_array: Instead of providing a filePath to read from, a pre-loaded array can be directly provided.
In that case, you'll probably want to configure the axistags member, or provide a tagged
vigra.VigraArray.
sequence_axis: Axis along which to stack (only applicable for stacks).
"""
assert preloaded_array is None or not filepath, "You can't provide filepath and a preloaded_array"
cwd = cwd or os.getcwd()
self.preloaded_array = preloaded_array # See description above.
Location = DatasetInfo.Location
# The original path to the data (also used as a fallback if the data isn't in the project yet)
self._filePath = ""
self._datasetId = "" # The name of the data within the project file (if it is stored locally)
# OBSOLETE: Whether or not this dataset should be used for training a classifier.
self.allowLabels = True
self.drange = None
self.normalizeDisplay = True
self.sequenceAxis = None
self.fromstack = False
self.nickname = ""
self.axistags = None
self.original_axistags = None
# Necessary in headless mode in order to recover the shape of the raw data
self.laneShape = None
self.laneDtype = None
# A flag indicating whether the dataset is backed by a real source (e.g. file)
# or by the fake provided (e.g. in headless mode when raw data are not necessary)
self.realDataSource = True
self.subvolume_roi = None
self.location = Location.FileSystem
self.display_mode = 'default' # choices: default, grayscale, rgba, random-colortable, binary-mask.
if self.preloaded_array is not None:
self.filePath = "" # set property to ensure unique _datasetId
self.location = Location.PreloadedArray
self.nickname = "preloaded-{}-array".format(self.preloaded_array.dtype.name)
if hasattr(self.preloaded_array, 'axistags'):
self.axistags = self.preloaded_array.axistags
# Set defaults for location, nickname, filepath, and fromstack
if filepath:
# Check for sequences (either globstring or separated paths),
file_list = None
# To support h5 sequences, filepath may contain external and
# internal path components
if not isUrl(filepath):
file_list = filepath.split(os.path.pathsep)
pathComponents = [PathComponents(x) for x in file_list]
externalPaths = [pc.externalPath for pc in pathComponents]
internalPaths = [pc.internalPath for pc in pathComponents]
if len(file_list) > 0:
if len(externalPaths) == 1:
if '*' in externalPaths[0]:
if internalPaths[0] is not None:
assert ('*' not in internalPaths[0]), (
"Only internal OR external glob placeholder supported"
)
file_list = sorted(glob.glob(filepath))
else:
file_list = [externalPaths[0]]
if internalPaths[0] is not None:
if '*' in internalPaths[0]:
# overwrite internalPaths, will be assembled further down
glob_string = "{}{}".format(externalPaths[0], internalPaths[0])
internalPaths = \
OpStreamingH5N5SequenceReaderS.expandGlobStrings(
externalPaths[0], glob_string)
if internalPaths:
file_list = [externalPaths[0]] * len(internalPaths)
else:
file_list = None
else:
assert (not any('*' in ep for ep in externalPaths)), (
"Multiple glob paths shouldn't be happening"
)
file_list = [ex for ex in externalPaths]
assert all(pc.extension == pathComponents[0].extension
for pc in pathComponents[1::]), (
"Supplied multiple files with multiple extensions"
)
# The following is necessary for h5 as well as npz-files
internalPathExts = (
OpInputDataReader.h5_n5_Exts +
OpInputDataReader.npzExts
)
internalPathExts = [".{}".format(ipx) for ipx in internalPathExts]
if pathComponents[0].extension in internalPathExts and internalPaths:
if len(file_list) == len(internalPaths):
# assuming a matching internal paths to external paths
file_list_with_internal = []
for external, internal in zip(file_list, internalPaths):
if internal:
file_list_with_internal.append('{}/{}'.format(external, internal))
else:
file_list_with_internal.append(external)
file_list = file_list_with_internal
else:
# sort of fallback, in case of a mismatch in lengths
for i in range(len(file_list)):
file_list[i] += '/' + internalPaths[0]
# For stacks, choose nickname based on a common prefix
if file_list:
fromstack = True
# Convert all paths to absolute
file_list = [make_absolute(f, cwd) for f in file_list]
if '*' in filepath:
filepath = make_absolute(filepath, cwd)
else:
filepath = os.path.pathsep.join(file_list)
# Add an underscore for each wildcard digit
prefix = os.path.commonprefix(file_list)
num_wildcards = len(file_list[-1]) - len(prefix) - len(os.path.splitext(file_list[-1])[1])
nickname = PathComponents(prefix).filenameBase + ("_" * num_wildcards)
else:
fromstack = False
if not isUrl(filepath):
# Convert all (non-url) paths to absolute
filepath = make_absolute(filepath, cwd)
nickname = PathComponents(filepath).filenameBase
self.location = DatasetInfo.Location.FileSystem
self.nickname = nickname
self.filePath = filepath
self.fromstack = fromstack
self.sequenceAxis = sequence_axis
if jsonNamespace is not None:
self.updateFromJson(jsonNamespace)