def test_Writer(self):
opData = OpArrayCache( graph=self.graph )
opData.blockShape.setValue( self.testData.shape )
opData.Input.setValue( self.testData )
opExport = OpExportMultipageTiffSequence(graph=self.graph)
opExport.FilepathPattern.setValue( self._stack_filepattern )
opExport.Input.connect( opData.Output )
opExport.SliceIndexOffset.setValue(22)
# Run the export
opExport.run_export()
globstring = self._stack_filepattern.format( slice_index=999 )
globstring = globstring.replace('999', '*')
opReader = OpTiffSequenceReader( graph=self.graph )
opReader.GlobString.setValue( globstring )
# (The OpStackLoader produces txyzc order.)
opReorderAxes = OpReorderAxes( graph=self.graph )
opReorderAxes.AxisOrder.setValue( self._axisorder )
opReorderAxes.Input.connect( opReader.Output )
readData = opReorderAxes.Output[:].wait()
logger.debug("Expected shape={}".format( self.testData.shape ) )
logger.debug("Read shape={}".format( readData.shape ) )
assert opReorderAxes.Output.meta.shape == self.testData.shape, "Exported files were of the wrong shape or number."
assert (opReorderAxes.Output[:].wait() == self.testData.view( numpy.ndarray )).all(), "Exported data was not correct"
opReorderAxes.cleanUp()
opReader.cleanUp()
def test_Writer(self):
opData = OpArrayCache(graph=self.graph)
opData.blockShape.setValue(self.testData.shape)
opData.Input.setValue(self.testData)
opExport = OpExportMultipageTiffSequence(graph=self.graph)
opExport.FilepathPattern.setValue(self._stack_filepattern)
opExport.Input.connect(opData.Output)
opExport.SliceIndexOffset.setValue(22)
# Run the export
opExport.run_export()
globstring = self._stack_filepattern.format(slice_index=999)
globstring = globstring.replace('999', '*')
opReader = OpTiffSequenceReader(graph=self.graph)
opReader.GlobString.setValue(globstring)
# (The OpStackLoader produces txyzc order.)
opReorderAxes = OpReorderAxes(graph=self.graph)
opReorderAxes.AxisOrder.setValue(self._axisorder)
opReorderAxes.Input.connect(opReader.Output)
readData = opReorderAxes.Output[:].wait()
logger.debug("Expected shape={}".format(self.testData.shape))
logger.debug("Read shape={}".format(readData.shape))
assert opReorderAxes.Output.meta.shape == self.testData.shape, "Exported files were of the wrong shape or number."
assert (opReorderAxes.Output[:].wait() == self.testData.view(
numpy.ndarray)).all(), "Exported data was not correct"
opReorderAxes.cleanUp()
opReader.cleanUp()
def importStackAsLocalDataset(self, info, sequence_axis='t'):
"""
Add the given stack data to the project file as a local dataset.
Does not update the topLevelOperator.
:param info: A DatasetInfo object.
Note: info.filePath must be a str which lists the stack files, delimited with os.path.pathsep
Note: info will be MODIFIED by this function. Use the modified info when assigning it to a dataset.
"""
self.progressSignal.emit(0)
projectFileHdf5 = self.topLevelOperator.ProjectFile.value
globstring = info.filePath
info.location = DatasetInfo.Location.ProjectInternal
firstPathParts = PathComponents(info.filePath.split(os.path.pathsep)[0])
info.filePath = firstPathParts.externalDirectory + '/??' + firstPathParts.extension
info.fromstack = True
# Use absolute path
cwd = self.topLevelOperator.WorkingDirectory
if os.path.pathsep not in globstring and not os.path.isabs(globstring):
globstring = os.path.normpath( os.path.join(cwd, globstring) )
if firstPathParts.extension.lower() in OpTiffReader.TIFF_EXTS:
# Special loader for TIFFs
opLoader = OpTiffSequenceReader( parent=self.topLevelOperator.parent )
opLoader.SequenceAxis.setValue(sequence_axis)
opLoader.GlobString.setValue(globstring)
data_slot = opLoader.Output
else:
# All other sequences (e.g. pngs, jpegs, etc.)
opLoader = OpStackLoader( parent=self.topLevelOperator.parent )
opLoader.SequenceAxis.setValue(sequence_axis)
opLoader.globstring.setValue(globstring)
data_slot = opLoader.stack
try:
opWriter = OpH5WriterBigDataset(parent=self.topLevelOperator.parent)
opWriter.hdf5File.setValue(projectFileHdf5)
opWriter.hdf5Path.setValue(self.topGroupName + '/local_data/' + info.datasetId)
opWriter.CompressionEnabled.setValue(False)
# We assume that the main bottleneck is the hard disk,
# so adding lots of threads to access it at once seems like a bad idea.
opWriter.BatchSize.setValue(1)
opWriter.Image.connect( data_slot )
# Forward progress from the writer directly to our applet
opWriter.progressSignal.subscribe( self.progressSignal.emit )
success = opWriter.WriteImage.value
finally:
opWriter.cleanUp()
opLoader.cleanUp()
self.progressSignal.emit(100)
return success
def testBasic_MultipageTiffSequence(self):
data = 255 * numpy.random.random((5, 10, 50, 100, 3))
data = data.astype(numpy.uint8)
data = vigra.taggedView(data, vigra.defaultAxistags('tzyxc'))
# Must run this through an operator
# Can't use opExport.setValue() because because OpStackWriter can't work with ValueRequests
graph = Graph()
opData = OpBlockedArrayCache(graph=graph)
opData.BlockShape.setValue(data.shape)
opData.Input.setValue(data)
filepattern = self._tmpdir + '/test_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}_t{slice_index}'
opExport = OpExportSlot(graph=graph)
opExport.Input.connect(opData.Output)
opExport.OutputFormat.setValue('multipage tiff sequence')
opExport.OutputFilenameFormat.setValue(filepattern)
opExport.CoordinateOffset.setValue((7, 10, 20, 30, 0))
opExport.run_export()
export_pattern = opExport.ExportPath.value
globstring = export_pattern.format(slice_index=999)
globstring = globstring.replace('999', '*')
opReader = OpTiffSequenceReader(graph=graph)
opReorderAxes = OpReorderAxes(graph=graph)
try:
opReader.GlobString.setValue(globstring)
# (The OpStackLoader produces txyzc order.)
opReorderAxes.AxisOrder.setValue('tzyxc')
opReorderAxes.Input.connect(opReader.Output)
assert opReorderAxes.Output.meta.shape == data.shape, "Exported files were of the wrong shape or number."
assert (opReorderAxes.Output[:].wait() == data.view(
numpy.ndarray)).all(), "Exported data was not correct"
finally:
opReorderAxes.cleanUp()
opReader.cleanUp()
def testBasic_MultipageTiffSequence(self):
data = 255 * numpy.random.random((5, 10, 50, 100, 3))
data = data.astype(numpy.uint8)
data = vigra.taggedView(data, vigra.defaultAxistags("tzyxc"))
# Must run this through an operator
# Can't use opExport.setValue() because because OpStackWriter can't work with ValueRequests
graph = Graph()
opData = OpBlockedArrayCache(graph=graph)
opData.BlockShape.setValue(data.shape)
opData.Input.setValue(data)
filepattern = self._tmpdir + "/test_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}_t{slice_index}"
opExport = OpExportSlot(graph=graph)
opExport.Input.connect(opData.Output)
opExport.OutputFormat.setValue("multipage tiff sequence")
opExport.OutputFilenameFormat.setValue(filepattern)
opExport.CoordinateOffset.setValue((7, 10, 20, 30, 0))
opExport.run_export()
export_pattern = opExport.ExportPath.value
globstring = export_pattern.format(slice_index=999)
globstring = globstring.replace("999", "*")
opReader = OpTiffSequenceReader(graph=graph)
opReorderAxes = OpReorderAxes(graph=graph)
try:
opReader.GlobString.setValue(globstring)
# (The OpStackLoader produces txyzc order.)
opReorderAxes.AxisOrder.setValue("tzyxc")
opReorderAxes.Input.connect(opReader.Output)
assert opReorderAxes.Output.meta.shape == data.shape, "Exported files were of the wrong shape or number."
assert (opReorderAxes.Output[:].wait() == data.view(numpy.ndarray)).all(), "Exported data was not correct"
finally:
opReorderAxes.cleanUp()
opReader.cleanUp()
def importStackAsLocalDataset(self, info, sequence_axis='t'):
"""
Add the given stack data to the project file as a local dataset.
Does not update the topLevelOperator.
:param info: A DatasetInfo object.
Note: info.filePath must be a str which lists the stack files, delimited with os.path.pathsep
Note: info will be MODIFIED by this function. Use the modified info when assigning it to a dataset.
"""
self.progressSignal(0)
projectFileHdf5 = self.topLevelOperator.ProjectFile.value
globstring = info.filePath
info.location = DatasetInfo.Location.ProjectInternal
firstPathParts = PathComponents(info.filePath.split(os.path.pathsep)[0])
info.filePath = firstPathParts.externalDirectory + '/??' + firstPathParts.extension
info.fromstack = True
# Use absolute path
cwd = self.topLevelOperator.WorkingDirectory
if os.path.pathsep not in globstring and not os.path.isabs(globstring):
globstring = os.path.normpath( os.path.join(cwd, globstring) )
if firstPathParts.extension.lower() in OpTiffReader.TIFF_EXTS:
# Special loader for TIFFs
opLoader = OpTiffSequenceReader( parent=self.topLevelOperator.parent )
opLoader.SequenceAxis.setValue(sequence_axis)
opLoader.GlobString.setValue(globstring)
data_slot = opLoader.Output
elif firstPathParts.extension.lower() in (OpStreamingH5N5SequenceReaderM.H5EXTS
+ OpStreamingH5N5SequenceReaderM.N5EXTS):
# Now use the .checkGlobString method of the stack readers
isSingleFile = True
try:
OpStreamingH5N5SequenceReaderS.checkGlobString(globstring)
except (OpStreamingH5N5SequenceReaderS.NoInternalPlaceholderError,
OpStreamingH5N5SequenceReaderS.NotTheSameFileError,
OpStreamingH5N5SequenceReaderS.ExternalPlaceholderError):
isSingleFile = False
isMultiFile = True
try:
OpStreamingH5N5SequenceReaderM.checkGlobString(globstring)
except (OpStreamingH5N5SequenceReaderM.NoExternalPlaceholderError,
OpStreamingH5N5SequenceReaderM.SameFileError,
OpStreamingH5N5SequenceReaderM.InternalPlaceholderError):
isMultiFile = False
assert (not(isMultiFile and isSingleFile)), (
"Something is wrong, glob string shouldn't allow both")
assert (isMultiFile or isSingleFile), (
"Glob string doesn't conform to h5 stack glob string rules")
if isSingleFile:
opLoader = OpStreamingH5N5SequenceReaderS(parent=self.topLevelOperator.parent)
elif isMultiFile:
opLoader = OpStreamingH5N5SequenceReaderM(parent=self.topLevelOperator.parent)
opLoader.SequenceAxis.setValue(sequence_axis)
opLoader.GlobString.setValue(globstring)
data_slot = opLoader.OutputImage
else:
# All other sequences (e.g. pngs, jpegs, etc.)
opLoader = OpStackLoader( parent=self.topLevelOperator.parent )
opLoader.SequenceAxis.setValue(sequence_axis)
opLoader.globstring.setValue(globstring)
data_slot = opLoader.stack
try:
opWriter = OpH5N5WriterBigDataset(parent=self.topLevelOperator.parent)
opWriter.h5N5File.setValue(projectFileHdf5)
opWriter.h5N5Path.setValue(self.topGroupName + '/local_data/' + info.datasetId)
opWriter.CompressionEnabled.setValue(False)
# We assume that the main bottleneck is the hard disk,
# so adding lots of threads to access it at once seems like a bad idea.
opWriter.BatchSize.setValue(1)
opWriter.Image.connect( data_slot )
# Forward progress from the writer directly to our applet
opWriter.progressSignal.subscribe(self.progressSignal)
success = opWriter.WriteImage.value
finally:
opWriter.cleanUp()
opLoader.cleanUp()
self.progressSignal(100)
return success
def importStackAsLocalDataset(self, info, sequence_axis='t'):
"""
Add the given stack data to the project file as a local dataset.
Does not update the topLevelOperator.
:param info: A DatasetInfo object.
Note: info.filePath must be a str which lists the stack files, delimited with os.path.pathsep
Note: info will be MODIFIED by this function. Use the modified info when assigning it to a dataset.
"""
self.progressSignal(0)
projectFileHdf5 = self.topLevelOperator.ProjectFile.value
globstring = info.filePath
info.location = DatasetInfo.Location.ProjectInternal
firstPathParts = PathComponents(
info.filePath.split(os.path.pathsep)[0])
info.filePath = firstPathParts.externalDirectory + '/??' + firstPathParts.extension
info.fromstack = True
# Use absolute path
cwd = self.topLevelOperator.WorkingDirectory
if os.path.pathsep not in globstring and not os.path.isabs(globstring):
globstring = os.path.normpath(os.path.join(cwd, globstring))
if firstPathParts.extension.lower() in OpTiffReader.TIFF_EXTS:
# Special loader for TIFFs
opLoader = OpTiffSequenceReader(
parent=self.topLevelOperator.parent)
opLoader.SequenceAxis.setValue(sequence_axis)
opLoader.GlobString.setValue(globstring)
data_slot = opLoader.Output
elif firstPathParts.extension.lower() in (
OpStreamingH5N5SequenceReaderM.H5EXTS +
OpStreamingH5N5SequenceReaderM.N5EXTS):
# Now use the .checkGlobString method of the stack readers
isSingleFile = True
try:
OpStreamingH5N5SequenceReaderS.checkGlobString(globstring)
except (OpStreamingH5N5SequenceReaderS.NoInternalPlaceholderError,
OpStreamingH5N5SequenceReaderS.NotTheSameFileError,
OpStreamingH5N5SequenceReaderS.ExternalPlaceholderError):
isSingleFile = False
isMultiFile = True
try:
OpStreamingH5N5SequenceReaderM.checkGlobString(globstring)
except (OpStreamingH5N5SequenceReaderM.NoExternalPlaceholderError,
OpStreamingH5N5SequenceReaderM.SameFileError,
OpStreamingH5N5SequenceReaderM.InternalPlaceholderError):
isMultiFile = False
assert (not (isMultiFile and isSingleFile)), (
"Something is wrong, glob string shouldn't allow both")
assert (isMultiFile or isSingleFile), (
"Glob string doesn't conform to h5 stack glob string rules")
if isSingleFile:
opLoader = OpStreamingH5N5SequenceReaderS(
parent=self.topLevelOperator.parent)
elif isMultiFile:
opLoader = OpStreamingH5N5SequenceReaderM(
parent=self.topLevelOperator.parent)
opLoader.SequenceAxis.setValue(sequence_axis)
opLoader.GlobString.setValue(globstring)
data_slot = opLoader.OutputImage
else:
# All other sequences (e.g. pngs, jpegs, etc.)
opLoader = OpStackLoader(parent=self.topLevelOperator.parent)
opLoader.SequenceAxis.setValue(sequence_axis)
opLoader.globstring.setValue(globstring)
data_slot = opLoader.stack
try:
opWriter = OpH5N5WriterBigDataset(
parent=self.topLevelOperator.parent)
opWriter.h5N5File.setValue(projectFileHdf5)
opWriter.h5N5Path.setValue(self.topGroupName + '/local_data/' +
info.datasetId)
opWriter.CompressionEnabled.setValue(False)
# We assume that the main bottleneck is the hard disk,
# so adding lots of threads to access it at once seems like a bad idea.
opWriter.BatchSize.setValue(1)
opWriter.Image.connect(data_slot)
# Forward progress from the writer directly to our applet
opWriter.progressSignal.subscribe(self.progressSignal)
success = opWriter.WriteImage.value
finally:
opWriter.cleanUp()
opLoader.cleanUp()
self.progressSignal(100)
return success