def setupClass(cls):
# data = vigra.taggedView(numpy.random.random((2, 20, 25, 30, 3)), 'tzyxc')
graph = Graph()
graph_old = Graph()
# Define operators
opFeatures = OperatorWrapper(OpFeatureSelection, graph=graph)
opFeaturesOld = OperatorWrapper(OpFeatureSelectionOld, graph=graph_old)
opFeatures.InputImage.resize(1)
opFeaturesOld.InputImage.resize(1)
# Configure scales
scales = [0.3, 0.7, 1, 1.6, 5.0]
opFeatures.Scales.setValue(scales)
opFeaturesOld.Scales.setValue(scales)
# Configure feature types
featureIds = [
'GaussianSmoothing', 'LaplacianOfGaussian',
'StructureTensorEigenvalues', 'HessianOfGaussianEigenvalues',
'GaussianGradientMagnitude', 'DifferenceOfGaussians'
]
opFeatures.FeatureIds.setValue(featureIds)
opFeaturesOld.FeatureIds.setValue(featureIds)
cls.opFeatures = opFeatures
cls.opFeaturesOld = opFeaturesOld
def setup_method(self, method):
# data = vigra.taggedView(numpy.random.random((2, 20, 25, 30, 3)), 'tzyxc')
graph = Graph()
graph_old = Graph()
# Define operators
opFeatures = OperatorWrapper(OpFeatureSelection, graph=graph)
opFeaturesOld = OperatorWrapper(OpFeatureSelectionOld, graph=graph_old)
opFeatures.InputImage.resize(1)
opFeaturesOld.InputImage.resize(1)
# Configure scales
scales = [0.3, 0.7, 1, 1.6, 5.0]
opFeatures.Scales.setValue(scales)
opFeaturesOld.Scales.setValue(scales)
# Configure feature types
featureIds = [
"GaussianSmoothing",
"LaplacianOfGaussian",
"StructureTensorEigenvalues",
"HessianOfGaussianEigenvalues",
"GaussianGradientMagnitude",
"DifferenceOfGaussians",
]
opFeatures.FeatureIds.setValue(featureIds)
opFeaturesOld.FeatureIds.setValue(featureIds)
self.opFeatures = opFeatures
self.opFeaturesOld = opFeaturesOld
def __init__(self, *args, **kwargs):
super( OpSplitBodySupervoxelExport, self ).__init__(*args, **kwargs)
# HACK: Be sure that the output slots are resized if the raveler body list changes
self.AnnotationBodyIds.notifyDirty( bind(self._setupOutputs) )
# Prepare a set of OpSelectLabels for easy access to raveler object masks
self._opSelectLabel = OperatorWrapper( OpSelectLabel, parent=self, broadcastingSlotNames=['Input'] )
self._opSelectLabel.Input.connect( self.RavelerLabels )
self.EditedRavelerBodies.connect( self._opSelectLabel.Output )
# Mask in the body of interest
self._opMaskedSelect = OperatorWrapper( OpMaskedSelectUint32, parent=self, broadcastingSlotNames=['Input'] )
self._opMaskedSelect.Input.connect( self.Supervoxels )
self._opMaskedSelect.Mask.connect( self._opSelectLabel.Output )
self.MaskedSupervoxels.connect( self._opMaskedSelect.Output )
# Must run CC before filter, to ensure that discontiguous labels can't avoid the filter.
self._opRelabelMaskedSupervoxels = OperatorWrapper( OpVigraLabelVolume, parent=self )
self._opRelabelMaskedSupervoxels.Input.connect( self._opMaskedSelect.Output )
self._opRelabeledMaskedSupervoxelCaches = OperatorWrapper( OpCompressedCache, parent=self )
self._opRelabeledMaskedSupervoxelCaches.Input.connect( self._opRelabelMaskedSupervoxels.Output )
# Filter out the small CC to eliminate tiny pieces of supervoxels that overlap the mask boundaries
self._opSmallLabelFilter = OperatorWrapper( OpFilterLabels, parent=self, broadcastingSlotNames=['MinLabelSize'] )
self._opSmallLabelFilter.MinLabelSize.setValue( 10 )
self._opSmallLabelFilter.Input.connect( self._opRelabeledMaskedSupervoxelCaches.Output )
self._opSmallLabelFilterCaches = OperatorWrapper( OpCompressedCache, parent=self )
self._opSmallLabelFilterCaches.Input.connect( self._opSmallLabelFilter.Output )
self.FilteredMaskedSupervoxels.connect( self._opSmallLabelFilterCaches.Output )
# Re-fill the holes left by the filter using region growing (with a mask)
self._opMaskedWatersheds = OperatorWrapper( OpMaskedWatershed, parent=self )
self._opMaskedWatersheds.Input.connect( self.InputData )
self._opMaskedWatersheds.Mask.connect( self._opSelectLabel.Output )
self._opMaskedWatersheds.Seeds.connect( self._opSmallLabelFilterCaches.Output )
# Cache is necessary because it ensures that the entire volume is used for watershed.
self._opMaskedWatershedCaches = OperatorWrapper( OpCompressedCache, parent=self )
self._opMaskedWatershedCaches.Input.connect( self._opMaskedWatersheds.Output )
self.HoleFilledSupervoxels.connect( self._opMaskedWatershedCaches.Output )
# Relabel the supervoxels in the mask to ensure contiguous supervoxels (after mask) and consecutive labels
self._opRelabelMergedSupervoxels = OperatorWrapper( OpVigraLabelVolume, parent=self )
self._opRelabelMergedSupervoxels.Input.connect( self._opMaskedWatershedCaches.Output )
self._opRelabeledMergedSupervoxelCaches = OperatorWrapper( OpCompressedCache, parent=self )
self._opRelabeledMergedSupervoxelCaches.Input.connect( self._opRelabelMergedSupervoxels.Output )
self.RelabeledSupervoxels.connect( self._opRelabeledMergedSupervoxelCaches.Output )
self._opAccumulateFinalImage = OpAccumulateFragmentSegmentations( parent=self )
self._opAccumulateFinalImage.RavelerLabels.connect( self.RavelerLabels )
self._opAccumulateFinalImage.FragmentSegmentations.connect( self._opRelabeledMergedSupervoxelCaches.Output )
self._opFinalCache = OpCompressedCache( parent=self )
self._opFinalCache.Input.connect( self._opAccumulateFinalImage.Output )
self.FinalSupervoxels.connect( self._opFinalCache.Output )
self.SupervoxelMapping.connect( self._opAccumulateFinalImage.Mapping )
def __init__(self, *args, **kwargs):
"""
Instantiate all internal operators and connect them together.
"""
super(OpLabeling, self).__init__(*args, **kwargs)
# Create internal operators
self.opInputShapeReader = OperatorWrapper(OpShapeReader,
parent=self,
graph=self.graph)
self.opLabelArray = OperatorWrapper(OpBlockedSparseLabelArray,
parent=self,
graph=self.graph)
# NOT wrapped
self.opMaxLabel = OpMaxValue(parent=self, graph=self.graph)
# Set up label cache shape input
self.opInputShapeReader.Input.connect(self.InputImages)
self.opLabelArray.inputs["shape"].connect(
self.opInputShapeReader.OutputShape)
# Set up other label cache inputs
self.LabelInputs.connect(self.InputImages)
self.opLabelArray.Input.connect(self.LabelInputs)
self.opLabelArray.eraser.connect(self.LabelEraserValue)
self.LabelEraserValue.setValue(255)
# Initialize the delete input to -1, which means "no label".
# Now changing this input to a positive value will cause label deletions.
# (The deleteLabel input is monitored for changes.)
self.opLabelArray.deleteLabel.connect(self.LabelDelete)
self.LabelDelete.setValue(-1)
# Find the highest label in all the label images
self.opMaxLabel.Inputs.connect(self.opLabelArray.outputs['maxLabel'])
# Connect our internal outputs to our external outputs
self.LabelImages.connect(self.opLabelArray.Output)
self.MaxLabelValue.connect(self.opMaxLabel.Output)
self.NonzeroLabelBlocks.connect(self.opLabelArray.nonzeroBlocks)
def inputResizeHandler(slot, oldsize, newsize):
if (newsize == 0):
self.LabelImages.resize(0)
self.NonzeroLabelBlocks.resize(0)
self.InputImages.notifyResized(inputResizeHandler)
# Check to make sure the non-wrapped operators stayed that way.
assert self.opMaxLabel.Inputs.operator == self.opMaxLabel
def handleNewInputImage(multislot, index, *args):
def handleInputReady(slot):
self.setupCaches(multislot.index(slot))
multislot[index].notifyReady(handleInputReady)
self.InputImages.notifyInserted(handleNewInputImage)
def __init__(self, *args, **kwargs):
"""
Instantiate the pipeline of internal operators and connect them together.
Most of the the operators we use here are designed to handle a single
input image and produce a single output image (slot level=0).
In those cases, we use the OperatorWrapper mechanism to dynamically manage
a list of these operators. (When wrapped, the operators have slots with level=1.)
(In ilastik, we use OpMultiLaneWrapper, which extends OperatorWrapper with extra functionality.)
"""
super(OpSimplePixelClassification, self).__init__(*args, **kwargs)
# SUMMARY SCHEMATIC:
#
# ClassifierFactory ---------------------------------
# \
# Labels ---> Wrapped(OpCompressedUserLabelArray) --> OpTrainClassifierBlocked --> OpValueCache -->
# / / \
# Features -- / Wrapped(OpClassifierPredict) --> Predictions
# \ / /
# Wrapped(OpBlockedArrayCache) --------------------------------------------------------
# LABEL CACHE(S)
# We are really just using this as a cache for label data, which is loaded 'manually' in ingest_labels (below).
# Therefore, none of these input slots is going to be used, but we need to configure them anyway,
# or else the operator won't be 'ready'.
self.opAllLabelCaches = OperatorWrapper( OpCompressedUserLabelArray, parent=self,
broadcastingSlotNames=['eraser', 'deleteLabel'] )
self.opAllLabelCaches.Input.connect( self.Labels )
self.opAllLabelCaches.deleteLabel.setValue( -1 )
self.opAllLabelCaches.eraser.setValue( 255 )
# FEATURE CACHE(S)
self.opAllFeatureCaches = OperatorWrapper( OpBlockedArrayCache, parent=self,
broadcastingSlotNames=['fixAtCurrent'] )
self.opAllFeatureCaches.fixAtCurrent.setValue(False) # Do not freeze caches
self.opAllFeatureCaches.Input.connect( self.Features )
# TRAINING OPERATOR
self.opTrain = OpTrainClassifierBlocked( parent=self )
self.opTrain.ClassifierFactory.connect( self.ClassifierFactory )
self.opTrain.Labels.connect( self.opAllLabelCaches.Output )
self.opTrain.Images.connect( self.opAllFeatureCaches.Output )
self.opTrain.nonzeroLabelBlocks.connect( self.opAllLabelCaches.nonzeroBlocks )
# CLASSIFIER CACHE
# This cache stores exactly one object: the classifier itself.
self.opClassifierCache = OpValueCache(parent=self)
self.opClassifierCache.Input.connect( self.opTrain.Classifier )
self.opClassifierCache.fixAtCurrent.setValue(False)
# PREDICTION OPERATOR(S)
self.opAllPredictors = OperatorWrapper( OpClassifierPredict, parent=self,
broadcastingSlotNames=['Classifier', 'LabelsCount'] )
self.opAllPredictors.Classifier.connect( self.opTrain.Classifier )
self.opAllPredictors.LabelsCount.connect( self.opTrain.MaxLabel )
self.opAllPredictors.Image.connect( self.opAllFeatureCaches.Output )
self.Predictions.connect( self.opAllPredictors.PMaps )
def _initBatchWorkflow(self):
"""
Connect the batch-mode top-level operators to the training workflow and to eachother.
"""
# Access applet operators from the training workflow
opTrainingFeatures = self.featureSelectionApplet.topLevelOperator
opClassify = self.pcApplet.topLevelOperator
# Access the batch operators
opBatchInputs = self.batchInputApplet.topLevelOperator
opBatchResults = self.batchResultsApplet.topLevelOperator
## Create additional batch workflow operators
opBatchFeatures = OperatorWrapper(
OpFeatureSelection,
operator_kwargs={'filter_implementation': 'Original'},
parent=self,
promotedSlotNames=['InputImage'])
opBatchPredictor = OperatorWrapper(OpAutocontextBatch,
parent=self,
promotedSlotNames=['FeatureImage'])
## Connect Operators ##
# Provide dataset paths from data selection applet to the batch export applet via an attribute selector
opBatchResults.DatasetPath.connect(opBatchInputs.ImageName)
# Connect (clone) the feature operator inputs from
# the interactive workflow's features operator (which gets them from the GUI)
opBatchFeatures.Scales.connect(opTrainingFeatures.Scales)
opBatchFeatures.FeatureIds.connect(opTrainingFeatures.FeatureIds)
opBatchFeatures.SelectionMatrix.connect(
opTrainingFeatures.SelectionMatrix)
# Classifier and LabelsCount are provided by the interactive workflow
opBatchPredictor.Classifiers.connect(opClassify.Classifiers)
opBatchPredictor.MaxLabelValue.connect(opClassify.MaxLabelValue)
# Sync autocontext contant
opBatchPredictor.AutocontextIterations.connect(
opClassify.AutocontextIterations)
# Connect Image pathway:
# Input Image -> Features Op -> Prediction Op -> Export
opBatchFeatures.InputImage.connect(opBatchInputs.Image)
opBatchPredictor.FeatureImage.connect(opBatchFeatures.OutputImage)
opBatchResults.ImageToExport.connect(
opBatchPredictor.PredictionProbabilities)
def testBasic3DWrongAxes(self):
"""Test if 3D file with intentionally wrong axes is rejected """
for fileName in self.imgFileNames3D:
graph = lazyflow.graph.Graph()
reader = OperatorWrapper(OpDataSelection, graph=graph)
reader.ProjectFile.setValue(self.projectFile)
reader.WorkingDirectory.setValue(os.getcwd())
reader.ProjectDataGroup.setValue('DataSelection/local_data')
info = DatasetInfo()
# Will be read from the filesystem since the data won't be found in the project file.
info.location = DatasetInfo.Location.ProjectInternal
info.filePath = fileName
info.internalPath = ""
info.invertColors = False
info.convertToGrayscale = False
info.axistags = vigra.defaultAxistags('tzyc')
try:
reader.Dataset.setValues([info])
assert False, "Should have thrown an exception!"
except DatasetConstraintError:
pass
except:
assert False, "Should have thrown a DatasetConstraintError!"
def testBasic3D(self):
"""Test if plane 2d files are loaded correctly"""
for fileName in self.imgFileNames3D:
graph = lazyflow.graph.Graph()
reader = OperatorWrapper(OpDataSelection, graph=graph)
reader.ProjectFile.setValue(self.projectFile)
reader.WorkingDirectory.setValue(os.getcwd())
reader.ProjectDataGroup.setValue('DataSelection/local_data')
info = DatasetInfo()
# Will be read from the filesystem since the data won't be found in the project file.
info.location = DatasetInfo.Location.ProjectInternal
info.filePath = fileName
info.internalPath = ""
info.invertColors = False
info.convertToGrayscale = False
reader.Dataset.setValues([info])
# Read the test files using the data selection operator and verify the contents
imgData3D = reader.Image[0][...].wait()
# Check the file name output
assert reader.ImageName[0].value == fileName
# Check raw images
assert imgData3D.shape == self.imgData3D.shape
# skip this if image was saved compressed:
numpy.testing.assert_array_equal(imgData3D, self.imgData3D)
def testBasic3DcStackFromGlobString(self, empty_project_file):
"""Test if stacked 2d 3-channel files are loaded correctly"""
# For some reason vigra saves 2D+c data compressed in gifs, so skip!
for fileName, nickname in zip(self.imgFileNameGlobs2Dc,
self.imgFileNameGlobs2DcNicknames):
reader = OperatorWrapper(OpDataSelection,
graph=Graph(),
operator_kwargs={"forceAxisOrder": False})
reader.WorkingDirectory.setValue(
str(Path(empty_project_file.filename).parent))
reader.Dataset.setValues(
[FilesystemDatasetInfo(filePath=fileName, sequence_axis="z")])
# Read the test files using the data selection operator and verify the contents
imgData3Dc = reader.Image[0][...].wait()
# Check the file name output
assert reader.ImageName[0].value == nickname
# Check raw images
assert imgData3Dc.shape == self.imgData3Dc.shape, (
imgData3Dc.shape, self.imgData3Dc.shape)
# skip this if image was saved compressed:
if any(x in fileName.lower()
for x in self.compressedExtensions + [".gif"]):
print("Skipping raw comparison for compressed data: {}".format(
fileName))
continue
numpy.testing.assert_array_equal(imgData3Dc, self.imgData3Dc)
def __init__(self, *args, **kwargs):
super(OpTrainVectorwiseClassifierBlocked, self).__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
self._opFeatureMatrixCaches = OperatorWrapper( OpFeatureMatrixCache, parent=self )
self._opFeatureMatrixCaches.LabelImage.connect( self.Labels )
self._opFeatureMatrixCaches.FeatureImage.connect( self.Images )
self._opConcatenateFeatureMatrices = OpConcatenateFeatureMatrices( parent=self )
self._opConcatenateFeatureMatrices.FeatureMatrices.connect( self._opFeatureMatrixCaches.LabelAndFeatureMatrix )
self._opConcatenateFeatureMatrices.ProgressSignals.connect( self._opFeatureMatrixCaches.ProgressSignal )
self._opTrainFromFeatures = OpTrainClassifierFromFeatureVectors( parent=self )
self._opTrainFromFeatures.ClassifierFactory.connect( self.ClassifierFactory )
self._opTrainFromFeatures.LabelAndFeatureMatrix.connect( self._opConcatenateFeatureMatrices.ConcatenatedOutput )
self._opTrainFromFeatures.MaxLabel.connect( self.MaxLabel )
self.Classifier.connect( self._opTrainFromFeatures.Classifier )
# Progress reporting
def _handleFeatureProgress( progress ):
# Note that these progress messages will probably appear out-of-order.
# See comments in OpFeatureMatrixCache
logger.debug("Training: {:02}% (Computing features)".format(int(progress)))
self.progressSignal( 0.8*progress )
self._opConcatenateFeatureMatrices.progressSignal.subscribe( _handleFeatureProgress )
def _handleTrainingComplete():
logger.debug("Training: 100% (Complete)")
self.progressSignal( 100.0 )
self._opTrainFromFeatures.trainingCompleteSignal.subscribe( _handleTrainingComplete )
def prepare_node_cluster_operator(config, cluster_args, finalOutputSlot):
# We're doing node work
opClusterTaskWorker = OperatorWrapper( OpTaskWorker, parent=finalOutputSlot.getRealOperator().parent )
# TODO: Raise an error if finalOutputSlot has len=0. That means the user didn't load a batch dataset into the project.
# FIXME: Image index is hard-coded as 0. We assume we are working with only one (big) dataset in cluster mode.
opClusterTaskWorker.Input.connect( finalOutputSlot )
opClusterTaskWorker.RoiString[0].setValue( cluster_args._node_work_ )
opClusterTaskWorker.TaskName.setValue( cluster_args.process_name )
opClusterTaskWorker.ConfigFilePath.setValue( cluster_args.option_config_file )
opClusterTaskWorker.OutputFilesetDescription.setValue( cluster_args.output_description_file )
# If we have a way to report task progress (e.g. by updating the job name),
# then subscribe to progress signals
if config.task_progress_update_command is not None:
def report_progress( progress ):
cmd = config.task_progress_update_command.format( progress=int(progress) )
def shell_call(shell_cmd):
logger.debug( "Executing progress command: " + cmd )
subprocess.call( shell_cmd, shell=True )
background_tasks.put( functools.partial( shell_call, cmd ) )
opClusterTaskWorker.innerOperators[0].progressSignal.subscribe( report_progress )
resultSlot = opClusterTaskWorker.ReturnCode
clusterOperator = opClusterTaskWorker
return (clusterOperator, resultSlot)
def __init__(self,
graph,
title,
projectFileGroupName,
supportIlastik05Import=False,
batchDataGui=False):
super(DataSelectionApplet, self).__init__(title)
# Our top-level operator is wrapped to enable multi-image support.
# All inputs are common to all inputs except for the 'Dataset' input, which is unique for each image.
# Hence, 'Dataset' is the only 'promoted' slot.
self._topLevelOperator = OperatorWrapper(OpDataSelection,
graph=graph,
promotedSlotNames=set(
['Dataset']))
self._serializableItems = [
DataSelectionSerializer(self._topLevelOperator,
projectFileGroupName)
]
if supportIlastik05Import:
self._serializableItems.append(
Ilastik05DataSelectionDeserializer(self._topLevelOperator))
self._gui = None
self.batchDataGui = batchDataGui
self._preferencesManager = DataSelectionPreferencesManager()
def testBasic3DcStackFromGlobString(self):
"""Test if stacked 2d 3-channel files are loaded correctly"""
# For some reason vigra saves 2D+c data compressed in gifs, so skip!
self.compressedExtensions.append('.gif')
for fileName in self.imgFileNameGlobs2Dc:
graph = lazyflow.graph.Graph()
reader = OperatorWrapper(OpDataSelection, graph=graph)
reader.ProjectFile.setValue(self.projectFile)
reader.WorkingDirectory.setValue(os.getcwd())
reader.ProjectDataGroup.setValue('DataSelection/local_data')
info = DatasetInfo()
# Will be read from the filesystem since the data won't be found in the project file.
info.location = DatasetInfo.Location.ProjectInternal
info.filePath = fileName
info.internalPath = ""
info.invertColors = False
info.convertToGrayscale = False
reader.Dataset.setValues([info])
# Read the test files using the data selection operator and verify the contents
imgData3Dc = reader.Image[0][...].wait()
# Check the file name output
assert reader.ImageName[0].value == fileName
# Check raw images
assert imgData3Dc.shape == self.imgData3Dc.shape
# skip this if image was saved compressed:
if any(x in fileName.lower() for x in self.compressedExtensions):
print("Skipping raw comparison for compressed data: {}".format(
fileName))
continue
numpy.testing.assert_array_equal(imgData3Dc, self.imgData3Dc)
def test_load_single_file_with_list(self):
graph = lazyflow.graph.Graph()
reader = OperatorWrapper(OpDataSelection,
graph=graph,
operator_kwargs={'forceAxisOrder': False})
reader.ProjectFile.setValue(self.projectFile)
reader.WorkingDirectory.setValue(os.getcwd())
reader.ProjectDataGroup.setValue('DataSelection/local_data')
fileNameString = os.path.pathsep.join(self.file_names)
info = DatasetInfo(filepath=fileNameString)
# Will be read from the filesystem since the data won't be found in the project file.
info.location = DatasetInfo.Location.ProjectInternal
info.internalPath = ""
info.invertColors = False
info.convertToGrayscale = False
reader.Dataset.setValues([info])
# Read the test files using the data selection operator and verify the contents
imgData = reader.Image[0][...].wait()
print('imgData', reader.Image.meta.axistags,
reader.Image.meta.original_axistags)
# Check raw images
assert imgData.shape == self.imgData3Dct.shape, (
imgData.shape, self.imgData3Dct.shape)
numpy.testing.assert_array_equal(imgData, self.imgData3Dct)
def testBasic2D(self):
"""Test if plane 2d files are loaded correctly"""
for fileName in self.imgFileNames2D:
graph = lazyflow.graph.Graph()
reader = OperatorWrapper(OpDataSelection,
graph=graph,
operator_kwargs={'forceAxisOrder': False})
reader.ProjectFile.setValue(self.projectFile)
reader.WorkingDirectory.setValue(os.getcwd())
reader.ProjectDataGroup.setValue('DataSelection/local_data')
info = DatasetInfo()
# Will be read from the filesystem since the data won't be found in the project file.
info.location = DatasetInfo.Location.ProjectInternal
info.filePath = fileName
info.internalPath = ""
info.invertColors = False
info.convertToGrayscale = False
reader.Dataset.setValues([info])
# Read the test files using the data selection operator and verify the contents
imgData2D = reader.Image[0][...].wait()
# Check the file name output
assert reader.ImageName[0].value == fileName
# Check raw images
assert imgData2D.shape == self.imgData2D.shape
# skip this if image was saved compressed:
if any(x in fileName.lower() for x in self.compressedExtensions):
print("Skipping raw comparison for compressed data: {}".format(
fileName))
continue
numpy.testing.assert_array_equal(imgData2D, self.imgData2D)
def _initBatchWorkflow(self):
# Access applet operators from the training workflow
opTrainingTopLevel = self.objectClassificationApplet.topLevelOperator
opBlockwiseObjectClassification = self.blockwiseObjectClassificationApplet.topLevelOperator
# Access the batch INPUT applets
opRawBatchInput = self.rawBatchInputApplet.topLevelOperator
opBinaryBatchInput = self.binaryBatchInputApplet.topLevelOperator
# Connect the blockwise classification operator
opBatchClassify = OperatorWrapper(OpBlockwiseObjectClassification,
parent=self)
opBatchClassify.RawImage.connect(opRawBatchInput.Image)
opBatchClassify.BinaryImage.connect(opBinaryBatchInput.Image)
opBatchClassify.Classifier.connect(opTrainingTopLevel.Classifier)
opBatchClassify.LabelsCount.connect(opTrainingTopLevel.LabelsCount)
opBatchClassify.BlockShape3dDict.connect(
opBlockwiseObjectClassification.BlockShape3dDict)
opBatchClassify.HaloPadding3dDict.connect(
opBlockwiseObjectClassification.HaloPadding3dDict)
self.opBatchClassify = opBatchClassify
# Connect the batch OUTPUT applet
opBatchOutput = self.batchResultsApplet.topLevelOperator
opBatchOutput.DatasetPath.connect(opRawBatchInput.ImageName)
opBatchOutput.RawImage.connect(opRawBatchInput.Image)
opBatchOutput.ImageToExport.connect(opBatchClassify.PredictionImage)
def testBasic2Dc(self):
"""Test if 2d 3-channel files are loaded correctly"""
# For some reason vigra saves 2D+c data compressed in gifs, so skip!
self.compressedExtensions.append(".gif")
for fileName in self.imgFileNames2Dc:
graph = lazyflow.graph.Graph()
reader = OperatorWrapper(OpDataSelection, graph=graph, operator_kwargs={"forceAxisOrder": False})
reader.ProjectFile.setValue(self.projectFile)
reader.WorkingDirectory.setValue(os.getcwd())
info = FilesystemDatasetInfo(filePath=fileName)
reader.Dataset.setValues([info])
# Read the test files using the data selection operator and verify the contents
imgData2Dc = reader.Image[0][...].wait()
# Check the file name output
assert reader.ImageName[0].value == self.create_nickname(fileName)
# Check raw images
assert imgData2Dc.shape == self.imgData2Dc.shape, (imgData2Dc.shape, self.imgData2Dc.shape)
# skip this if image was saved compressed:
if any(x in fileName.lower() for x in self.compressedExtensions):
print("Skipping raw comparison for compressed data: {}".format(fileName))
continue
numpy.testing.assert_array_equal(imgData2Dc, self.imgData2Dc)
def setupOutputs(self):
# Create internal operators
if self.DatasetRoles.value != self._roles:
self._roles = self.DatasetRoles.value
# Clean up the old operators
self.ImageGroup.disconnect()
self.Image.disconnect()
self.Image1.disconnect()
self.Image2.disconnect()
self._NonTransposedImageGroup.disconnect()
if self._opDatasets is not None:
self._opDatasets.cleanUp()
self._opDatasets = OperatorWrapper(
OpDataSelection,
parent=self,
operator_kwargs={"forceAxisOrder": self._forceAxisOrder},
broadcastingSlotNames=[
"ProjectFile", "ProjectDataGroup", "WorkingDirectory"
],
)
self.ImageGroup.connect(self._opDatasets.Image)
self._NonTransposedImageGroup.connect(
self._opDatasets._NonTransposedImage)
self._opDatasets.Dataset.connect(self.DatasetGroup)
self._opDatasets.ProjectFile.connect(self.ProjectFile)
self._opDatasets.ProjectDataGroup.connect(self.ProjectDataGroup)
self._opDatasets.WorkingDirectory.connect(self.WorkingDirectory)
for role_index, opDataSelection in enumerate(self._opDatasets):
opDataSelection.RoleName.setValue(self._roles[role_index])
if len(self._opDatasets.Image) > 0:
self.Image.connect(self._opDatasets.Image[0])
if len(self._opDatasets.Image) >= 2:
self.Image1.connect(self._opDatasets.Image[1])
else:
self.Image1.disconnect()
self.Image1.meta.NOTREADY = True
if len(self._opDatasets.Image) >= 3:
self.Image2.connect(self._opDatasets.Image[2])
else:
self.Image2.disconnect()
self.Image2.meta.NOTREADY = True
self.ImageName.connect(self._opDatasets.ImageName[0])
self.AllowLabels.connect(self._opDatasets.AllowLabels[0])
else:
self.Image.disconnect()
self.Image1.disconnect()
self.Image2.disconnect()
self.ImageName.disconnect()
self.AllowLabels.disconnect()
self.Image.meta.NOTREADY = True
self.Image1.meta.NOTREADY = True
self.Image2.meta.NOTREADY = True
self.ImageName.meta.NOTREADY = True
self.AllowLabels.meta.NOTREADY = True
def __init__(self, carvingGraphFile):
super(CarvingWorkflow, self).__init__()
self._applets = []
graph = Graph()
self._graph = graph
## Create applets
self.projectMetadataApplet = ProjectMetadataApplet()
self.dataSelectionApplet = DataSelectionApplet(graph, "Input Data", "Input Data", supportIlastik05Import=True, batchDataGui=False)
self.carvingApplet = CarvingApplet(graph, "xxx", carvingGraphFile)
self.carvingApplet.topLevelOperator.RawData.connect( self.dataSelectionApplet.topLevelOperator.Image )
self.carvingApplet.topLevelOperator.opLabeling.LabelsAllowedFlags.connect( self.dataSelectionApplet.topLevelOperator.AllowLabels )
self.carvingApplet.gui.minLabelNumber = 2
self.carvingApplet.gui.maxLabelNumber = 2
## Access applet operators
opData = self.dataSelectionApplet.topLevelOperator
## Connect operators ##
self._applets.append(self.projectMetadataApplet)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.carvingApplet)
# The shell needs a slot from which he can read the list of image names to switch between.
# Use an OpAttributeSelector to create a slot containing just the filename from the OpDataSelection's DatasetInfo slot.
opSelectFilename = OperatorWrapper( OpAttributeSelector, graph=graph )
opSelectFilename.InputObject.connect( opData.Dataset )
opSelectFilename.AttributeName.setValue( 'filePath' )
self._imageNameListSlot = opSelectFilename.Result
def testProjectLocalData(self):
graph = lazyflow.graph.Graph()
reader = OperatorWrapper(OpDataSelection, graph=graph)
reader.ProjectFile.setValue(self.projectFile)
reader.WorkingDirectory.setValue(os.getcwd())
reader.ProjectDataGroup.setValue('DataSelection/local_data')
# Create a list of dataset infos . . .
datasetInfos = []
# From project
info = DatasetInfo()
info.location = DatasetInfo.Location.ProjectInternal
info.filePath = "This string should be ignored..."
info._datasetId = 'dataset1' # (Cheating a bit here...)
info.invertColors = False
info.convertToGrayscale = False
datasetInfos.append(info)
reader.Dataset.setValues(datasetInfos)
projectInternalData = reader.Image[0][...].wait()
assert projectInternalData.shape == self.imgData3Dc.shape
assert (projectInternalData == self.imgData3Dc).all()
def testBasic3DstacksFromFileList(self):
for ext, fileNames in self.imgFileLists2D.items():
fileNameString = os.path.pathsep.join(fileNames)
graph = lazyflow.graph.Graph()
reader = OperatorWrapper(OpDataSelection, graph=graph)
reader.ProjectFile.setValue(self.projectFile)
reader.WorkingDirectory.setValue(os.getcwd())
reader.ProjectDataGroup.setValue('DataSelection/local_data')
info = DatasetInfo(filepath=fileNameString)
# Will be read from the filesystem since the data won't be found in the project file.
info.location = DatasetInfo.Location.ProjectInternal
info.internalPath = ""
info.invertColors = False
info.convertToGrayscale = False
reader.Dataset.setValues([info])
# Read the test files using the data selection operator and verify the contents
imgData3D = reader.Image[0][...].wait()
# Check raw images
assert imgData3D.shape == self.imgData3D.shape
# skip this if image was saved compressed:
if any(
x.strip('.') in ext.lower()
for x in self.compressedExtensions):
print("Skipping raw comparison for compressed data: {}".format(
ext))
continue
numpy.testing.assert_array_equal(imgData3D, self.imgData3D)
def __init__(self):
super(VigraWatershedWorkflow, self).__init__()
self._applets = []
# Create a graph to be shared by all operators
graph = Graph()
self._graph = graph
# Create applets
self.dataSelectionApplet = DataSelectionApplet(
graph,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False)
self.watershedApplet = VigraWatershedViewerApplet(
graph, "Watershed", "Watershed")
# Connect top-level operators
self.watershedApplet.topLevelOperator.InputImage.connect(
self.dataSelectionApplet.topLevelOperator.Image)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.watershedApplet)
# The shell needs a slot from which he can read the list of image names to switch between.
# Use an OpAttributeSelector to create a slot containing just the filename from the OpDataSelection's DatasetInfo slot.
opSelectFilename = OperatorWrapper(OpAttributeSelector, graph=graph)
opSelectFilename.InputObject.connect(
self.dataSelectionApplet.topLevelOperator.Dataset)
opSelectFilename.AttributeName.setValue('filePath')
self._imageNameListSlot = opSelectFilename.Result
def __init__(self, *args, **kwargs):
super(OpTrainVectorwiseClassifierBlocked, self).__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
self._opFeatureMatrixCaches = OperatorWrapper( OpFeatureMatrixCache, parent=self )
self._opFeatureMatrixCaches.LabelImage.connect( self.Labels )
self._opFeatureMatrixCaches.FeatureImage.connect( self.Images )
self._opFeatureMatrixCaches.NonZeroLabelBlocks.connect( self.nonzeroLabelBlocks )
self._opConcatenateFeatureMatrices = OpConcatenateFeatureMatrices( parent=self )
self._opConcatenateFeatureMatrices.FeatureMatrices.connect( self._opFeatureMatrixCaches.LabelAndFeatureMatrix )
self._opConcatenateFeatureMatrices.ProgressSignals.connect( self._opFeatureMatrixCaches.ProgressSignal )
self._opTrainFromFeatures = OpTrainClassifierFromFeatureVectors( parent=self )
self._opTrainFromFeatures.ClassifierFactory.connect( self.ClassifierFactory )
self._opTrainFromFeatures.LabelAndFeatureMatrix.connect( self._opConcatenateFeatureMatrices.ConcatenatedOutput )
self._opTrainFromFeatures.MaxLabel.connect( self.MaxLabel )
self.Classifier.connect( self._opTrainFromFeatures.Classifier )
# Progress reporting
def _handleFeatureProgress( progress ):
self.progressSignal( 0.8*progress )
self._opConcatenateFeatureMatrices.progressSignal.subscribe( _handleFeatureProgress )
def _handleTrainingComplete():
self.progressSignal( 100.0 )
self._opTrainFromFeatures.trainingCompleteSignal.subscribe( _handleTrainingComplete )
def test3DProjectLocalData(self, serializer, empty_project_file):
empty_project_file.create_group("DataSelection")
empty_project_file["DataSelection"].create_group("local_data")
empty_project_file["DataSelection/local_data"].create_dataset(
"dataset1", data=self.imgData3Dc)
reader = OperatorWrapper(OpDataSelection,
graph=Graph(),
operator_kwargs={"forceAxisOrder": False})
reader.WorkingDirectory.setValue(
str(Path(empty_project_file.filename).parent))
info = ProjectInternalDatasetInfo(
inner_path="DataSelection/local_data/dataset1",
project_file=empty_project_file)
reader.Dataset.setValues([info])
projectInternalData = reader.Image[0][...].wait()
assert projectInternalData.shape == self.imgData3Dc.shape, (
projectInternalData.shape, self.imgData3Dc.shape)
assert (projectInternalData == self.imgData3Dc).all()
for fileName in self.generatedImages3Dc:
inner_path = serializer.importStackAsLocalDataset([fileName])
info = ProjectInternalDatasetInfo(project_file=empty_project_file,
inner_path=inner_path)
reader.Dataset.setValues([info])
projectInternalData = reader.Image[0][...].wait()
assert projectInternalData.shape == self.imgData3Dc.shape, (
projectInternalData.shape,
self.imgData3Dc.shape,
)
assert (projectInternalData == self.imgData3Dc).all()
def testNonReady(self):
n = self.vol.shape[2]
op = OpMultiArrayStacker(graph=self.g)
op.AxisFlag.setValue('z')
op.AxisIndex.setValue(0)
providers = [OpNonReady(graph=self.g), OpNonReady(graph=self.g)]
provider = OperatorWrapper(OpArrayPiper, graph=self.g)
provider.Input.resize(n)
vol = self.vol
op.Images.resize(n)
for i in range(n):
provider.Input[i].setValue(vol[..., i])
providers[i].Input.connect(provider.Output[i])
op.Images[i].connect(providers[i].Output)
req = op.Output[...]
req.notify_failed(
lambda *args: None
) # Replace the default handler: dont' show a traceback
out = req.wait()
with self.assertRaises(InputSlot.SlotNotReadyError):
providers[0].screwWithOutput()
req = op.Output[...]
req.notify_failed(
lambda *args: None
) # Replace the default handler: dont' show a traceback
out = req.wait()
def test_partialWrapping(self):
"""
By default, OperatorWrapper promotes all slots.
This function tests what happens when only a subset of the inputs are promoted.
"""
wrapped = OperatorWrapper( OpSimple, graph=self.graph, promotedSlotNames=set(['InputA']) )
assert type(wrapped.InputA) == InputSlot
assert type(wrapped.InputB) == InputSlot
assert type(wrapped.Output) == OutputSlot
assert wrapped.InputA.level == 1 # Promoted because it was listed in the constructor call
assert wrapped.InputB.level == 0 # NOT promoted
assert wrapped.Output.level == 1 # Promoted because it's an output
assert len(wrapped.InputA) == 0
assert len(wrapped.InputB) == 0
assert len(wrapped.Output) == 0
wrapped.InputA.resize(2)
assert len(wrapped.InputB) == 0 # Not promoted
assert len(wrapped.Output) == 2
a = numpy.array([[1,2],[3,4]])
b = numpy.array([2])
wrapped.InputA[0].setValue(a)
wrapped.InputB.setValue(b)
wrapped.InputA[1].setValue(2*a)
result0 = wrapped.Output[0][0:2,0:2].wait()
result1 = wrapped.Output[1][0:2,0:2].wait()
assert ( result0 == a * b[0] ).all()
assert ( result1 == 2*a * b[0] ).all()
def testBasic3Dc(self):
"""Test if 2d 3-channel files are loaded correctly"""
# For some reason vigra saves 2D+c data compressed in gifs, so skip!
for fileName, nickname in zip(self.imgFileNames3Dc,
self.imgFileNames3DcNicknames):
graph = lazyflow.graph.Graph()
reader = OperatorWrapper(OpDataSelection,
graph=graph,
operator_kwargs={"forceAxisOrder": False})
reader.ProjectFile.setValue(self.projectFile)
reader.WorkingDirectory.setValue(os.getcwd())
reader.ProjectDataGroup.setValue("DataSelection/local_data")
reader.Dataset.setValues(
[FilesystemDatasetInfo(filePath=fileName)])
# Read the test files using the data selection operator and verify the contents
imgData3Dc = reader.Image[0][...].wait()
# Check the file name output
assert reader.ImageName[0].value == nickname
# Check raw images
assert imgData3Dc.shape == self.imgData3Dc.shape, (
imgData3Dc.shape, self.imgData3Dc.shape)
# skip this if image was saved compressed:
numpy.testing.assert_array_equal(imgData3Dc, self.imgData3Dc)
def test_fullWrapping(self):
"""
Test basic wrapping functionality (all slots are promoted)
"""
wrapped = OperatorWrapper( OpSimple, graph=self.graph )
assert type(wrapped.InputA) == InputSlot
assert type(wrapped.InputB) == InputSlot
assert type(wrapped.Output) == OutputSlot
assert wrapped.InputA.level == 1
assert wrapped.InputB.level == 1
assert wrapped.Output.level == 1
assert len(wrapped.InputA) == 0
assert len(wrapped.InputB) == 0
assert len(wrapped.Output) == 0
wrapped.InputA.resize(2)
assert len(wrapped.InputB) == 2
assert len(wrapped.Output) == 2
a = numpy.array([[1,2],[3,4]])
b = numpy.array([2])
wrapped.InputA[0].setValue(a)
wrapped.InputB[0].setValue(b)
wrapped.InputA[1].setValue(2*a)
wrapped.InputB[1].setValue(3*b)
result0 = wrapped.Output[0][0:2,0:2].wait()
result1 = wrapped.Output[1][0:2,0:2].wait()
assert ( result0 == a * b[0] ).all()
assert ( result1 == 2*a * 3*b[0] ).all()
def __init__(self, *args, **kwargs):
super(OpRegionFeatures, self).__init__(*args, **kwargs)
# Distribute the raw data
self.opRawTimeSlicer = OpMultiArraySlicer2(parent=self)
self.opRawTimeSlicer.name = 'OpRegionFeatures.opRawTimeSlicer'
self.opRawTimeSlicer.AxisFlag.setValue('t')
self.opRawTimeSlicer.Input.connect(self.RawImage)
assert self.opRawTimeSlicer.Slices.level == 1
# Distribute the labels
self.opLabelTimeSlicer = OpMultiArraySlicer2(parent=self)
self.opLabelTimeSlicer.name = 'OpRegionFeatures.opLabelTimeSlicer'
self.opLabelTimeSlicer.AxisFlag.setValue('t')
self.opLabelTimeSlicer.Input.connect(self.LabelImage)
assert self.opLabelTimeSlicer.Slices.level == 1
self.opRegionFeatures3dBlocks = OperatorWrapper(OpRegionFeatures3d, operator_args=[], parent=self)
assert self.opRegionFeatures3dBlocks.RawVolume.level == 1
assert self.opRegionFeatures3dBlocks.LabelVolume.level == 1
self.opRegionFeatures3dBlocks.RawVolume.connect(self.opRawTimeSlicer.Slices)
self.opRegionFeatures3dBlocks.LabelVolume.connect(self.opLabelTimeSlicer.Slices)
self.opRegionFeatures3dBlocks.Features.connect(self.Features)
assert self.opRegionFeatures3dBlocks.Output.level == 1
self.opTimeStacker = OpMultiArrayStacker(parent=self)
self.opTimeStacker.name = 'OpRegionFeatures.opTimeStacker'
self.opTimeStacker.AxisFlag.setValue('t')
assert self.opTimeStacker.Images.level == 1
self.opTimeStacker.Images.connect(self.opRegionFeatures3dBlocks.Output)
# Connect our outputs
self.Output.connect(self.opTimeStacker.Output)
def testBasic3DstacksFromFileList(self, empty_project_file):
for ext, fileNames in list(self.imgFileLists2D.items()):
fileNameString = os.path.pathsep.join(fileNames)
reader = OperatorWrapper(OpDataSelection,
graph=Graph(),
operator_kwargs={"forceAxisOrder": False})
reader.WorkingDirectory.setValue(
str(Path(empty_project_file.filename).parent))
reader.Dataset.setValues([
FilesystemDatasetInfo(filePath=fileNameString,
sequence_axis="z")
])
# Read the test files using the data selection operator and verify the contents
imgData3D = reader.Image[0][...].wait()
# Check raw images
assert imgData3D.shape == self.imgData3D.shape, (
imgData3D.shape, self.imgData3D.shape)
# skip this if image was saved compressed:
if any(
x.strip(".") in ext.lower()
for x in self.compressedExtensions):
print("Skipping raw comparison for compressed data: {}".format(
ext))
continue
numpy.testing.assert_array_equal(imgData3D, self.imgData3D)
