def __init__(self,
shell,
headless,
workflow_cmdline_args,
project_creation_args,
appendBatchOperators=True,
*args,
**kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super(AutocontextClassificationWorkflow,
self).__init__(shell,
headless,
workflow_cmdline_args,
project_creation_args,
graph=graph,
*args,
**kwargs)
self._applets = []
## Create applets
self.projectMetadataApplet = ProjectMetadataApplet()
self.dataSelectionApplet = DataSelectionApplet(
self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False)
self.featureSelectionApplet = FeatureSelectionApplet(
self, "Feature Selection", "FeatureSelections")
self.pcApplet = AutocontextClassificationApplet(
self, "PixelClassification")
# Autocontext constant
opClassifyTopLevel = self.pcApplet.topLevelOperator
opClassifyTopLevel.AutocontextIterations.setValue(2)
# Expose for shell
self._applets.append(self.projectMetadataApplet)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.featureSelectionApplet)
self._applets.append(self.pcApplet)
if appendBatchOperators:
# Create applets for batch workflow
self.batchInputApplet = DataSelectionApplet(
self,
"Batch Prediction Input Selections",
"BatchDataSelection",
supportIlastik05Import=False,
batchDataGui=True)
self.batchResultsApplet = BatchIoApplet(
self, "Batch Prediction Output Locations")
# Expose in shell
self._applets.append(self.batchInputApplet)
self._applets.append(self.batchResultsApplet)
# Connect batch workflow (NOT lane-based)
self._initBatchWorkflow()
def __init__(self,
headless=False,
workflow_cmdline_args=(),
parent=None,
graph=None):
# Create a graph to be shared by all operators
graph = Graph()
super(LabelImageViewerWorkflow,
self).__init__(headless=headless,
workflow_cmdline_args=workflow_cmdline_args,
parent=parent,
graph=graph)
self._applets = []
# Two data selection applets...
self.rawDataSelectionApplet = DataSelectionApplet(
self,
"Raw Data",
"Raw Data",
supportIlastik05Import=False,
batchDataGui=False)
self.labelDataSelectionApplet = DataSelectionApplet(
self,
"Label Image",
"Label Image",
supportIlastik05Import=False,
batchDataGui=False)
self.viewerApplet = LabelImageViewerApplet(self)
# Expose for shell
self._applets.append(self.rawDataSelectionApplet)
self._applets.append(self.labelDataSelectionApplet)
self._applets.append(self.viewerApplet)
def __init__(self, shell, headless, workflow_cmdline_args, project_creation_args, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super(DataConversionWorkflow, self).__init__(
shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs
)
self._applets = []
# Instantiate DataSelection applet
self.dataSelectionApplet = DataSelectionApplet(
self, "Input Data", "Input Data", supportIlastik05Import=True, forceAxisOrder=None
)
# Configure global DataSelection settings
role_names = ["Input Data"]
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue(role_names)
# Instantiate DataExport applet
self.dataExportApplet = DataExportApplet(self, "Data Export")
# Configure global DataExport settings
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.WorkingDirectory.connect(opDataSelection.WorkingDirectory)
opDataExport.SelectionNames.setValue(["Input"])
# No special data pre/post processing necessary in this workflow,
# but this is where we'd hook it up if we needed it.
#
# self.dataExportApplet.prepare_for_entire_export = self.prepare_for_entire_export
# self.dataExportApplet.prepare_lane_for_export = self.prepare_lane_for_export
# self.dataExportApplet.post_process_lane_export = self.post_process_lane_export
# self.dataExportApplet.post_process_entire_export = self.post_process_entire_export
# Instantiate BatchProcessing applet
self.batchProcessingApplet = BatchProcessingApplet(
self, "Batch Processing", self.dataSelectionApplet, self.dataExportApplet
)
# Expose our applets in a list (for the shell to use)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.dataExportApplet)
self._applets.append(self.batchProcessingApplet)
# Parse command-line arguments
# Command-line args are applied in onProjectLoaded(), below.
if workflow_cmdline_args:
self._data_export_args, unused_args = self.dataExportApplet.parse_known_cmdline_args(workflow_cmdline_args)
self._batch_input_args, unused_args = self.dataSelectionApplet.parse_known_cmdline_args(
unused_args, role_names
)
else:
unused_args = None
self._batch_input_args = None
self._data_export_args = None
if unused_args:
logger.warning("Unused command-line args: {}".format(unused_args))
def __init__(self,
headless,
workflow_cmdline_args,
appendBatchOperators=True,
*args,
**kwargs):
graph = kwargs['graph'] if 'graph' in kwargs else Graph()
if 'graph' in kwargs: del kwargs['graph']
super(CountingWorkflow, self).__init__(headless,
graph=graph,
*args,
**kwargs)
######################
# Interactive workflow
######################
self.projectMetadataApplet = ProjectMetadataApplet()
self.dataSelectionApplet = DataSelectionApplet(self,
"Data Selection",
"DataSelection",
batchDataGui=False,
force5d=False)
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue(['Raw Data'])
self.featureSelectionApplet = FeatureSelectionApplet(
self, "Feature Selection", "FeatureSelections")
#self.pcApplet = PixelClassificationApplet(self, "PixelClassification")
self.countingApplet = CountingApplet(workflow=self)
self._applets = []
self._applets.append(self.projectMetadataApplet)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.featureSelectionApplet)
self._applets.append(self.countingApplet)
if appendBatchOperators:
# Create applets for batch workflow
self.batchInputApplet = DataSelectionApplet(
self,
"Batch Prediction Input Selections",
"BatchDataSelection",
supportIlastik05Import=False,
batchDataGui=True)
self.batchResultsApplet = CountingBatchResultsApplet(
self, "Batch Prediction Output Locations")
# Expose in shell
self._applets.append(self.batchInputApplet)
self._applets.append(self.batchResultsApplet)
# Connect batch workflow (NOT lane-based)
self._initBatchWorkflow()
def __init__(self, shell, headless, workflow_cmdline_args,
project_creation_args, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super(DataConversionWorkflow, self).__init__(shell,
headless,
workflow_cmdline_args,
project_creation_args,
graph=graph,
*args,
**kwargs)
self._applets = []
# Create applets
self.dataSelectionApplet = DataSelectionApplet(
self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False,
force5d=False)
opDataSelection = self.dataSelectionApplet.topLevelOperator
role_names = ["Input Data"]
opDataSelection.DatasetRoles.setValue(role_names)
self.dataExportApplet = DataExportApplet(self, "Data Export")
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.WorkingDirectory.connect(opDataSelection.WorkingDirectory)
opDataExport.SelectionNames.setValue(["Input"])
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.dataExportApplet)
# Parse command-line arguments
# Command-line args are applied in onProjectLoaded(), below.
self._workflow_cmdline_args = workflow_cmdline_args
self._data_input_args = None
self._data_export_args = None
if workflow_cmdline_args:
self._data_export_args, unused_args = self.dataExportApplet.parse_known_cmdline_args(
unused_args)
self._data_input_args, unused_args = self.dataSelectionApplet.parse_known_cmdline_args(
workflow_cmdline_args, role_names)
if unused_args:
logger.warn("Unused command-line args: {}".format(unused_args))
def __init__(self, headless, workflow_cmdline_args, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super(DataConversionWorkflow, self).__init__(headless,
graph=graph,
*args,
**kwargs)
self._applets = []
# Create applets
self.dataSelectionApplet = DataSelectionApplet(
self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False,
force5d=False)
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue(["Input Data"])
self.dataExportApplet = DataExportApplet(self, "Data Export")
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.WorkingDirectory.connect(opDataSelection.WorkingDirectory)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.dataExportApplet)
self._workflow_cmdline_args = workflow_cmdline_args
def __init__(self, shell, headless, workflow_cmdline_args,
project_creation_args):
# Create a graph to be shared by all operators
graph = Graph()
super(ThresholdMaskingWorkflow, self).__init__(shell,
headless,
workflow_cmdline_args,
project_creation_args,
graph=graph)
self._applets = []
# Create applets
self.dataSelectionApplet = DataSelectionApplet(
self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False)
self.thresholdMaskingApplet = ThresholdMaskingApplet(
self, "Thresholding", "Thresholding Stage 1")
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue(self.ROLE_NAMES)
# Instantiate DataExport applet
self.dataExportApplet = DataExportApplet(self, "Data Export")
# Configure global DataExport settings
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.WorkingDirectory.connect(opDataSelection.WorkingDirectory)
opDataExport.SelectionNames.setValue(self.EXPORT_NAMES)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.thresholdMaskingApplet)
self._applets.append(self.dataExportApplet)
def __init__(self, shell, headless, workflow_cmdline_args,
project_creation_args, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super(LayerViewerWorkflow, self).__init__(shell,
headless,
workflow_cmdline_args,
project_creation_args,
graph=graph,
*args,
**kwargs)
self._applets = []
# Create applets
self.dataSelectionApplet = DataSelectionApplet(
self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
forceAxisOrder='tzyxc')
self.viewerApplet = LayerViewerApplet(self)
self.dataExportApplet = DataExportApplet(self, "Data Export")
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.SelectionNames.setValue(['Raw Data', 'Other Data'])
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue(["Raw Data", "Other Data"])
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.viewerApplet)
self._applets.append(self.dataExportApplet)
self._workflow_cmdline_args = workflow_cmdline_args
def __init__(self, shell, headless, workflow_cmdline_args,
project_creation_workflow, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super(VigraWatershedWorkflow, self).__init__(shell,
headless,
workflow_cmdline_args,
project_creation_workflow,
graph=graph,
*args,
**kwargs)
self._applets = []
# Create applets
self.dataSelectionApplet = DataSelectionApplet(
self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False)
self.watershedApplet = VigraWatershedViewerApplet(
self, "Watershed", "Watershed")
# Dataset inputs
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue(['Raw Data'])
# Expose to shell
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.watershedApplet)
def __init__(self, carvingGraphFile):
super(CarvingWorkflow, self).__init__()
self._applets = []
graph = Graph()
self._graph = graph
## Create applets
self.projectMetadataApplet = ProjectMetadataApplet()
self.dataSelectionApplet = DataSelectionApplet(graph, "Input Data", "Input Data", supportIlastik05Import=True, batchDataGui=False)
self.carvingApplet = CarvingApplet(graph, "xxx", carvingGraphFile)
self.carvingApplet.topLevelOperator.RawData.connect( self.dataSelectionApplet.topLevelOperator.Image )
self.carvingApplet.topLevelOperator.opLabeling.LabelsAllowedFlags.connect( self.dataSelectionApplet.topLevelOperator.AllowLabels )
self.carvingApplet.gui.minLabelNumber = 2
self.carvingApplet.gui.maxLabelNumber = 2
## Access applet operators
opData = self.dataSelectionApplet.topLevelOperator
## Connect operators ##
self._applets.append(self.projectMetadataApplet)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.carvingApplet)
# The shell needs a slot from which he can read the list of image names to switch between.
# Use an OpAttributeSelector to create a slot containing just the filename from the OpDataSelection's DatasetInfo slot.
opSelectFilename = OperatorWrapper( OpAttributeSelector, graph=graph )
opSelectFilename.InputObject.connect( opData.Dataset )
opSelectFilename.AttributeName.setValue( 'filePath' )
self._imageNameListSlot = opSelectFilename.Result
def __init__(self, shell, headless, workflow_cmdline_args,
project_creation_args, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super(LayerViewerWorkflow, self).__init__(shell,
headless,
workflow_cmdline_args,
project_creation_args,
graph=graph,
*args,
**kwargs)
self._applets = []
# Roles
# Create applets
self.dataSelectionApplet = DataSelectionApplet(self, "Input Data",
"Input Data")
self.viewerApplet = LayerViewerApplet(self)
self.dataExportApplet = DataExportApplet(self, "Data Export")
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.SelectionNames.setValue(ROLES)
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue(ROLES)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.viewerApplet)
self._applets.append(self.dataExportApplet)
self._workflow_cmdline_args = workflow_cmdline_args
def setupInputs(self):
data_instructions = 'Use the "Raw Data" tab on the right to load your intensity image(s).'
self.dataSelectionApplet = DataSelectionApplet( self,
"Input Data",
"Input Data",
batchDataGui=False,
forceAxisOrder=None,
instructionText=data_instructions )
opData = self.dataSelectionApplet.topLevelOperator
opData.DatasetRoles.setValue(['Raw Data'])
self.featureSelectionApplet = FeatureSelectionApplet(
self,
"Feature Selection",
"FeatureSelections",
)
self.pcApplet = PixelClassificationApplet(
self, "PixelClassification")
self.thresholdingApplet = ThresholdTwoLevelsApplet(
self, "Thresholding", "ThresholdTwoLevels")
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.featureSelectionApplet)
self._applets.append(self.pcApplet)
self._applets.append(self.thresholdingApplet)
if not self._headless:
self._shell.currentAppletChanged.connect( self.handle_applet_changed )
def __init__(self, headless, workflow_cmdline_args, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super(LayerViewerWorkflow, self).__init__(headless, graph=graph, *args, **kwargs)
self._applets = []
# Create applets
self.dataSelectionApplet = DataSelectionApplet(self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False,
force5d=True)
self.viewerApplet = LayerViewerApplet(self)
self.dataExportApplet = DataExportApplet(self, "Data Export")
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue( ["Raw Data", "Other Data"] )
self._applets.append( self.dataSelectionApplet )
self._applets.append( self.viewerApplet )
self._applets.append( self.dataExportApplet )
self._workflow_cmdline_args = workflow_cmdline_args
def __init__(self, shell, headless, workflow_cmdline_args, project_creation_args, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super(DataConversionWorkflow, self).__init__(shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs)
self._applets = []
# Instantiate DataSelection applet
self.dataSelectionApplet = DataSelectionApplet(
self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
forceAxisOrder=None)
# Configure global DataSelection settings
role_names = ["Input Data"]
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue( role_names )
# Instantiate DataExport applet
self.dataExportApplet = DataExportApplet(self, "Data Export")
# Configure global DataExport settings
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.WorkingDirectory.connect( opDataSelection.WorkingDirectory )
opDataExport.SelectionNames.setValue( ["Input"] )
# No special data pre/post processing necessary in this workflow,
# but this is where we'd hook it up if we needed it.
#
#self.dataExportApplet.prepare_for_entire_export = self.prepare_for_entire_export
#self.dataExportApplet.prepare_lane_for_export = self.prepare_lane_for_export
#self.dataExportApplet.post_process_lane_export = self.post_process_lane_export
#self.dataExportApplet.post_process_entire_export = self.post_process_entire_export
# Instantiate BatchProcessing applet
self.batchProcessingApplet = BatchProcessingApplet(self,
"Batch Processing",
self.dataSelectionApplet,
self.dataExportApplet)
# Expose our applets in a list (for the shell to use)
self._applets.append( self.dataSelectionApplet )
self._applets.append( self.dataExportApplet )
self._applets.append(self.batchProcessingApplet)
# Parse command-line arguments
# Command-line args are applied in onProjectLoaded(), below.
if workflow_cmdline_args:
self._data_export_args, unused_args = self.dataExportApplet.parse_known_cmdline_args( workflow_cmdline_args )
self._batch_input_args, unused_args = self.dataSelectionApplet.parse_known_cmdline_args( unused_args, role_names )
else:
unused_args = None
self._batch_input_args = None
self._data_export_args = None
if unused_args:
logger.warning("Unused command-line args: {}".format( unused_args ))
def setupInputs(self):
data_instructions = 'Use the "Raw Data" tab on the right to load your intensity image(s).'
self.dataSelectionApplet = DataSelectionApplet( self,
"Input Data",
"Input Data",
batchDataGui=False,
forceAxisOrder=None,
instructionText=data_instructions )
opData = self.dataSelectionApplet.topLevelOperator
opData.DatasetRoles.setValue(['Raw Data'])
self.featureSelectionApplet = FeatureSelectionApplet(
self,
"Feature Selection",
"FeatureSelections",
filter_implementation=self.filter_implementation
)
self.pcApplet = PixelClassificationApplet(
self, "PixelClassification")
self.thresholdingApplet = ThresholdTwoLevelsApplet(
self, "Thresholding", "ThresholdTwoLevels")
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.featureSelectionApplet)
self._applets.append(self.pcApplet)
self._applets.append(self.thresholdingApplet)
def __init__(self, shell, headless, workflow_cmdline_args,
project_creation_args, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super(FillMissingSlicesWorkflow, self).__init__(shell,
headless,
workflow_cmdline_args,
project_creation_args,
graph=graph,
*args,
**kwargs)
self._applets = []
# Create applets
self.dataSelectionApplet = DataSelectionApplet(
self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False)
self.fillMissingSlicesApplet = FillMissingSlicesApplet(
self, "Fill Missing Slices", "Fill Missing Slices")
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.fillMissingSlicesApplet)
def __init__(self, shell, headless, workflow_cmdline_args,
project_creation_args, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super(LabelingWorkflow, self).__init__(shell,
headless,
workflow_cmdline_args,
project_creation_args,
graph=graph,
*args,
**kwargs)
self._applets = []
# Create applets
self.dataSelectionApplet = DataSelectionApplet(
self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False)
self.labelingSingleLaneApplet = LabelingSingleLaneApplet(
self, "Generic Labeling (single-lane)")
self.labelingMultiLaneApplet = LabelingApplet(
self, "Generic Labeling (multi-lane)")
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue(["Raw Data"])
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.labelingSingleLaneApplet)
self._applets.append(self.labelingMultiLaneApplet)
def __init__( self, headless, workflow_cmdline_args, *args, **kwargs ):
graph = kwargs['graph'] if 'graph' in kwargs else Graph()
if 'graph' in kwargs: del kwargs['graph']
super(ChaingraphTrackingWorkflow, self).__init__(headless=headless, graph=graph, *args, **kwargs)
data_instructions = 'Use the "Raw Data" tab to load your intensity image(s).\n\n'\
'Use the "Prediction Maps" tab to load your pixel-wise probability image(s).'
## Create applets
self.dataSelectionApplet = DataSelectionApplet(self,
"Input Data",
"Input Data",
batchDataGui=False,
force5d=True,
instructionText=data_instructions,
max_lanes=1 )
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue( ['Raw Data', 'Prediction Maps'] )
self.thresholdTwoLevelsApplet = ThresholdTwoLevelsApplet( self,
"Threshold and Size Filter",
"ThresholdTwoLevels" )
self.objectExtractionApplet = ObjectExtractionApplet( name="Object Feature Computation",
workflow=self, interactive=False )
self.trackingApplet = ChaingraphTrackingApplet( workflow=self )
self._applets = []
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.thresholdTwoLevelsApplet)
self._applets.append(self.objectExtractionApplet)
self._applets.append(self.trackingApplet)
def __init__(self, shell, headless, workflow_cmdline_args,
project_creation_args, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super(PredictionViewerWorkflow, self).__init__(shell,
headless,
workflow_cmdline_args,
project_creation_args,
graph=graph,
*args,
**kwargs)
self._applets = []
# Applets for training (interactive) workflow
self.dataSelectionApplet = DataSelectionApplet(
self,
"Viewed Predictions",
"Viewed Predictions",
supportIlastik05Import=False,
batchDataGui=False)
self.viewerApplet = PredictionViewerApplet(self)
# Expose for shell
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.viewerApplet)
def __init__(self, shell, headless, workflow_cmdline_args,
project_creation_args):
# Create a graph to be shared by all operators
graph = Graph()
super(DeviationFromMeanWorkflow, self).__init__(shell,
headless,
workflow_cmdline_args,
project_creation_args,
graph=graph)
self._applets = []
# Create applets
self.dataSelectionApplet = DataSelectionApplet(
self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False)
self.deviationFromMeanApplet = DeviationFromMeanApplet(
self, "Deviation From Mean")
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue(['Raw Data'])
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.deviationFromMeanApplet)
def __init__(self):
super(VigraWatershedWorkflow, self).__init__()
self._applets = []
# Create a graph to be shared by all operators
graph = Graph()
self._graph = graph
# Create applets
self.dataSelectionApplet = DataSelectionApplet(
graph,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False)
self.watershedApplet = VigraWatershedViewerApplet(
graph, "Watershed", "Watershed")
# Connect top-level operators
self.watershedApplet.topLevelOperator.InputImage.connect(
self.dataSelectionApplet.topLevelOperator.Image)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.watershedApplet)
# The shell needs a slot from which he can read the list of image names to switch between.
# Use an OpAttributeSelector to create a slot containing just the filename from the OpDataSelection's DatasetInfo slot.
opSelectFilename = OperatorWrapper(OpAttributeSelector, graph=graph)
opSelectFilename.InputObject.connect(
self.dataSelectionApplet.topLevelOperator.Dataset)
opSelectFilename.AttributeName.setValue('filePath')
self._imageNameListSlot = opSelectFilename.Result
def __init__(
self,
shell,
headless,
workflow_cmdline_args,
project_creation_args,
hintoverlayFile=None,
pmapoverlayFile=None,
*args,
**kwargs,
):
if workflow_cmdline_args:
assert False, "Not using workflow cmdline args yet."
if hintoverlayFile is not None:
assert isinstance(hintoverlayFile, str), "hintoverlayFile should be a string, not '%s'" % type(
hintoverlayFile
)
if pmapoverlayFile is not None:
assert isinstance(pmapoverlayFile, str), "pmapoverlayFile should be a string, not '%s'" % type(
pmapoverlayFile
)
graph = Graph()
super(CarvingFromPixelPredictionsWorkflow, self).__init__(
shell, headless, workflow_cmdline_args, project_creation_args, *args, graph=graph, **kwargs
)
## Create applets
self.dataSelectionApplet = DataSelectionApplet(
self, "Input Data", "Input Data", supportIlastik05Import=True, batchDataGui=False
)
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue(["Raw Data"])
self.featureSelectionApplet = FeatureSelectionApplet(self, "Feature Selection", "FeatureSelections")
self.pixelClassificationApplet = PixelClassificationApplet(self, "PixelClassification")
self.carvingApplet = CarvingApplet(
workflow=self,
projectFileGroupName="carving",
hintOverlayFile=hintoverlayFile,
pmapOverlayFile=pmapoverlayFile,
)
self.preprocessingApplet = PreprocessingApplet(
workflow=self, title="Preprocessing", projectFileGroupName="carving"
)
# self.carvingApplet.topLevelOperator.MST.connect(self.preprocessingApplet.topLevelOperator.PreprocessedData)
# Expose to shell
self._applets = []
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.featureSelectionApplet)
self._applets.append(self.pixelClassificationApplet)
self._applets.append(self.preprocessingApplet)
self._applets.append(self.carvingApplet)
def __init__(self, shell, headless, workflow_cmdline_args,
project_creation_args, *args, **kwargs):
graph = kwargs['graph'] if 'graph' in kwargs else Graph()
if 'graph' in kwargs: del kwargs['graph']
super(ManualTrackingWorkflow, self).__init__(shell,
headless,
workflow_cmdline_args,
project_creation_args,
graph=graph,
*args,
**kwargs)
data_instructions = 'Use the "Raw Data" tab to load your intensity image(s).\n\n'\
'Use the "Prediction Maps" tab to load your pixel-wise probability image(s).'
## Create applets
self.dataSelectionApplet = DataSelectionApplet(
self,
"Input Data",
"Input Data",
forceAxisOrder=['txyzc'],
instructionText=data_instructions,
max_lanes=1)
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue(['Raw Data', 'Prediction Maps'])
self.thresholdTwoLevelsApplet = ThresholdTwoLevelsApplet(
self, "Threshold and Size Filter", "ThresholdTwoLevels")
self.objectExtractionApplet = ObjectExtractionApplet(
name="Object Feature Computation",
workflow=self,
interactive=False)
self.trackingApplet = ManualTrackingApplet(workflow=self)
self.default_export_filename = '{dataset_dir}/{nickname}-exported_data.csv'
self.dataExportApplet = TrackingBaseDataExportApplet(
self,
"Tracking Result Export",
default_export_filename=self.default_export_filename,
)
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.SelectionNames.setValue(
['Manual Tracking', 'Object Identities'])
opDataExport.WorkingDirectory.connect(opDataSelection.WorkingDirectory)
# Extra configuration for object export table (as CSV table or HDF5 table)
opTracking = self.trackingApplet.topLevelOperator
self.dataExportApplet.set_exporting_operator(opTracking)
#self.dataExportApplet.post_process_lane_export = self.post_process_lane_export
self._applets = []
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.thresholdTwoLevelsApplet)
self._applets.append(self.objectExtractionApplet)
self._applets.append(self.trackingApplet)
self._applets.append(self.dataExportApplet)
def createDataSelectionApplet(self):
"""
Can be overridden by subclasses, if they want to use
special parameters to initialize the DataSelectionApplet.
"""
data_instructions = "Select your input data using the 'Raw Data' tab shown on the right"
return DataSelectionApplet(
self, "Input Data", "Input Data", supportIlastik05Import=True, instructionText=data_instructions
)
def __init__(self, shell, headless, workflow_cmdline_args,
project_creation_args, *args, **kwargs):
super(BlockwiseObjectClassificationWorkflow,
self).__init__(shell, headless, workflow_cmdline_args,
project_creation_args, *args, **kwargs)
### INTERACTIVE ###
# Create applet
self.blockwiseObjectClassificationApplet = BlockwiseObjectClassificationApplet(
self, "Blockwise Object Classification",
"Blockwise Object Classification")
# Expose for shell
self._applets.append(self.blockwiseObjectClassificationApplet)
### BATCH ###
# Create applets for batch workflow
self.rawBatchInputApplet = DataSelectionApplet(
self,
"Raw Batch Input Selections",
"RawBatchDataSelection",
supportIlastik05Import=False,
batchDataGui=True)
# Create applets for batch workflow
self.binaryBatchInputApplet = DataSelectionApplet(
self,
"Binary Batch Input Selections",
"BinaryBatchDataSelection",
supportIlastik05Import=False,
batchDataGui=True)
self.batchResultsApplet = BlockwiseObjectClassificationBatchApplet(
self, "Batch Output Locations")
# Expose in shell
self._applets.append(self.rawBatchInputApplet)
self._applets.append(self.binaryBatchInputApplet)
self._applets.append(self.batchResultsApplet)
# Connect batch workflow (NOT lane-based)
self._initBatchWorkflow()
def parse_known_cmdline_args(self, cmdline_args):
# We use the same parser as the DataSelectionApplet
parser = DataSelectionApplet.get_arg_parser(
self.dataSelectionApplet.role_names)
parser.add_argument(
"--distributed",
help=
"Distributed mode. Used for running ilastik on HPCs via SLURM/srun/mpirun",
action="store_true",
)
default_block_roi = Slice5D.all(x=slice(0, 256),
y=slice(0, 256),
z=slice(0, 256),
t=slice(0, 1))
def parse_distributed_block_roi(value: str) -> Slice5D:
parsed = ast.literal_eval(value)
if isinstance(parsed, dict):
if not set(parsed.keys()).issubset("xyztc"):
raise ValueError(
f"Bad keys for distributed-block-roi: {value}")
if not all(
isinstance(v, (int, None.__class__))
for v in parsed.values()):
raise ValueError(
f"Bad values for distributed-block-roi: {value}")
overrides = {k: slice(0, int(v)) for k, v in parsed.items()}
elif isinstance(parsed, int):
overrides = {k: slice(0, parsed) for k in "xyz"}
else:
raise TypeError(
f"Could not convert value {value} into a Slice5D")
return Slice5D(**{**default_block_roi.to_dict(), **overrides})
parser.add_argument(
"--distributed_block_roi",
"--distributed-block-roi",
help=textwrap.dedent("""
Determines the dimensions of the blocks used to split the data in distributed mode.
Values can be either:"
An integer, which will be interpreted as if the following dict was passed in:
{'x': value, 'y': value, 'z': value, 't': 1, 'c': None}
or a literal python Dict[str, Optional[int]], with keys in 'xyztc'.
Missing keys will default like so:
{'x': 256, 'y': 256, 'z': 256, 't': 1, 'c': None}
Use None anywhere in the dict to mean "the whole dimension".
"""),
type=parse_distributed_block_roi,
default=default_block_roi,
)
parsed_args, unused_args = parser.parse_known_args(cmdline_args)
return parsed_args, unused_args
def onProjectLoaded(self, projectManager):
"""
Overridden from Workflow base class. Called by the Project Manager.
If the user provided command-line arguments, apply them to the workflow operators.
Currently, we support command-line configuration of:
- DataSelection
- Preprocessing, in which case preprocessing is immediately executed
"""
# If input data files were provided on the command line, configure the DataSelection applet now.
# (Otherwise, we assume the project already had a dataset selected.)
input_data_args, unused_args = DataSelectionApplet.parse_known_cmdline_args(self.workflow_cmdline_args, DATA_ROLES)
if input_data_args.raw_data:
self.dataSelectionApplet.configure_operator_with_parsed_args(input_data_args)
#
# Parse the remaining cmd-line arguments
#
filter_indexes = { 'bright-lines' : OpFilter.HESSIAN_BRIGHT,
'dark-lines' : OpFilter.HESSIAN_DARK,
'step-edges' : OpFilter.STEP_EDGES,
'original' : OpFilter.RAW,
'inverted' : OpFilter.RAW_INVERTED }
parser = argparse.ArgumentParser()
parser.add_argument('--run-preprocessing', action='store_true')
parser.add_argument('--preprocessing-sigma', type=float, required=False)
parser.add_argument('--preprocessing-filter', required=False, type=str.lower,
choices=filter_indexes.keys())
parsed_args, unused_args = parser.parse_known_args(unused_args)
if unused_args:
logger.warn("Did not use the following command-line arguments: {}".format(unused_args))
# Execute pre-processing.
if parsed_args.run_preprocessing:
if len(self.preprocessingApplet.topLevelOperator) != 1:
raise RuntimeError("Can't run preprocessing on a project with no images.")
opPreprocessing = self.preprocessingApplet.topLevelOperator.getLane(0) # Carving has only one 'lane'
# If user provided parameters, override the defaults.
if parsed_args.preprocessing_sigma is not None:
opPreprocessing.Sigma.setValue(parsed_args.preprocessing_sigma)
if parsed_args.preprocessing_filter:
filter_index = filter_indexes[parsed_args.preprocessing_filter]
opPreprocessing.Filter.setValue(filter_index)
logger.info("Running Preprocessing...")
opPreprocessing.PreprocessedData[:].wait()
logger.info("FINISHED Preprocessing...")
logger.info("Saving project...")
self._shell.projectManager.saveProject()
logger.info("Done saving.")
def createDataSelectionApplet(self):
data_instructions = "Select your input data using the 'Raw Data' tab shown on the right"
return DataSelectionApplet(
self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False,
instructionText=data_instructions,
show_axis_details=self.supports_anisotropic_data)
def _createInputAndConfigApplets(self):
data_instructions = "Select your input data using the 'Raw Data' tab shown on the right"
self.dataSelectionApplet = DataSelectionApplet(
self,
"Input Data",
"Input Data",
instructionText=data_instructions)
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue(self.ROLE_NAMES)
self._applets.append(self.dataSelectionApplet)
def __init__(self, shell, headless, workflow_cmdline_args, project_creation_args, hintoverlayFile=None, pmapoverlayFile=None, *args, **kwargs):
if hintoverlayFile is not None:
assert isinstance(hintoverlayFile, str), "hintoverlayFile should be a string, not '%s'" % type(hintoverlayFile)
if pmapoverlayFile is not None:
assert isinstance(pmapoverlayFile, str), "pmapoverlayFile should be a string, not '%s'" % type(pmapoverlayFile)
graph = Graph()
super(CarvingWorkflow, self).__init__(shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs)
self.workflow_cmdline_args = workflow_cmdline_args
data_instructions = "Select your input data using the 'Raw Data' tab shown on the right"
## Create applets
self.projectMetadataApplet = ProjectMetadataApplet()
self.dataSelectionApplet = DataSelectionApplet( self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False,
instructionText=data_instructions,
max_lanes=1 )
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue( DATA_ROLES )
self.preprocessingApplet = PreprocessingApplet(workflow=self,
title = "Preprocessing",
projectFileGroupName="preprocessing")
self.carvingApplet = CarvingApplet(workflow=self,
projectFileGroupName="carving",
hintOverlayFile=hintoverlayFile,
pmapOverlayFile=pmapoverlayFile)
#self.carvingApplet.topLevelOperator.MST.connect(self.preprocessingApplet.topLevelOperator.PreprocessedData)
# Expose to shell
self._applets = []
self._applets.append(self.projectMetadataApplet)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.preprocessingApplet)
self._applets.append(self.carvingApplet)
def __init__(self, shell, headless, workflow_cmdline_args,
project_creation_args, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super(RemoteServerWorkflow, self).__init__(shell,
headless,
workflow_cmdline_args,
project_creation_args,
graph=graph,
*args,
**kwargs)
self._applets = []
# Parse workflow-specific command-line args
# TODO allow command-line (does this make sense? probably not...)
# parsed_creation_args, unused_args = parser.parse_known_args(
# project_creation_args)
# Create applets
self.dataSelectionApplet = DataSelectionApplet(
self,
"Remote Computation Plug-in",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False)
self.loginApplet = LoginApplet(self, "Log In")
self.serverBrowserApplet = ServerBrowserApplet(self, "Remote Service")
self.labelingSingleLaneApplet = LabelingSingleLaneApplet(
self, "Labeling")
self.sendToServApplet = SendToServApplet(self, "Request Service")
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue(["Raw Data"])
# Append applets
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.loginApplet)
self._applets.append(self.serverBrowserApplet)
self._applets.append(self.labelingSingleLaneApplet)
self._applets.append(self.sendToServApplet)
def __init__(self, shell, headless, workflow_cmdline_args, project_creation_workflow, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super(WsdtWorkflow, self).__init__( shell, headless, workflow_cmdline_args, project_creation_workflow, graph=graph, *args, **kwargs)
self._applets = []
# -- DataSelection applet
#
self.dataSelectionApplet = DataSelectionApplet(self, "Input Data", "Input Data")
# Dataset inputs
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue( self.ROLE_NAMES )
# -- Wsdt applet
#
self.wsdtApplet = WsdtApplet(self, "Watershed", "Wsdt Watershed")
# -- DataExport applet
#
self.dataExportApplet = DataExportApplet(self, "Data Export")
# Configure global DataExport settings
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.WorkingDirectory.connect( opDataSelection.WorkingDirectory )
opDataExport.SelectionNames.setValue( self.EXPORT_NAMES )
# -- BatchProcessing applet
#
self.batchProcessingApplet = BatchProcessingApplet(self,
"Batch Processing",
self.dataSelectionApplet,
self.dataExportApplet)
# -- Expose applets to shell
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.wsdtApplet)
self._applets.append(self.dataExportApplet)
self._applets.append(self.batchProcessingApplet)
# -- Parse command-line arguments
# (Command-line args are applied in onProjectLoaded(), below.)
if workflow_cmdline_args:
self._data_export_args, unused_args = self.dataExportApplet.parse_known_cmdline_args( workflow_cmdline_args )
self._batch_input_args, unused_args = self.dataSelectionApplet.parse_known_cmdline_args( unused_args, role_names )
else:
unused_args = None
self._batch_input_args = None
self._data_export_args = None
if unused_args:
logger.warning("Unused command-line args: {}".format( unused_args ))
def _run_export_with_empty_batch_lane(self, role_input_paths, batch_lane_index, template_infos, progress_callback):
"""
Configure the fresh batch lane with the given input files, and export the results.
"""
assert role_input_paths[0], "At least one file must be provided for each dataset (the first role)."
opDataSelectionBatchLaneView = self.dataSelectionApplet.topLevelOperator.getLane( batch_lane_index )
# Apply new settings for each role
for role_index, path_for_role in enumerate(role_input_paths):
if not path_for_role:
continue
if template_infos[role_index]:
info = copy.copy(template_infos[role_index])
else:
info = DatasetInfo()
# Override the template settings with the current filepath.
default_info = DataSelectionApplet.create_default_headless_dataset_info(path_for_role)
info.filePath = default_info.filePath
info.location = default_info.location
info.nickname = default_info.nickname
# Apply to the data selection operator
opDataSelectionBatchLaneView.DatasetGroup[role_index].setValue(info)
# Make sure nothing went wrong
opDataExportBatchlaneView = self.dataExportApplet.topLevelOperator.getLane( batch_lane_index )
assert opDataExportBatchlaneView.ImageToExport.ready()
assert opDataExportBatchlaneView.ExportPath.ready()
# New lanes were added.
# Give the workflow a chance to restore anything that was unecessarily invalidated (e.g. classifiers)
self.workflow.handleNewLanesAdded()
# Call customization hook
self.dataExportApplet.prepare_lane_for_export(batch_lane_index)
# Finally, run the export
logger.info("Exporting to {}".format( opDataExportBatchlaneView.ExportPath.value ))
opDataExportBatchlaneView.progressSignal.subscribe(progress_callback)
opDataExportBatchlaneView.run_export()
# Call customization hook
self.dataExportApplet.post_process_lane_export(batch_lane_index)
def __init__(self, shell, headless, workflow_cmdline_args, project_creation_args, hintoverlayFile=None, pmapoverlayFile=None, *args, **kwargs):
if hintoverlayFile is not None:
assert isinstance(hintoverlayFile, str), "hintoverlayFile should be a string, not '%s'" % type(hintoverlayFile)
if pmapoverlayFile is not None:
assert isinstance(pmapoverlayFile, str), "pmapoverlayFile should be a string, not '%s'" % type(pmapoverlayFile)
graph = Graph()
super(CarvingWorkflow, self).__init__(shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs)
self.workflow_cmdline_args = workflow_cmdline_args
data_instructions = "Select your input data using the 'Raw Data' tab shown on the right.\n\n"\
"Additionally, you may optionally add an 'Overlay' data volume if it helps you annotate. (It won't be used for any computation.)"
## Create applets
self.projectMetadataApplet = ProjectMetadataApplet()
self.dataSelectionApplet = DataSelectionApplet( self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False,
instructionText=data_instructions,
max_lanes=1 )
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue( DATA_ROLES )
self.preprocessingApplet = PreprocessingApplet(workflow=self,
title = "Preprocessing",
projectFileGroupName="preprocessing")
self.carvingApplet = CarvingApplet(workflow=self,
projectFileGroupName="carving",
hintOverlayFile=hintoverlayFile,
pmapOverlayFile=pmapoverlayFile)
#self.carvingApplet.topLevelOperator.MST.connect(self.preprocessingApplet.topLevelOperator.PreprocessedData)
# Expose to shell
self._applets = []
self._applets.append(self.projectMetadataApplet)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.preprocessingApplet)
self._applets.append(self.carvingApplet)
def append_lane(workflow, input_filepath, axisorder=None):
# Sanity checks
assert isinstance(workflow, PixelClassificationWorkflow)
opPixelClassification = workflow.pcApplet.topLevelOperator
assert opPixelClassification.Classifier.ready()
# If the filepath is a globstring, convert the stack to h5
input_filepath = DataSelectionApplet.convertStacksToH5( [input_filepath], TMP_DIR )[0]
info = DatasetInfo()
info.location = DatasetInfo.Location.FileSystem
info.filePath = input_filepath
comp = PathComponents(input_filepath)
# Convert all (non-url) paths to absolute
# (otherwise they are relative to the project file, which probably isn't what the user meant)
if not isUrl(input_filepath):
comp.externalPath = os.path.abspath(comp.externalPath)
info.filePath = comp.totalPath()
info.nickname = comp.filenameBase
if axisorder:
info.axistags = vigra.defaultAxistags(axisorder)
logger.debug( "adding lane: {}".format( info ) )
opDataSelection = workflow.dataSelectionApplet.topLevelOperator
# Add a lane
num_lanes = len( opDataSelection.DatasetGroup )+1
logger.debug( "num_lanes: {}".format( num_lanes ) )
opDataSelection.DatasetGroup.resize( num_lanes )
# Configure it.
role_index = 0 # raw data
opDataSelection.DatasetGroup[-1][role_index].setValue( info )
# Sanity check
assert len(opPixelClassification.InputImages) == num_lanes
return opPixelClassification
def append_lane(workflow, input_filepath, axisorder=None):
"""
Add a lane to the project file for the given input file.
If axisorder is given, override the default axisorder for
the file and force the project to use the given one.
Globstrings are supported, in which case the files are converted to HDF5 first.
"""
# If the filepath is a globstring, convert the stack to h5
input_filepath = DataSelectionApplet.convertStacksToH5( [input_filepath], tempfile.mkdtemp() )[0]
info = DatasetInfo()
info.location = DatasetInfo.Location.FileSystem
info.filePath = input_filepath
comp = PathComponents(input_filepath)
# Convert all (non-url) paths to absolute
# (otherwise they are relative to the project file, which probably isn't what the user meant)
if not isUrl(input_filepath):
comp.externalPath = os.path.abspath(comp.externalPath)
info.filePath = comp.totalPath()
info.nickname = comp.filenameBase
if axisorder:
info.axistags = vigra.defaultAxistags(axisorder)
logger.debug( "adding lane: {}".format( info ) )
opDataSelection = workflow.dataSelectionApplet.topLevelOperator
# Add a lane
num_lanes = len( opDataSelection.DatasetGroup )+1
logger.debug( "num_lanes: {}".format( num_lanes ) )
opDataSelection.DatasetGroup.resize( num_lanes )
# Configure it.
role_index = 0 # raw data
opDataSelection.DatasetGroup[-1][role_index].setValue( info )
def __init__(self, shell, headless, workflow_cmdline_args, project_creation_args, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super(DataConversionWorkflow, self).__init__(shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs)
self._applets = []
# Create applets
self.dataSelectionApplet = DataSelectionApplet(self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False,
force5d=False)
opDataSelection = self.dataSelectionApplet.topLevelOperator
role_names = ["Input Data"]
opDataSelection.DatasetRoles.setValue( role_names )
self.dataExportApplet = DataExportApplet(self, "Data Export")
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.WorkingDirectory.connect( opDataSelection.WorkingDirectory )
opDataExport.SelectionNames.setValue( ["Input"] )
self._applets.append( self.dataSelectionApplet )
self._applets.append( self.dataExportApplet )
# Parse command-line arguments
# Command-line args are applied in onProjectLoaded(), below.
self._workflow_cmdline_args = workflow_cmdline_args
self._data_input_args = None
self._data_export_args = None
if workflow_cmdline_args:
self._data_export_args, unused_args = self.dataExportApplet.parse_known_cmdline_args( unused_args )
self._data_input_args, unused_args = self.dataSelectionApplet.parse_known_cmdline_args( workflow_cmdline_args, role_names )
if unused_args:
logger.warn("Unused command-line args: {}".format( unused_args ))
def __init__(self, shell, headless, workflow_cmdline_args, project_creation_args, appendBatchOperators=True, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super( PixelClassificationWorkflow, self ).__init__( shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs )
self._applets = []
self._workflow_cmdline_args = workflow_cmdline_args
# Parse workflow-specific command-line args
parser = argparse.ArgumentParser()
parser.add_argument('--filter', help="pixel feature filter implementation.", choices=['Original', 'Refactored', 'Interpolated'], default='Original')
parser.add_argument('--print-labels-by-slice', help="Print the number of labels for each Z-slice of each image.", action="store_true")
parser.add_argument('--label-search-value', help="If provided, only this value is considered when using --print-labels-by-slice", default=0, type=int)
parser.add_argument('--generate-random-labels', help="Add random labels to the project file.", action="store_true")
parser.add_argument('--random-label-value', help="The label value to use injecting random labels", default=1, type=int)
parser.add_argument('--random-label-count', help="The number of random labels to inject via --generate-random-labels", default=2000, type=int)
parser.add_argument('--retrain', help="Re-train the classifier based on labels stored in project file, and re-save.", action="store_true")
# Parse the creation args: These were saved to the project file when this project was first created.
parsed_creation_args, unused_args = parser.parse_known_args(project_creation_args)
self.filter_implementation = parsed_creation_args.filter
# Parse the cmdline args for the current session.
parsed_args, unused_args = parser.parse_known_args(workflow_cmdline_args)
self.print_labels_by_slice = parsed_args.print_labels_by_slice
self.label_search_value = parsed_args.label_search_value
self.generate_random_labels = parsed_args.generate_random_labels
self.random_label_value = parsed_args.random_label_value
self.random_label_count = parsed_args.random_label_count
self.retrain = parsed_args.retrain
if parsed_args.filter and parsed_args.filter != parsed_creation_args.filter:
logger.error("Ignoring new --filter setting. Filter implementation cannot be changed after initial project creation.")
data_instructions = "Select your input data using the 'Raw Data' tab shown on the right.\n\n"\
"Power users: Optionally use the 'Prediction Mask' tab to supply a binary image that tells ilastik where it should avoid computations you don't need."
# Applets for training (interactive) workflow
self.projectMetadataApplet = ProjectMetadataApplet()
self.dataSelectionApplet = DataSelectionApplet( self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False,
instructionText=data_instructions )
opDataSelection = self.dataSelectionApplet.topLevelOperator
# see role constants, above
opDataSelection.DatasetRoles.setValue( ['Raw Data', 'Prediction Mask'] )
self.featureSelectionApplet = FeatureSelectionApplet(self, "Feature Selection", "FeatureSelections", self.filter_implementation)
self.pcApplet = PixelClassificationApplet( self, "PixelClassification" )
opClassify = self.pcApplet.topLevelOperator
self.dataExportApplet = PixelClassificationDataExportApplet(self, "Prediction Export")
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.PmapColors.connect( opClassify.PmapColors )
opDataExport.LabelNames.connect( opClassify.LabelNames )
opDataExport.WorkingDirectory.connect( opDataSelection.WorkingDirectory )
opDataExport.SelectionNames.setValue( self.EXPORT_NAMES )
# Expose for shell
self._applets.append(self.projectMetadataApplet)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.featureSelectionApplet)
self._applets.append(self.pcApplet)
self._applets.append(self.dataExportApplet)
self._batch_input_args = None
self._batch_export_args = None
self.batchInputApplet = None
self.batchResultsApplet = None
if appendBatchOperators:
# Create applets for batch workflow
self.batchInputApplet = DataSelectionApplet(self, "Batch Prediction Input Selections", "Batch Inputs", supportIlastik05Import=False, batchDataGui=True)
self.batchResultsApplet = PixelClassificationDataExportApplet(self, "Batch Prediction Output Locations", isBatch=True)
# Expose in shell
self._applets.append(self.batchInputApplet)
self._applets.append(self.batchResultsApplet)
# Connect batch workflow (NOT lane-based)
self._initBatchWorkflow()
if unused_args:
# We parse the export setting args first. All remaining args are considered input files by the input applet.
self._batch_export_args, unused_args = self.batchResultsApplet.parse_known_cmdline_args( unused_args )
self._batch_input_args, unused_args = self.batchInputApplet.parse_known_cmdline_args( unused_args )
if unused_args:
logger.warn("Unused command-line args: {}".format( unused_args ))
class PixelClassificationWorkflow(Workflow):
workflowName = "Pixel Classification"
workflowDescription = "This is obviously self-explanatory."
defaultAppletIndex = 1 # show DataSelection by default
DATA_ROLE_RAW = 0
DATA_ROLE_PREDICTION_MASK = 1
EXPORT_NAMES = ['Probabilities', 'Simple Segmentation', 'Uncertainty', 'Features']
@property
def applets(self):
return self._applets
@property
def imageNameListSlot(self):
return self.dataSelectionApplet.topLevelOperator.ImageName
def __init__(self, shell, headless, workflow_cmdline_args, project_creation_args, appendBatchOperators=True, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super( PixelClassificationWorkflow, self ).__init__( shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs )
self._applets = []
self._workflow_cmdline_args = workflow_cmdline_args
# Parse workflow-specific command-line args
parser = argparse.ArgumentParser()
parser.add_argument('--filter', help="pixel feature filter implementation.", choices=['Original', 'Refactored', 'Interpolated'], default='Original')
parser.add_argument('--print-labels-by-slice', help="Print the number of labels for each Z-slice of each image.", action="store_true")
parser.add_argument('--label-search-value', help="If provided, only this value is considered when using --print-labels-by-slice", default=0, type=int)
parser.add_argument('--generate-random-labels', help="Add random labels to the project file.", action="store_true")
parser.add_argument('--random-label-value', help="The label value to use injecting random labels", default=1, type=int)
parser.add_argument('--random-label-count', help="The number of random labels to inject via --generate-random-labels", default=2000, type=int)
parser.add_argument('--retrain', help="Re-train the classifier based on labels stored in project file, and re-save.", action="store_true")
# Parse the creation args: These were saved to the project file when this project was first created.
parsed_creation_args, unused_args = parser.parse_known_args(project_creation_args)
self.filter_implementation = parsed_creation_args.filter
# Parse the cmdline args for the current session.
parsed_args, unused_args = parser.parse_known_args(workflow_cmdline_args)
self.print_labels_by_slice = parsed_args.print_labels_by_slice
self.label_search_value = parsed_args.label_search_value
self.generate_random_labels = parsed_args.generate_random_labels
self.random_label_value = parsed_args.random_label_value
self.random_label_count = parsed_args.random_label_count
self.retrain = parsed_args.retrain
if parsed_args.filter and parsed_args.filter != parsed_creation_args.filter:
logger.error("Ignoring new --filter setting. Filter implementation cannot be changed after initial project creation.")
data_instructions = "Select your input data using the 'Raw Data' tab shown on the right.\n\n"\
"Power users: Optionally use the 'Prediction Mask' tab to supply a binary image that tells ilastik where it should avoid computations you don't need."
# Applets for training (interactive) workflow
self.projectMetadataApplet = ProjectMetadataApplet()
self.dataSelectionApplet = DataSelectionApplet( self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False,
instructionText=data_instructions )
opDataSelection = self.dataSelectionApplet.topLevelOperator
# see role constants, above
opDataSelection.DatasetRoles.setValue( ['Raw Data', 'Prediction Mask'] )
self.featureSelectionApplet = FeatureSelectionApplet(self, "Feature Selection", "FeatureSelections", self.filter_implementation)
self.pcApplet = PixelClassificationApplet( self, "PixelClassification" )
opClassify = self.pcApplet.topLevelOperator
self.dataExportApplet = PixelClassificationDataExportApplet(self, "Prediction Export")
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.PmapColors.connect( opClassify.PmapColors )
opDataExport.LabelNames.connect( opClassify.LabelNames )
opDataExport.WorkingDirectory.connect( opDataSelection.WorkingDirectory )
opDataExport.SelectionNames.setValue( self.EXPORT_NAMES )
# Expose for shell
self._applets.append(self.projectMetadataApplet)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.featureSelectionApplet)
self._applets.append(self.pcApplet)
self._applets.append(self.dataExportApplet)
self._batch_input_args = None
self._batch_export_args = None
self.batchInputApplet = None
self.batchResultsApplet = None
if appendBatchOperators:
# Create applets for batch workflow
self.batchInputApplet = DataSelectionApplet(self, "Batch Prediction Input Selections", "Batch Inputs", supportIlastik05Import=False, batchDataGui=True)
self.batchResultsApplet = PixelClassificationDataExportApplet(self, "Batch Prediction Output Locations", isBatch=True)
# Expose in shell
self._applets.append(self.batchInputApplet)
self._applets.append(self.batchResultsApplet)
# Connect batch workflow (NOT lane-based)
self._initBatchWorkflow()
if unused_args:
# We parse the export setting args first. All remaining args are considered input files by the input applet.
self._batch_export_args, unused_args = self.batchResultsApplet.parse_known_cmdline_args( unused_args )
self._batch_input_args, unused_args = self.batchInputApplet.parse_known_cmdline_args( unused_args )
if unused_args:
logger.warn("Unused command-line args: {}".format( unused_args ))
def connectLane(self, laneIndex):
# Get a handle to each operator
opData = self.dataSelectionApplet.topLevelOperator.getLane(laneIndex)
opTrainingFeatures = self.featureSelectionApplet.topLevelOperator.getLane(laneIndex)
opClassify = self.pcApplet.topLevelOperator.getLane(laneIndex)
opDataExport = self.dataExportApplet.topLevelOperator.getLane(laneIndex)
# Input Image -> Feature Op
# and -> Classification Op (for display)
opTrainingFeatures.InputImage.connect( opData.Image )
opClassify.InputImages.connect( opData.Image )
if ilastik_config.getboolean('ilastik', 'debug'):
opClassify.PredictionMasks.connect( opData.ImageGroup[self.DATA_ROLE_PREDICTION_MASK] )
# Feature Images -> Classification Op (for training, prediction)
opClassify.FeatureImages.connect( opTrainingFeatures.OutputImage )
opClassify.CachedFeatureImages.connect( opTrainingFeatures.CachedOutputImage )
# Training flags -> Classification Op (for GUI restrictions)
opClassify.LabelsAllowedFlags.connect( opData.AllowLabels )
# Data Export connections
opDataExport.RawData.connect( opData.ImageGroup[self.DATA_ROLE_RAW] )
opDataExport.RawDatasetInfo.connect( opData.DatasetGroup[self.DATA_ROLE_RAW] )
opDataExport.ConstraintDataset.connect( opData.ImageGroup[self.DATA_ROLE_RAW] )
opDataExport.Inputs.resize( len(self.EXPORT_NAMES) )
opDataExport.Inputs[0].connect( opClassify.HeadlessPredictionProbabilities )
opDataExport.Inputs[1].connect( opClassify.SimpleSegmentation )
opDataExport.Inputs[2].connect( opClassify.HeadlessUncertaintyEstimate )
opDataExport.Inputs[3].connect( opClassify.FeatureImages )
for slot in opDataExport.Inputs:
assert slot.partner is not None
def _initBatchWorkflow(self):
"""
Connect the batch-mode top-level operators to the training workflow and to each other.
"""
# Access applet operators from the training workflow
opTrainingDataSelection = self.dataSelectionApplet.topLevelOperator
opTrainingFeatures = self.featureSelectionApplet.topLevelOperator
opClassify = self.pcApplet.topLevelOperator
# Access the batch operators
opBatchInputs = self.batchInputApplet.topLevelOperator
opBatchResults = self.batchResultsApplet.topLevelOperator
opBatchInputs.DatasetRoles.connect( opTrainingDataSelection.DatasetRoles )
opSelectFirstLane = OperatorWrapper( OpSelectSubslot, parent=self )
opSelectFirstLane.Inputs.connect( opTrainingDataSelection.ImageGroup )
opSelectFirstLane.SubslotIndex.setValue(0)
opSelectFirstRole = OpSelectSubslot( parent=self )
opSelectFirstRole.Inputs.connect( opSelectFirstLane.Output )
opSelectFirstRole.SubslotIndex.setValue(self.DATA_ROLE_RAW)
opBatchResults.ConstraintDataset.connect( opSelectFirstRole.Output )
## Create additional batch workflow operators
opBatchFeatures = OperatorWrapper( OpFeatureSelectionNoCache, operator_kwargs={'filter_implementation': self.filter_implementation}, parent=self, promotedSlotNames=['InputImage'] )
opBatchPredictionPipeline = OperatorWrapper( OpPredictionPipelineNoCache, parent=self )
## Connect Operators ##
opTranspose = OpTransposeSlots( parent=self )
opTranspose.OutputLength.setValue(2) # There are 2 roles
opTranspose.Inputs.connect( opBatchInputs.DatasetGroup )
opTranspose.name = "batchTransposeInputs"
# Provide dataset paths from data selection applet to the batch export applet
opBatchResults.RawDatasetInfo.connect( opTranspose.Outputs[self.DATA_ROLE_RAW] )
opBatchResults.WorkingDirectory.connect( opBatchInputs.WorkingDirectory )
# Connect (clone) the feature operator inputs from
# the interactive workflow's features operator (which gets them from the GUI)
opBatchFeatures.Scales.connect( opTrainingFeatures.Scales )
opBatchFeatures.FeatureIds.connect( opTrainingFeatures.FeatureIds )
opBatchFeatures.SelectionMatrix.connect( opTrainingFeatures.SelectionMatrix )
# Classifier and NumClasses are provided by the interactive workflow
opBatchPredictionPipeline.Classifier.connect( opClassify.Classifier )
opBatchPredictionPipeline.NumClasses.connect( opClassify.NumClasses )
# Provide these for the gui
opBatchResults.RawData.connect( opBatchInputs.Image )
opBatchResults.PmapColors.connect( opClassify.PmapColors )
opBatchResults.LabelNames.connect( opClassify.LabelNames )
# Connect Image pathway:
# Input Image -> Features Op -> Prediction Op -> Export
opBatchFeatures.InputImage.connect( opBatchInputs.Image )
opBatchPredictionPipeline.PredictionMask.connect( opBatchInputs.Image1 )
opBatchPredictionPipeline.FeatureImages.connect( opBatchFeatures.OutputImage )
opBatchResults.SelectionNames.setValue( self.EXPORT_NAMES )
# opBatchResults.Inputs is indexed by [lane][selection],
# Use OpTranspose to allow connection.
opTransposeBatchInputs = OpTransposeSlots( parent=self )
opTransposeBatchInputs.name = "opTransposeBatchInputs"
opTransposeBatchInputs.OutputLength.setValue(0)
opTransposeBatchInputs.Inputs.resize( len(self.EXPORT_NAMES) )
opTransposeBatchInputs.Inputs[0].connect( opBatchPredictionPipeline.HeadlessPredictionProbabilities ) # selection 0
opTransposeBatchInputs.Inputs[1].connect( opBatchPredictionPipeline.SimpleSegmentation ) # selection 1
opTransposeBatchInputs.Inputs[2].connect( opBatchPredictionPipeline.HeadlessUncertaintyEstimate ) # selection 2
opTransposeBatchInputs.Inputs[3].connect( opBatchPredictionPipeline.FeatureImages ) # selection 3
for slot in opTransposeBatchInputs.Inputs:
assert slot.partner is not None
# Now opTransposeBatchInputs.Outputs is level-2 indexed by [lane][selection]
opBatchResults.Inputs.connect( opTransposeBatchInputs.Outputs )
# We don't actually need the cached path in the batch pipeline.
# Just connect the uncached features here to satisfy the operator.
#opBatchPredictionPipeline.CachedFeatureImages.connect( opBatchFeatures.OutputImage )
self.opBatchFeatures = opBatchFeatures
self.opBatchPredictionPipeline = opBatchPredictionPipeline
def handleAppletStateUpdateRequested(self):
"""
Overridden from Workflow base class
Called when an applet has fired the :py:attr:`Applet.appletStateUpdateRequested`
"""
# If no data, nothing else is ready.
opDataSelection = self.dataSelectionApplet.topLevelOperator
input_ready = len(opDataSelection.ImageGroup) > 0 and not self.dataSelectionApplet.busy
opFeatureSelection = self.featureSelectionApplet.topLevelOperator
featureOutput = opFeatureSelection.OutputImage
features_ready = input_ready and \
len(featureOutput) > 0 and \
featureOutput[0].ready() and \
(TinyVector(featureOutput[0].meta.shape) > 0).all()
opDataExport = self.dataExportApplet.topLevelOperator
opPixelClassification = self.pcApplet.topLevelOperator
invalid_classifier = opPixelClassification.classifier_cache.fixAtCurrent.value and \
opPixelClassification.classifier_cache.Output.ready() and\
opPixelClassification.classifier_cache.Output.value is None
predictions_ready = features_ready and \
not invalid_classifier and \
len(opDataExport.Inputs) > 0 and \
opDataExport.Inputs[0][0].ready() and \
(TinyVector(opDataExport.Inputs[0][0].meta.shape) > 0).all()
# Problems can occur if the features or input data are changed during live update mode.
# Don't let the user do that.
live_update_active = not opPixelClassification.FreezePredictions.value
self._shell.setAppletEnabled(self.dataSelectionApplet, not live_update_active)
self._shell.setAppletEnabled(self.featureSelectionApplet, input_ready and not live_update_active)
self._shell.setAppletEnabled(self.pcApplet, features_ready)
self._shell.setAppletEnabled(self.dataExportApplet, predictions_ready)
if self.batchInputApplet is not None:
# Training workflow must be fully configured before batch can be used
self._shell.setAppletEnabled(self.batchInputApplet, predictions_ready)
opBatchDataSelection = self.batchInputApplet.topLevelOperator
batch_input_ready = predictions_ready and \
len(opBatchDataSelection.ImageGroup) > 0
self._shell.setAppletEnabled(self.batchResultsApplet, batch_input_ready)
# Lastly, check for certain "busy" conditions, during which we
# should prevent the shell from closing the project.
busy = False
busy |= self.dataSelectionApplet.busy
busy |= self.featureSelectionApplet.busy
busy |= self.dataExportApplet.busy
self._shell.enableProjectChanges( not busy )
def getHeadlessOutputSlot(self, slotId):
# "Regular" (i.e. with the images that the user selected as input data)
if slotId == "Predictions":
return self.pcApplet.topLevelOperator.HeadlessPredictionProbabilities
elif slotId == "PredictionsUint8":
return self.pcApplet.topLevelOperator.HeadlessUint8PredictionProbabilities
# "Batch" (i.e. with the images that the user selected as batch inputs).
elif slotId == "BatchPredictions":
return self.opBatchPredictionPipeline.HeadlessPredictionProbabilities
if slotId == "BatchPredictionsUint8":
return self.opBatchPredictionPipeline.HeadlessUint8PredictionProbabilities
raise Exception("Unknown headless output slot")
def onProjectLoaded(self, projectManager):
"""
Overridden from Workflow base class. Called by the Project Manager.
If the user provided command-line arguments, use them to configure
the workflow for batch mode and export all results.
(This workflow's headless mode supports only batch mode for now.)
"""
if self.generate_random_labels:
self._generate_random_labels(self.random_label_count, self.random_label_value)
logger.info("Saving project...")
self._shell.projectManager.saveProject()
logger.info("Done.")
if self.print_labels_by_slice:
self._print_labels_by_slice( self.label_search_value )
# Configure the batch data selection operator.
if self._batch_input_args and (self._batch_input_args.input_files or self._batch_input_args.raw_data):
self.batchInputApplet.configure_operator_with_parsed_args( self._batch_input_args )
# Configure the data export operator.
if self._batch_export_args:
self.batchResultsApplet.configure_operator_with_parsed_args( self._batch_export_args )
if self._batch_input_args and self.pcApplet.topLevelOperator.classifier_cache._dirty:
logger.warn("Your project file has no classifier. A new classifier will be trained for this run.")
if self._headless:
# In headless mode, let's see the messages from the training operator.
logging.getLogger("lazyflow.operators.classifierOperators").setLevel(logging.DEBUG)
if self.retrain:
# Cause the classifier to be dirty so it is forced to retrain.
# (useful if the stored labels were changed outside ilastik)
self.pcApplet.topLevelOperator.opTrain.ClassifierFactory.setDirty()
# Request the classifier, which forces training
self.pcApplet.topLevelOperator.FreezePredictions.setValue(False)
_ = self.pcApplet.topLevelOperator.Classifier.value
# store new classifier to project file
projectManager.saveProject(force_all_save=False)
if self._headless and self._batch_input_args and self._batch_export_args:
# Make sure we're using the up-to-date classifier.
self.pcApplet.topLevelOperator.FreezePredictions.setValue(False)
# Now run the batch export and report progress....
opBatchDataExport = self.batchResultsApplet.topLevelOperator
for i, opExportDataLaneView in enumerate(opBatchDataExport):
logger.info( "Exporting result {} to {}".format(i, opExportDataLaneView.ExportPath.value) )
sys.stdout.write( "Result {}/{} Progress: ".format( i, len( opBatchDataExport ) ) )
sys.stdout.flush()
def print_progress( progress ):
sys.stdout.write( "{} ".format( progress ) )
sys.stdout.flush()
# If the operator provides a progress signal, use it.
slotProgressSignal = opExportDataLaneView.progressSignal
slotProgressSignal.subscribe( print_progress )
opExportDataLaneView.run_export()
# Finished.
sys.stdout.write("\n")
def _print_labels_by_slice(self, search_value):
"""
Iterate over each label image in the project and print the number of labels present on each Z-slice of the image.
(This is a special feature requested by the FlyEM proofreaders.)
"""
opTopLevelClassify = self.pcApplet.topLevelOperator
project_label_count = 0
for image_index, label_slot in enumerate(opTopLevelClassify.LabelImages):
tagged_shape = label_slot.meta.getTaggedShape()
if 'z' not in tagged_shape:
logger.error("Can't print label counts by Z-slices. Image #{} has no Z-dimension.".format(image_index))
else:
logger.info("Label counts in Z-slices of Image #{}:".format( image_index ))
slicing = [slice(None)] * len(tagged_shape)
blank_slices = []
image_label_count = 0
for z in range(tagged_shape['z']):
slicing[tagged_shape.keys().index('z')] = slice(z, z+1)
label_slice = label_slot[slicing].wait()
if search_value:
count = (label_slice == search_value).sum()
else:
count = (label_slice != 0).sum()
if count > 0:
logger.info("Z={}: {}".format( z, count ))
image_label_count += count
else:
blank_slices.append( z )
project_label_count += image_label_count
if len(blank_slices) > 20:
# Don't list the blank slices if there were a lot of them.
logger.info("Image #{} has {} blank slices.".format( image_index, len(blank_slices) ))
elif len(blank_slices) > 0:
logger.info( "Image #{} has {} blank slices: {}".format( image_index, len(blank_slices), blank_slices ) )
else:
logger.info( "Image #{} has no blank slices.".format( image_index ) )
logger.info( "Total labels for Image #{}: {}".format( image_index, image_label_count ) )
logger.info( "Total labels for project: {}".format( project_label_count ) )
def _generate_random_labels(self, labels_per_image, label_value):
"""
Inject random labels into the project file.
(This is a special feature requested by the FlyEM proofreaders.)
"""
logger.info( "Injecting {} labels of value {} into all images.".format( labels_per_image, label_value ) )
opTopLevelClassify = self.pcApplet.topLevelOperator
label_names = copy.copy(opTopLevelClassify.LabelNames.value)
while len(label_names) < label_value:
label_names.append( "Label {}".format( len(label_names)+1 ) )
opTopLevelClassify.LabelNames.setValue( label_names )
for image_index in range(len(opTopLevelClassify.LabelImages)):
logger.info( "Injecting labels into image #{}".format( image_index ) )
# For reproducibility of label generation
SEED = 1
numpy.random.seed([SEED, image_index])
label_input_slot = opTopLevelClassify.LabelInputs[image_index]
label_output_slot = opTopLevelClassify.LabelImages[image_index]
shape = label_output_slot.meta.shape
random_labels = numpy.zeros( shape=shape, dtype=numpy.uint8 )
num_pixels = len(random_labels.flat)
current_progress = -1
for sample_index in range(labels_per_image):
flat_index = numpy.random.randint(0,num_pixels)
# Don't overwrite existing labels
# Keep looking until we find a blank pixel
while random_labels.flat[flat_index]:
flat_index = numpy.random.randint(0,num_pixels)
random_labels.flat[flat_index] = label_value
# Print progress every 10%
progress = float(sample_index) / labels_per_image
progress = 10 * (int(100*progress)/10)
if progress != current_progress:
current_progress = progress
sys.stdout.write( "{}% ".format( current_progress ) )
sys.stdout.flush()
sys.stdout.write( "100%\n" )
# Write into the operator
label_input_slot[fullSlicing(shape)] = random_labels
logger.info( "Done injecting labels" )
class PixelClassificationWorkflow(Workflow):
workflowName = "Pixel Classification"
workflowDescription = "This is obviously self-explanoratory."
defaultAppletIndex = 1 # show DataSelection by default
@property
def applets(self):
return self._applets
@property
def imageNameListSlot(self):
return self.dataSelectionApplet.topLevelOperator.ImageName
def __init__(self, shell, headless, workflow_cmdline_args, appendBatchOperators=True, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super( PixelClassificationWorkflow, self ).__init__( shell, headless, graph=graph, *args, **kwargs )
self._applets = []
self._workflow_cmdline_args = workflow_cmdline_args
data_instructions = "Select your input data using the 'Raw Data' tab shown on the right"
# Parse workflow-specific command-line args
parser = argparse.ArgumentParser()
parser.add_argument('--filter', help="pixel feature filter implementation.", choices=['Original', 'Refactored', 'Interpolated'], default='Original')
parsed_args, unused_args = parser.parse_known_args(workflow_cmdline_args)
self.filter_implementation = parsed_args.filter
# Applets for training (interactive) workflow
self.projectMetadataApplet = ProjectMetadataApplet()
self.dataSelectionApplet = DataSelectionApplet( self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False,
instructionText=data_instructions )
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue( ['Raw Data'] )
self.featureSelectionApplet = FeatureSelectionApplet(self, "Feature Selection", "FeatureSelections", self.filter_implementation)
self.pcApplet = PixelClassificationApplet(self, "PixelClassification")
opClassify = self.pcApplet.topLevelOperator
self.dataExportApplet = PixelClassificationDataExportApplet(self, "Prediction Export")
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.PmapColors.connect( opClassify.PmapColors )
opDataExport.LabelNames.connect( opClassify.LabelNames )
opDataExport.WorkingDirectory.connect( opDataSelection.WorkingDirectory )
# Expose for shell
self._applets.append(self.projectMetadataApplet)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.featureSelectionApplet)
self._applets.append(self.pcApplet)
self._applets.append(self.dataExportApplet)
self._batch_input_args = None
self._batch_export_args = None
self.batchInputApplet = None
self.batchResultsApplet = None
if appendBatchOperators:
# Create applets for batch workflow
self.batchInputApplet = DataSelectionApplet(self, "Batch Prediction Input Selections", "Batch Inputs", supportIlastik05Import=False, batchDataGui=True)
self.batchResultsApplet = PixelClassificationDataExportApplet(self, "Batch Prediction Output Locations", isBatch=True)
# Expose in shell
self._applets.append(self.batchInputApplet)
self._applets.append(self.batchResultsApplet)
# Connect batch workflow (NOT lane-based)
self._initBatchWorkflow()
if unused_args:
# We parse the export setting args first. All remaining args are considered input files by the input applet.
self._batch_export_args, unused_args = self.batchResultsApplet.parse_known_cmdline_args( unused_args )
self._batch_input_args, unused_args = self.batchInputApplet.parse_known_cmdline_args( unused_args )
if unused_args:
logger.warn("Unused command-line args: {}".format( unused_args ))
def connectLane(self, laneIndex):
# Get a handle to each operator
opData = self.dataSelectionApplet.topLevelOperator.getLane(laneIndex)
opTrainingFeatures = self.featureSelectionApplet.topLevelOperator.getLane(laneIndex)
opClassify = self.pcApplet.topLevelOperator.getLane(laneIndex)
opDataExport = self.dataExportApplet.topLevelOperator.getLane(laneIndex)
# Input Image -> Feature Op
# and -> Classification Op (for display)
opTrainingFeatures.InputImage.connect( opData.Image )
opClassify.InputImages.connect( opData.Image )
# Feature Images -> Classification Op (for training, prediction)
opClassify.FeatureImages.connect( opTrainingFeatures.OutputImage )
opClassify.CachedFeatureImages.connect( opTrainingFeatures.CachedOutputImage )
# Training flags -> Classification Op (for GUI restrictions)
opClassify.LabelsAllowedFlags.connect( opData.AllowLabels )
# Data Export connections
opDataExport.RawData.connect( opData.ImageGroup[0] )
opDataExport.Input.connect( opClassify.HeadlessPredictionProbabilities )
opDataExport.RawDatasetInfo.connect( opData.DatasetGroup[0] )
opDataExport.ConstraintDataset.connect( opData.ImageGroup[0] )
def _initBatchWorkflow(self):
"""
Connect the batch-mode top-level operators to the training workflow and to each other.
"""
# Access applet operators from the training workflow
opTrainingDataSelection = self.dataSelectionApplet.topLevelOperator
opTrainingFeatures = self.featureSelectionApplet.topLevelOperator
opClassify = self.pcApplet.topLevelOperator
# Access the batch operators
opBatchInputs = self.batchInputApplet.topLevelOperator
opBatchResults = self.batchResultsApplet.topLevelOperator
opBatchInputs.DatasetRoles.connect( opTrainingDataSelection.DatasetRoles )
opSelectFirstLane = OperatorWrapper( OpSelectSubslot, parent=self )
opSelectFirstLane.Inputs.connect( opTrainingDataSelection.ImageGroup )
opSelectFirstLane.SubslotIndex.setValue(0)
opSelectFirstRole = OpSelectSubslot( parent=self )
opSelectFirstRole.Inputs.connect( opSelectFirstLane.Output )
opSelectFirstRole.SubslotIndex.setValue(0)
opBatchResults.ConstraintDataset.connect( opSelectFirstRole.Output )
## Create additional batch workflow operators
opBatchFeatures = OperatorWrapper( OpFeatureSelectionNoCache, operator_kwargs={'filter_implementation': self.filter_implementation}, parent=self, promotedSlotNames=['InputImage'] )
opBatchPredictionPipeline = OperatorWrapper( OpPredictionPipelineNoCache, parent=self )
## Connect Operators ##
opTranspose = OpTransposeSlots( parent=self )
opTranspose.OutputLength.setValue(1)
opTranspose.Inputs.connect( opBatchInputs.DatasetGroup )
# Provide dataset paths from data selection applet to the batch export applet
opBatchResults.RawDatasetInfo.connect( opTranspose.Outputs[0] )
opBatchResults.WorkingDirectory.connect( opBatchInputs.WorkingDirectory )
# Connect (clone) the feature operator inputs from
# the interactive workflow's features operator (which gets them from the GUI)
opBatchFeatures.Scales.connect( opTrainingFeatures.Scales )
opBatchFeatures.FeatureIds.connect( opTrainingFeatures.FeatureIds )
opBatchFeatures.SelectionMatrix.connect( opTrainingFeatures.SelectionMatrix )
# Classifier and NumClasses are provided by the interactive workflow
opBatchPredictionPipeline.Classifier.connect( opClassify.Classifier )
opBatchPredictionPipeline.FreezePredictions.setValue( False )
opBatchPredictionPipeline.NumClasses.connect( opClassify.NumClasses )
# Provide these for the gui
opBatchResults.RawData.connect( opBatchInputs.Image )
opBatchResults.PmapColors.connect( opClassify.PmapColors )
opBatchResults.LabelNames.connect( opClassify.LabelNames )
# Connect Image pathway:
# Input Image -> Features Op -> Prediction Op -> Export
opBatchFeatures.InputImage.connect( opBatchInputs.Image )
opBatchPredictionPipeline.FeatureImages.connect( opBatchFeatures.OutputImage )
opBatchResults.Input.connect( opBatchPredictionPipeline.HeadlessPredictionProbabilities )
# We don't actually need the cached path in the batch pipeline.
# Just connect the uncached features here to satisfy the operator.
#opBatchPredictionPipeline.CachedFeatureImages.connect( opBatchFeatures.OutputImage )
self.opBatchPredictionPipeline = opBatchPredictionPipeline
def handleAppletStateUpdateRequested(self):
"""
Overridden from Workflow base class
Called when an applet has fired the :py:attr:`Applet.appletStateUpdateRequested`
"""
# If no data, nothing else is ready.
opDataSelection = self.dataSelectionApplet.topLevelOperator
input_ready = len(opDataSelection.ImageGroup) > 0 and not self.dataSelectionApplet.busy
opFeatureSelection = self.featureSelectionApplet.topLevelOperator
featureOutput = opFeatureSelection.OutputImage
features_ready = input_ready and \
len(featureOutput) > 0 and \
featureOutput[0].ready() and \
(TinyVector(featureOutput[0].meta.shape) > 0).all()
opDataExport = self.dataExportApplet.topLevelOperator
predictions_ready = features_ready and \
len(opDataExport.Input) > 0 and \
opDataExport.Input[0].ready() and \
(TinyVector(opDataExport.Input[0].meta.shape) > 0).all()
# Problems can occur if the features or input data are changed during live update mode.
# Don't let the user do that.
opPixelClassification = self.pcApplet.topLevelOperator
live_update_active = not opPixelClassification.FreezePredictions.value
self._shell.setAppletEnabled(self.dataSelectionApplet, not live_update_active)
self._shell.setAppletEnabled(self.featureSelectionApplet, input_ready and not live_update_active)
self._shell.setAppletEnabled(self.pcApplet, features_ready)
self._shell.setAppletEnabled(self.dataExportApplet, predictions_ready)
if self.batchInputApplet is not None:
# Training workflow must be fully configured before batch can be used
self._shell.setAppletEnabled(self.batchInputApplet, predictions_ready)
opBatchDataSelection = self.batchInputApplet.topLevelOperator
batch_input_ready = predictions_ready and \
len(opBatchDataSelection.ImageGroup) > 0
self._shell.setAppletEnabled(self.batchResultsApplet, batch_input_ready)
# Lastly, check for certain "busy" conditions, during which we
# should prevent the shell from closing the project.
busy = False
busy |= self.dataSelectionApplet.busy
busy |= self.featureSelectionApplet.busy
busy |= self.dataExportApplet.busy
self._shell.enableProjectChanges( not busy )
def getHeadlessOutputSlot(self, slotId):
# "Regular" (i.e. with the images that the user selected as input data)
if slotId == "Predictions":
return self.pcApplet.topLevelOperator.HeadlessPredictionProbabilities
elif slotId == "PredictionsUint8":
return self.pcApplet.topLevelOperator.HeadlessUint8PredictionProbabilities
# "Batch" (i.e. with the images that the user selected as batch inputs).
elif slotId == "BatchPredictions":
return self.opBatchPredictionPipeline.HeadlessPredictionProbabilities
if slotId == "BatchPredictionsUint8":
return self.opBatchPredictionPipeline.HeadlessUint8PredictionProbabilities
raise Exception("Unknown headless output slot")
def onProjectLoaded(self, projectManager):
"""
Overridden from Workflow base class. Called by the Project Manager.
If the user provided command-line arguments, use them to configure
the workflow for batch mode and export all results.
(This workflow's headless mode supports only batch mode for now.)
"""
# Configure the batch data selection operator.
if self._batch_input_args and self._batch_input_args.input_files:
self.batchInputApplet.configure_operator_with_parsed_args( self._batch_input_args )
# Configure the data export operator.
if self._batch_export_args:
self.batchResultsApplet.configure_operator_with_parsed_args( self._batch_export_args )
if self._headless and self._batch_input_args and self._batch_export_args:
# Make sure we're using the up-to-date classifier.
self.pcApplet.topLevelOperator.FreezePredictions.setValue(False)
# Now run the batch export and report progress....
opBatchDataExport = self.batchResultsApplet.topLevelOperator
for i, opExportDataLaneView in enumerate(opBatchDataExport):
print "Exporting result {} to {}".format(i, opExportDataLaneView.ExportPath.value)
sys.stdout.write( "Result {}/{} Progress: ".format( i, len( opBatchDataExport ) ) )
def print_progress( progress ):
sys.stdout.write( "{} ".format( progress ) )
# If the operator provides a progress signal, use it.
slotProgressSignal = opExportDataLaneView.progressSignal
slotProgressSignal.subscribe( print_progress )
opExportDataLaneView.run_export()
# Finished.
sys.stdout.write("\n")
def __init__(self, shell, headless, workflow_cmdline_args, appendBatchOperators=True, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super( PixelClassificationWorkflow, self ).__init__( shell, headless, graph=graph, *args, **kwargs )
self._applets = []
self._workflow_cmdline_args = workflow_cmdline_args
data_instructions = "Select your input data using the 'Raw Data' tab shown on the right"
# Parse workflow-specific command-line args
parser = argparse.ArgumentParser()
parser.add_argument('--filter', help="pixel feature filter implementation.", choices=['Original', 'Refactored', 'Interpolated'], default='Original')
parsed_args, unused_args = parser.parse_known_args(workflow_cmdline_args)
self.filter_implementation = parsed_args.filter
# Applets for training (interactive) workflow
self.projectMetadataApplet = ProjectMetadataApplet()
self.dataSelectionApplet = DataSelectionApplet( self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False,
instructionText=data_instructions )
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue( ['Raw Data'] )
self.featureSelectionApplet = FeatureSelectionApplet(self, "Feature Selection", "FeatureSelections", self.filter_implementation)
self.pcApplet = PixelClassificationApplet(self, "PixelClassification")
opClassify = self.pcApplet.topLevelOperator
self.dataExportApplet = PixelClassificationDataExportApplet(self, "Prediction Export")
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.PmapColors.connect( opClassify.PmapColors )
opDataExport.LabelNames.connect( opClassify.LabelNames )
opDataExport.WorkingDirectory.connect( opDataSelection.WorkingDirectory )
# Expose for shell
self._applets.append(self.projectMetadataApplet)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.featureSelectionApplet)
self._applets.append(self.pcApplet)
self._applets.append(self.dataExportApplet)
self._batch_input_args = None
self._batch_export_args = None
self.batchInputApplet = None
self.batchResultsApplet = None
if appendBatchOperators:
# Create applets for batch workflow
self.batchInputApplet = DataSelectionApplet(self, "Batch Prediction Input Selections", "Batch Inputs", supportIlastik05Import=False, batchDataGui=True)
self.batchResultsApplet = PixelClassificationDataExportApplet(self, "Batch Prediction Output Locations", isBatch=True)
# Expose in shell
self._applets.append(self.batchInputApplet)
self._applets.append(self.batchResultsApplet)
# Connect batch workflow (NOT lane-based)
self._initBatchWorkflow()
if unused_args:
# We parse the export setting args first. All remaining args are considered input files by the input applet.
self._batch_export_args, unused_args = self.batchResultsApplet.parse_known_cmdline_args( unused_args )
self._batch_input_args, unused_args = self.batchInputApplet.parse_known_cmdline_args( unused_args )
if unused_args:
logger.warn("Unused command-line args: {}".format( unused_args ))
class ObjectClassificationWorkflow(Workflow):
workflowName = "Object Classification Workflow Base"
defaultAppletIndex = 1 # show DataSelection by default
def __init__(self, shell, headless,
workflow_cmdline_args,
project_creation_args,
*args, **kwargs):
graph = kwargs['graph'] if 'graph' in kwargs else Graph()
if 'graph' in kwargs:
del kwargs['graph']
super(ObjectClassificationWorkflow, self).__init__(shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs)
# Parse workflow-specific command-line args
parser = argparse.ArgumentParser()
parser.add_argument('--fillmissing', help="use 'fill missing' applet with chosen detection method", choices=['classic', 'svm', 'none'], default='none')
parser.add_argument('--filter', help="pixel feature filter implementation.", choices=['Original', 'Refactored', 'Interpolated'], default='Original')
parser.add_argument('--nobatch', help="do not append batch applets", action='store_true', default=False)
parsed_creation_args, unused_args = parser.parse_known_args(project_creation_args)
self.fillMissing = parsed_creation_args.fillmissing
self.filter_implementation = parsed_creation_args.filter
parsed_args, unused_args = parser.parse_known_args(workflow_cmdline_args)
if parsed_args.fillmissing != 'none' and parsed_creation_args.fillmissing != parsed_args.fillmissing:
logger.error( "Ignoring --fillmissing cmdline arg. Can't specify a different fillmissing setting after the project has already been created." )
if parsed_args.filter != 'Original' and parsed_creation_args.filter != parsed_args.filter:
logger.error( "Ignoring --filter cmdline arg. Can't specify a different filter setting after the project has already been created." )
self.batch = not parsed_args.nobatch
self._applets = []
self.projectMetadataApplet = ProjectMetadataApplet()
self._applets.append(self.projectMetadataApplet)
self.setupInputs()
if self.fillMissing != 'none':
self.fillMissingSlicesApplet = FillMissingSlicesApplet(
self, "Fill Missing Slices", "Fill Missing Slices", self.fillMissing)
self._applets.append(self.fillMissingSlicesApplet)
if isinstance(self, ObjectClassificationWorkflowPixel):
self.input_types = 'raw'
elif isinstance(self, ObjectClassificationWorkflowBinary):
self.input_types = 'raw+binary'
elif isinstance( self, ObjectClassificationWorkflowPrediction ):
self.input_types = 'raw+pmaps'
# our main applets
self.objectExtractionApplet = ObjectExtractionApplet(workflow=self, name = "Object Feature Selection")
self.objectClassificationApplet = ObjectClassificationApplet(workflow=self)
self.dataExportApplet = ObjectClassificationDataExportApplet(self, "Object Prediction Export")
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.WorkingDirectory.connect( self.dataSelectionApplet.topLevelOperator.WorkingDirectory )
# See EXPORT_SELECTION_PREDICTIONS and EXPORT_SELECTION_PROBABILITIES, above
opDataExport.SelectionNames.setValue( ['Object Predictions', 'Object Probabilities'] )
if self.input_types == 'raw':
# Re-configure to add the pixel probabilities option
# See EXPORT_SELECTION_PIXEL_PROBABILITIES, above
opDataExport.SelectionNames.setValue( ['Object Predictions', 'Object Probabilities', 'Pixel Probabilities'] )
self._applets.append(self.objectExtractionApplet)
self._applets.append(self.objectClassificationApplet)
self._applets.append(self.dataExportApplet)
if self.batch:
self.dataSelectionAppletBatch = DataSelectionApplet(
self, "Batch Inputs", "Batch Inputs", batchDataGui=True)
self.opDataSelectionBatch = self.dataSelectionAppletBatch.topLevelOperator
if self.input_types == 'raw':
self.opDataSelectionBatch.DatasetRoles.setValue(['Raw Data'])
elif self.input_types == 'raw+binary':
self.opDataSelectionBatch.DatasetRoles.setValue(['Raw Data', 'Binary Data'])
elif self.input_types == 'raw+pmaps':
self.opDataSelectionBatch.DatasetRoles.setValue(['Raw Data', 'Prediction Maps'])
else:
assert False, "Unknown object classification subclass type."
self.blockwiseObjectClassificationApplet = BlockwiseObjectClassificationApplet(
self, "Blockwise Object Classification", "Blockwise Object Classification")
self._applets.append(self.blockwiseObjectClassificationApplet)
self.batchExportApplet = ObjectClassificationDataExportApplet(
self, "Batch Object Prediction Export", isBatch=True)
opBatchDataExport = self.batchExportApplet.topLevelOperator
opBatchDataExport.WorkingDirectory.connect( self.dataSelectionApplet.topLevelOperator.WorkingDirectory )
self._applets.append(self.dataSelectionAppletBatch)
self._applets.append(self.batchExportApplet)
self._initBatchWorkflow()
if unused_args:
# Additional export args (specific to the object classification workflow)
export_arg_parser = argparse.ArgumentParser()
export_arg_parser.add_argument( "--table_filename", help="The location to export the object feature/prediction CSV file.", required=False )
export_arg_parser.add_argument( "--export_object_prediction_img", action="store_true" )
export_arg_parser.add_argument( "--export_object_probability_img", action="store_true" )
# TODO: Support this, too, someday?
#export_arg_parser.add_argument( "--export_object_label_img", action="store_true" )
if self.input_types == 'raw':
export_arg_parser.add_argument( "--export_pixel_probability_img", action="store_true" )
self._export_args, unused_args = export_arg_parser.parse_known_args(unused_args)
# We parse the export setting args first. All remaining args are considered input files by the input applet.
self._batch_export_args, unused_args = self.batchExportApplet.parse_known_cmdline_args( unused_args )
self._batch_input_args, unused_args = self.dataSelectionAppletBatch.parse_known_cmdline_args( unused_args )
if unused_args:
warnings.warn("Unused command-line args: {}".format( unused_args ))
@property
def applets(self):
return self._applets
@property
def imageNameListSlot(self):
return self.dataSelectionApplet.topLevelOperator.ImageName
def connectLane(self, laneIndex):
rawslot, binaryslot = self.connectInputs(laneIndex)
opData = self.dataSelectionApplet.topLevelOperator.getLane(laneIndex)
opObjExtraction = self.objectExtractionApplet.topLevelOperator.getLane(laneIndex)
opObjClassification = self.objectClassificationApplet.topLevelOperator.getLane(laneIndex)
opDataExport = self.dataExportApplet.topLevelOperator.getLane(laneIndex)
opObjExtraction.RawImage.connect(rawslot)
opObjExtraction.BinaryImage.connect(binaryslot)
opObjClassification.RawImages.connect(rawslot)
opObjClassification.LabelsAllowedFlags.connect(opData.AllowLabels)
opObjClassification.BinaryImages.connect(binaryslot)
opObjClassification.SegmentationImages.connect(opObjExtraction.LabelImage)
opObjClassification.ObjectFeatures.connect(opObjExtraction.RegionFeatures)
opObjClassification.ComputedFeatureNames.connect(opObjExtraction.ComputedFeatureNames)
# Data Export connections
opDataExport.RawData.connect( opData.ImageGroup[0] )
opDataExport.RawDatasetInfo.connect( opData.DatasetGroup[0] )
opDataExport.Inputs.resize(2)
opDataExport.Inputs[EXPORT_SELECTION_PREDICTIONS].connect( opObjClassification.UncachedPredictionImages )
opDataExport.Inputs[EXPORT_SELECTION_PROBABILITIES].connect( opObjClassification.ProbabilityChannelImage )
if self.input_types == 'raw':
# Append the prediction probabilities to the list of slots that can be exported.
opDataExport.Inputs.resize(3)
# Pull from this slot since the data has already been through the Op5 operator
# (All data in the export operator must have matching spatial dimensions.)
opThreshold = self.thresholdingApplet.topLevelOperator.getLane(laneIndex)
opDataExport.Inputs[EXPORT_SELECTION_PIXEL_PROBABILITIES].connect( opThreshold.InputImage )
if self.batch:
opObjClassification = self.objectClassificationApplet.topLevelOperator.getLane(laneIndex)
opBlockwiseObjectClassification = self.blockwiseObjectClassificationApplet.topLevelOperator.getLane(laneIndex)
opBlockwiseObjectClassification.RawImage.connect(opObjClassification.RawImages)
opBlockwiseObjectClassification.BinaryImage.connect(opObjClassification.BinaryImages)
opBlockwiseObjectClassification.Classifier.connect(opObjClassification.Classifier)
opBlockwiseObjectClassification.LabelsCount.connect(opObjClassification.NumLabels)
opBlockwiseObjectClassification.SelectedFeatures.connect(opObjClassification.SelectedFeatures)
def _initBatchWorkflow(self):
# Access applet operators from the training workflow
opObjectTrainingTopLevel = self.objectClassificationApplet.topLevelOperator
opBlockwiseObjectClassification = self.blockwiseObjectClassificationApplet.topLevelOperator
# If we are not in the binary workflow, connect the thresholding operator.
# Parameter inputs are cloned from the interactive workflow,
if isinstance(self, ObjectClassificationWorkflowBinary):
#FIXME
pass
else:
opInteractiveThreshold = self.thresholdingApplet.topLevelOperator
opBatchThreshold = OperatorWrapper(OpThresholdTwoLevels, parent=self)
opBatchThreshold.MinSize.connect(opInteractiveThreshold.MinSize)
opBatchThreshold.MaxSize.connect(opInteractiveThreshold.MaxSize)
opBatchThreshold.HighThreshold.connect(opInteractiveThreshold.HighThreshold)
opBatchThreshold.LowThreshold.connect(opInteractiveThreshold.LowThreshold)
opBatchThreshold.SingleThreshold.connect(opInteractiveThreshold.SingleThreshold)
opBatchThreshold.SmootherSigma.connect(opInteractiveThreshold.SmootherSigma)
opBatchThreshold.Channel.connect(opInteractiveThreshold.Channel)
opBatchThreshold.CurOperator.connect(opInteractiveThreshold.CurOperator)
# OpDataSelectionGroup.ImageGroup is indexed by [laneIndex][roleIndex],
# but we need a slot that is indexed by [roleIndex][laneIndex]
# so we can pass each role to the appropriate slots.
# We use OpTransposeSlots to do this.
opBatchInputByRole = OpTransposeSlots( parent=self )
opBatchInputByRole.Inputs.connect( self.opDataSelectionBatch.ImageGroup )
opBatchInputByRole.OutputLength.setValue(2)
# Lane-indexed multislot for raw data
batchInputsRaw = opBatchInputByRole.Outputs[0]
# Lane-indexed multislot for binary/prediction-map data
batchInputsOther = opBatchInputByRole.Outputs[1]
# Connect the blockwise classification operator
# Parameter inputs are cloned from the interactive workflow,
opBatchClassify = OperatorWrapper(OpBlockwiseObjectClassification, parent=self,
promotedSlotNames=['RawImage', 'BinaryImage'])
opBatchClassify.Classifier.connect(opObjectTrainingTopLevel.Classifier)
opBatchClassify.LabelsCount.connect(opObjectTrainingTopLevel.NumLabels)
opBatchClassify.SelectedFeatures.connect(opObjectTrainingTopLevel.SelectedFeatures)
opBatchClassify.BlockShape3dDict.connect(opBlockwiseObjectClassification.BlockShape3dDict)
opBatchClassify.HaloPadding3dDict.connect(opBlockwiseObjectClassification.HaloPadding3dDict)
# but image pathway is from the batch pipeline
op5Raw = OperatorWrapper(OpReorderAxes, parent=self)
if self.fillMissing != 'none':
opBatchFillMissingSlices = OperatorWrapper(OpFillMissingSlicesNoCache, parent=self)
opBatchFillMissingSlices.Input.connect(batchInputsRaw)
op5Raw.Input.connect(opBatchFillMissingSlices.Output)
else:
op5Raw.Input.connect(batchInputsRaw)
op5Binary = OperatorWrapper(OpReorderAxes, parent=self)
if self.input_types != 'raw+binary':
op5Pred = OperatorWrapper(OpReorderAxes, parent=self)
op5Pred.Input.connect(batchInputsOther)
opBatchThreshold.RawInput.connect(op5Raw.Output)
opBatchThreshold.InputImage.connect(op5Pred.Output)
op5Binary.Input.connect(opBatchThreshold.Output)
else:
op5Binary.Input.connect(batchInputsOther)
opBatchClassify.RawImage.connect(op5Raw.Output)
opBatchClassify.BinaryImage.connect(op5Binary.Output)
self.opBatchClassify = opBatchClassify
# We need to transpose the dataset group, because it is indexed by [image_index][group_index]
# But we want it to be indexed by [group_index][image_index] for the RawDatasetInfo connection, below.
opTransposeDatasetGroup = OpTransposeSlots( parent=self )
opTransposeDatasetGroup.OutputLength.setValue(1)
opTransposeDatasetGroup.Inputs.connect( self.opDataSelectionBatch.DatasetGroup )
# Connect the batch OUTPUT applet
opBatchExport = self.batchExportApplet.topLevelOperator
opBatchExport.RawData.connect( batchInputsRaw )
opBatchExport.RawDatasetInfo.connect( opTransposeDatasetGroup.Outputs[0] )
# See EXPORT_SELECTION_PREDICTIONS and EXPORT_SELECTION_PROBABILITIES, above
opBatchExport.SelectionNames.setValue( ['Object Predictions', 'Object Probabilities'] )
# opBatchResults.Inputs is indexed by [lane][selection],
# Use OpTranspose to allow connection.
opTransposeBatchInputs = OpTransposeSlots( parent=self )
opTransposeBatchInputs.OutputLength.setValue(0)
opTransposeBatchInputs.Inputs.resize(2)
opTransposeBatchInputs.Inputs[EXPORT_SELECTION_PREDICTIONS].connect( opBatchClassify.PredictionImage ) # selection 0
opTransposeBatchInputs.Inputs[EXPORT_SELECTION_PROBABILITIES].connect( opBatchClassify.ProbabilityChannelImage ) # selection 1
# Now opTransposeBatchInputs.Outputs is level-2 indexed by [lane][selection]
opBatchExport.Inputs.connect( opTransposeBatchInputs.Outputs )
def onProjectLoaded(self, projectManager):
if self._headless and self._batch_input_args and self._batch_export_args:
# Check for problems: Is the project file ready to use?
opObjClassification = self.objectClassificationApplet.topLevelOperator
if not opObjClassification.Classifier.ready():
logger.error( "Can't run batch prediction.\n"
"Couldn't obtain a classifier from your project file: {}.\n"
"Please make sure your project is fully configured with a trained classifier."
.format(projectManager.currentProjectPath) )
return
# Configure the batch data selection operator.
if self._batch_input_args and self._batch_input_args.raw_data:
self.dataSelectionAppletBatch.configure_operator_with_parsed_args( self._batch_input_args )
# Configure the data export operator.
if self._batch_export_args:
self.batchExportApplet.configure_operator_with_parsed_args( self._batch_export_args )
self.opBatchClassify.BlockShape3dDict.disconnect()
# For each BATCH lane...
for lane_index, opBatchClassifyView in enumerate(self.opBatchClassify):
# Force the block size to be the same as image size (1 big block)
tagged_shape = opBatchClassifyView.RawImage.meta.getTaggedShape()
try:
tagged_shape.pop('t')
except KeyError:
pass
try:
tagged_shape.pop('c')
except KeyError:
pass
opBatchClassifyView.BlockShape3dDict.setValue( tagged_shape )
# For now, we force the entire result to be computed as one big block.
# Force the batch classify op to create an internal pipeline for our block.
opBatchClassifyView._ensurePipelineExists( (0,0,0,0,0) )
opSingleBlockClassify = opBatchClassifyView._blockPipelines[(0,0,0,0,0)]
# Export the images (if any)
if self.input_types == 'raw':
# If pixel probabilities need export, do that first.
# (They are needed by the other outputs, anyway)
if self._export_args.export_pixel_probability_img:
self._export_batch_image( lane_index, EXPORT_SELECTION_PIXEL_PROBABILITIES, 'pixel-probability-img' )
if self._export_args.export_object_prediction_img:
self._export_batch_image( lane_index, EXPORT_SELECTION_PREDICTIONS, 'object-prediction-img' )
if self._export_args.export_object_probability_img:
self._export_batch_image( lane_index, EXPORT_SELECTION_PROBABILITIES, 'object-probability-img' )
# Export the CSV
csv_filename = self._export_args.table_filename
if csv_filename:
feature_table = opSingleBlockClassify._opPredict.createExportTable([])
if len(self.opBatchClassify) > 1:
base, ext = os.path.splitext( csv_filename )
csv_filename = base + '-' + str(lane_index) + ext
print "Exporting object table for image #{}:\n{}".format( lane_index, csv_filename )
self.record_array_to_csv(feature_table, csv_filename)
print "FINISHED."
def _export_batch_image(self, lane_index, selection_index, selection_name):
opBatchExport = self.batchExportApplet.topLevelOperator
opBatchExport.InputSelection.setValue(selection_index)
opBatchExportView = opBatchExport.getLane(lane_index)
# Remember this so we can restore it later
default_output_path = opBatchExport.OutputFilenameFormat.value
export_path = opBatchExportView.ExportPath.value
path_comp = PathComponents( export_path, os.getcwd() )
path_comp.filenameBase += '-' + selection_name
opBatchExport.OutputFilenameFormat.setValue( path_comp.externalPath )
logger.info( "Exporting {} for image #{} to {}"
.format(selection_name, lane_index+1, opBatchExportView.ExportPath.value) )
sys.stdout.write( "Result {}/{} Progress: "
.format( lane_index+1, len( self.opBatchClassify ) ) )
sys.stdout.flush()
def print_progress( progress ):
sys.stdout.write( "{} ".format( progress ) )
sys.stdout.flush()
# If the operator provides a progress signal, use it.
slotProgressSignal = opBatchExportView.progressSignal
slotProgressSignal.subscribe( print_progress )
opBatchExportView.run_export()
# Finished.
sys.stdout.write("\n")
# Restore original format
opBatchExport.OutputFilenameFormat.setValue( default_output_path )
def record_array_to_csv(self, record_array, filename):
"""
Save the given record array to a CSV file.
"""
# Sort by offset
with open(filename, 'w') as csv_file:
sorted_fields = sorted( record_array.dtype.fields.items(), key=lambda (k,v): v[1] )
field_names = map( lambda (k,v): k, sorted_fields )
for name in field_names:
# Remove any commas in the header (this is csv, after all)
name = name.replace(',', '/')
csv_file.write(name + ',')
csv_file.write('\n')
for row in record_array:
for name in field_names:
csv_file.write(str(row[name]) + ',')
csv_file.write('\n')
def getHeadlessOutputSlot(self, slotId):
if slotId == "BatchPredictionImage":
return self.opBatchClassify.PredictionImage
raise Exception("Unknown headless output slot")
def getSecondaryHeadlessOutputSlots(self, slotId):
if slotId == "BatchPredictionImage":
return [self.opBatchClassify.BlockwiseRegionFeatures]
raise Exception("Unknown headless output slot")
def handleAppletStateUpdateRequested(self, upstream_ready=False):
"""
Overridden from Workflow base class
Called when an applet has fired the :py:attr:`Applet.appletStateUpdateRequested`
This method will be called by the child classes with the result of their
own applet readyness findings as keyword argument.
"""
# all workflows have these applets in common:
# object feature selection
# object classification
# object prediction export
# blockwise classification
# batch input
# batch prediction export
cumulated_readyness = upstream_ready
self._shell.setAppletEnabled(self.objectExtractionApplet, cumulated_readyness)
if len(self.objectExtractionApplet.topLevelOperator.ComputedFeatureNames) == 0:
object_features_ready = False
else:
object_features_ready = True
for slot in self.objectExtractionApplet.topLevelOperator.ComputedFeatureNames:
object_features_ready = object_features_ready and len(slot.value) > 0
#object_features_ready = self.objectExtractionApplet.topLevelOperator.RegionFeatures.ready()
cumulated_readyness = cumulated_readyness and object_features_ready
self._shell.setAppletEnabled(self.objectClassificationApplet, cumulated_readyness)
object_classification_ready = \
self.objectClassificationApplet.predict_enabled
cumulated_readyness = cumulated_readyness and object_classification_ready
self._shell.setAppletEnabled(self.dataExportApplet, cumulated_readyness)
if self.batch:
object_prediction_ready = True # TODO is that so?
cumulated_readyness = cumulated_readyness and object_prediction_ready
self._shell.setAppletEnabled(self.blockwiseObjectClassificationApplet, cumulated_readyness)
self._shell.setAppletEnabled(self.dataSelectionAppletBatch, cumulated_readyness)
self._shell.setAppletEnabled(self.batchExportApplet, cumulated_readyness)
# Lastly, check for certain "busy" conditions, during which we
# should prevent the shell from closing the project.
#TODO implement
busy = False
self._shell.enableProjectChanges( not busy )
def _inputReady(self, nRoles):
slot = self.dataSelectionApplet.topLevelOperator.ImageGroup
if len(slot) > 0:
input_ready = True
for sub in slot:
input_ready = input_ready and \
all([sub[i].ready() for i in range(nRoles)])
else:
input_ready = False
return input_ready
class ObjectClassificationWorkflow(Workflow):
workflowName = "Object Classification Workflow Base"
defaultAppletIndex = 1 # show DataSelection by default
def __init__(self, shell, headless,
workflow_cmdline_args,
project_creation_args,
*args, **kwargs):
graph = kwargs['graph'] if 'graph' in kwargs else Graph()
if 'graph' in kwargs:
del kwargs['graph']
super(ObjectClassificationWorkflow, self).__init__(shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs)
# Parse workflow-specific command-line args
parser = argparse.ArgumentParser()
parser.add_argument('--fillmissing', help="use 'fill missing' applet with chosen detection method", choices=['classic', 'svm', 'none'], default='none')
parser.add_argument('--filter', help="pixel feature filter implementation.", choices=['Original', 'Refactored', 'Interpolated'], default='Original')
parser.add_argument('--nobatch', help="do not append batch applets", action='store_true', default=False)
parsed_creation_args, unused_args = parser.parse_known_args(project_creation_args)
self.fillMissing = parsed_creation_args.fillmissing
self.filter_implementation = parsed_creation_args.filter
parsed_args, unused_args = parser.parse_known_args(workflow_cmdline_args)
if parsed_args.fillmissing != 'none' and parsed_creation_args.fillmissing != parsed_args.fillmissing:
logger.error( "Ignoring --fillmissing cmdline arg. Can't specify a different fillmissing setting after the project has already been created." )
if parsed_args.filter != 'Original' and parsed_creation_args.filter != parsed_args.filter:
logger.error( "Ignoring --filter cmdline arg. Can't specify a different filter setting after the project has already been created." )
self.batch = not parsed_args.nobatch
self._applets = []
self.projectMetadataApplet = ProjectMetadataApplet()
self._applets.append(self.projectMetadataApplet)
self.setupInputs()
if self.fillMissing != 'none':
self.fillMissingSlicesApplet = FillMissingSlicesApplet(
self, "Fill Missing Slices", "Fill Missing Slices", self.fillMissing)
self._applets.append(self.fillMissingSlicesApplet)
if isinstance(self, ObjectClassificationWorkflowPixel):
self.input_types = 'raw'
elif isinstance(self, ObjectClassificationWorkflowBinary):
self.input_types = 'raw+binary'
elif isinstance( self, ObjectClassificationWorkflowPrediction ):
self.input_types = 'raw+pmaps'
# our main applets
self.objectExtractionApplet = ObjectExtractionApplet(workflow=self, name = "Object Feature Selection")
self.objectClassificationApplet = ObjectClassificationApplet(workflow=self)
self.dataExportApplet = ObjectClassificationDataExportApplet(self, "Object Information Export")
self.dataExportApplet.set_exporting_operator(self.objectClassificationApplet.topLevelOperator)
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.WorkingDirectory.connect( self.dataSelectionApplet.topLevelOperator.WorkingDirectory )
# See EXPORT_SELECTION_PREDICTIONS and EXPORT_SELECTION_PROBABILITIES, above
opDataExport.SelectionNames.setValue( ['Object Predictions', 'Object Probabilities'] )
if self.input_types == 'raw':
# Re-configure to add the pixel probabilities option
# See EXPORT_SELECTION_PIXEL_PROBABILITIES, above
opDataExport.SelectionNames.setValue( ['Object Predictions', 'Object Probabilities', 'Pixel Probabilities'] )
self._applets.append(self.objectExtractionApplet)
self._applets.append(self.objectClassificationApplet)
self._applets.append(self.dataExportApplet)
if self.batch:
self.dataSelectionAppletBatch = DataSelectionApplet(
self, "Batch Inputs", "Batch Inputs", batchDataGui=True)
self.opDataSelectionBatch = self.dataSelectionAppletBatch.topLevelOperator
if self.input_types == 'raw':
self.opDataSelectionBatch.DatasetRoles.setValue(['Raw Data'])
elif self.input_types == 'raw+binary':
self.opDataSelectionBatch.DatasetRoles.setValue(['Raw Data', 'Binary Data'])
elif self.input_types == 'raw+pmaps':
self.opDataSelectionBatch.DatasetRoles.setValue(['Raw Data', 'Prediction Maps'])
else:
assert False, "Unknown object classification subclass type."
self.blockwiseObjectClassificationApplet = BlockwiseObjectClassificationApplet(
self, "Blockwise Object Classification", "Blockwise Object Classification")
self._applets.append(self.blockwiseObjectClassificationApplet)
self.batchExportApplet = ObjectClassificationDataExportApplet(
self, "Batch Object Prediction Export", isBatch=True)
opBatchDataExport = self.batchExportApplet.topLevelOperator
opBatchDataExport.WorkingDirectory.connect( self.dataSelectionApplet.topLevelOperator.WorkingDirectory )
self._applets.append(self.dataSelectionAppletBatch)
self._applets.append(self.batchExportApplet)
self._initBatchWorkflow()
self._batch_export_args = None
self._batch_input_args = None
if unused_args:
# Additional export args (specific to the object classification workflow)
export_arg_parser = argparse.ArgumentParser()
export_arg_parser.add_argument( "--table_filename", help="The location to export the object feature/prediction CSV file.", required=False )
export_arg_parser.add_argument( "--export_object_prediction_img", action="store_true" )
export_arg_parser.add_argument( "--export_object_probability_img", action="store_true" )
# TODO: Support this, too, someday?
#export_arg_parser.add_argument( "--export_object_label_img", action="store_true" )
if self.input_types == 'raw':
export_arg_parser.add_argument( "--export_pixel_probability_img", action="store_true" )
self._export_args, unused_args = export_arg_parser.parse_known_args(unused_args)
# We parse the export setting args first. All remaining args are considered input files by the input applet.
self._batch_export_args, unused_args = self.batchExportApplet.parse_known_cmdline_args( unused_args )
self._batch_input_args, unused_args = self.dataSelectionAppletBatch.parse_known_cmdline_args( unused_args )
if unused_args:
warnings.warn("Unused command-line args: {}".format( unused_args ))
@property
def applets(self):
return self._applets
@property
def imageNameListSlot(self):
return self.dataSelectionApplet.topLevelOperator.ImageName
def connectLane(self, laneIndex):
rawslot, binaryslot = self.connectInputs(laneIndex)
opData = self.dataSelectionApplet.topLevelOperator.getLane(laneIndex)
opObjExtraction = self.objectExtractionApplet.topLevelOperator.getLane(laneIndex)
opObjClassification = self.objectClassificationApplet.topLevelOperator.getLane(laneIndex)
opDataExport = self.dataExportApplet.topLevelOperator.getLane(laneIndex)
opObjExtraction.RawImage.connect(rawslot)
opObjExtraction.BinaryImage.connect(binaryslot)
opObjClassification.RawImages.connect(rawslot)
opObjClassification.LabelsAllowedFlags.connect(opData.AllowLabels)
opObjClassification.BinaryImages.connect(binaryslot)
opObjClassification.SegmentationImages.connect(opObjExtraction.LabelImage)
opObjClassification.ObjectFeatures.connect(opObjExtraction.RegionFeatures)
opObjClassification.ComputedFeatureNames.connect(opObjExtraction.ComputedFeatureNames)
# Data Export connections
opDataExport.RawData.connect( opData.ImageGroup[0] )
opDataExport.RawDatasetInfo.connect( opData.DatasetGroup[0] )
opDataExport.Inputs.resize(2)
opDataExport.Inputs[EXPORT_SELECTION_PREDICTIONS].connect( opObjClassification.UncachedPredictionImages )
opDataExport.Inputs[EXPORT_SELECTION_PROBABILITIES].connect( opObjClassification.ProbabilityChannelImage )
if self.input_types == 'raw':
# Append the prediction probabilities to the list of slots that can be exported.
opDataExport.Inputs.resize(3)
# Pull from this slot since the data has already been through the Op5 operator
# (All data in the export operator must have matching spatial dimensions.)
opThreshold = self.thresholdingApplet.topLevelOperator.getLane(laneIndex)
opDataExport.Inputs[EXPORT_SELECTION_PIXEL_PROBABILITIES].connect( opThreshold.InputImage )
if self.batch:
opObjClassification = self.objectClassificationApplet.topLevelOperator.getLane(laneIndex)
opBlockwiseObjectClassification = self.blockwiseObjectClassificationApplet.topLevelOperator.getLane(laneIndex)
opBlockwiseObjectClassification.RawImage.connect(opObjClassification.RawImages)
opBlockwiseObjectClassification.BinaryImage.connect(opObjClassification.BinaryImages)
opBlockwiseObjectClassification.Classifier.connect(opObjClassification.Classifier)
opBlockwiseObjectClassification.LabelsCount.connect(opObjClassification.NumLabels)
opBlockwiseObjectClassification.SelectedFeatures.connect(opObjClassification.SelectedFeatures)
def _initBatchWorkflow(self):
# Access applet operators from the training workflow
opObjectTrainingTopLevel = self.objectClassificationApplet.topLevelOperator
opBlockwiseObjectClassification = self.blockwiseObjectClassificationApplet.topLevelOperator
# If we are not in the binary workflow, connect the thresholding operator.
# Parameter inputs are cloned from the interactive workflow,
if isinstance(self, ObjectClassificationWorkflowBinary):
#FIXME
pass
else:
opInteractiveThreshold = self.thresholdingApplet.topLevelOperator
opBatchThreshold = OperatorWrapper(OpThresholdTwoLevels, parent=self)
opBatchThreshold.MinSize.connect(opInteractiveThreshold.MinSize)
opBatchThreshold.MaxSize.connect(opInteractiveThreshold.MaxSize)
opBatchThreshold.HighThreshold.connect(opInteractiveThreshold.HighThreshold)
opBatchThreshold.LowThreshold.connect(opInteractiveThreshold.LowThreshold)
opBatchThreshold.SingleThreshold.connect(opInteractiveThreshold.SingleThreshold)
opBatchThreshold.SmootherSigma.connect(opInteractiveThreshold.SmootherSigma)
opBatchThreshold.Channel.connect(opInteractiveThreshold.Channel)
opBatchThreshold.CurOperator.connect(opInteractiveThreshold.CurOperator)
# OpDataSelectionGroup.ImageGroup is indexed by [laneIndex][roleIndex],
# but we need a slot that is indexed by [roleIndex][laneIndex]
# so we can pass each role to the appropriate slots.
# We use OpTransposeSlots to do this.
opBatchInputByRole = OpTransposeSlots( parent=self )
opBatchInputByRole.Inputs.connect( self.opDataSelectionBatch.ImageGroup )
opBatchInputByRole.OutputLength.setValue(2)
# Lane-indexed multislot for raw data
batchInputsRaw = opBatchInputByRole.Outputs[0]
# Lane-indexed multislot for binary/prediction-map data
batchInputsOther = opBatchInputByRole.Outputs[1]
# Connect the blockwise classification operator
# Parameter inputs are cloned from the interactive workflow,
opBatchClassify = OperatorWrapper(OpBlockwiseObjectClassification, parent=self,
promotedSlotNames=['RawImage', 'BinaryImage'])
opBatchClassify.Classifier.connect(opObjectTrainingTopLevel.Classifier)
opBatchClassify.LabelsCount.connect(opObjectTrainingTopLevel.NumLabels)
opBatchClassify.SelectedFeatures.connect(opObjectTrainingTopLevel.SelectedFeatures)
opBatchClassify.BlockShape3dDict.connect(opBlockwiseObjectClassification.BlockShape3dDict)
opBatchClassify.HaloPadding3dDict.connect(opBlockwiseObjectClassification.HaloPadding3dDict)
# but image pathway is from the batch pipeline
op5Raw = OperatorWrapper(OpReorderAxes, parent=self)
if self.fillMissing != 'none':
opBatchFillMissingSlices = OperatorWrapper(OpFillMissingSlicesNoCache, parent=self)
opBatchFillMissingSlices.Input.connect(batchInputsRaw)
op5Raw.Input.connect(opBatchFillMissingSlices.Output)
else:
op5Raw.Input.connect(batchInputsRaw)
op5Binary = OperatorWrapper(OpReorderAxes, parent=self)
if self.input_types != 'raw+binary':
op5Pred = OperatorWrapper(OpReorderAxes, parent=self)
op5Pred.Input.connect(batchInputsOther)
opBatchThreshold.RawInput.connect(op5Raw.Output)
opBatchThreshold.InputImage.connect(op5Pred.Output)
op5Binary.Input.connect(opBatchThreshold.Output)
else:
op5Binary.Input.connect(batchInputsOther)
opBatchClassify.RawImage.connect(op5Raw.Output)
opBatchClassify.BinaryImage.connect(op5Binary.Output)
self.opBatchClassify = opBatchClassify
# We need to transpose the dataset group, because it is indexed by [image_index][group_index]
# But we want it to be indexed by [group_index][image_index] for the RawDatasetInfo connection, below.
opTransposeDatasetGroup = OpTransposeSlots( parent=self )
opTransposeDatasetGroup.OutputLength.setValue(1)
opTransposeDatasetGroup.Inputs.connect( self.opDataSelectionBatch.DatasetGroup )
# Connect the batch OUTPUT applet
opBatchExport = self.batchExportApplet.topLevelOperator
opBatchExport.RawData.connect( batchInputsRaw )
opBatchExport.RawDatasetInfo.connect( opTransposeDatasetGroup.Outputs[0] )
# See EXPORT_SELECTION_PREDICTIONS and EXPORT_SELECTION_PROBABILITIES, above
opBatchExport.SelectionNames.setValue( ['Object Predictions', 'Object Probabilities'] )
# opBatchResults.Inputs is indexed by [lane][selection],
# Use OpTranspose to allow connection.
opTransposeBatchInputs = OpTransposeSlots( parent=self )
opTransposeBatchInputs.OutputLength.setValue(0)
opTransposeBatchInputs.Inputs.resize(2)
opTransposeBatchInputs.Inputs[EXPORT_SELECTION_PREDICTIONS].connect( opBatchClassify.PredictionImage ) # selection 0
opTransposeBatchInputs.Inputs[EXPORT_SELECTION_PROBABILITIES].connect( opBatchClassify.ProbabilityChannelImage ) # selection 1
# Now opTransposeBatchInputs.Outputs is level-2 indexed by [lane][selection]
opBatchExport.Inputs.connect( opTransposeBatchInputs.Outputs )
def onProjectLoaded(self, projectManager):
if self._headless and self._batch_input_args and self._batch_export_args:
# Check for problems: Is the project file ready to use?
opObjClassification = self.objectClassificationApplet.topLevelOperator
if not opObjClassification.Classifier.ready():
logger.error( "Can't run batch prediction.\n"
"Couldn't obtain a classifier from your project file: {}.\n"
"Please make sure your project is fully configured with a trained classifier."
.format(projectManager.currentProjectPath) )
return
# Configure the batch data selection operator.
if self._batch_input_args and self._batch_input_args.raw_data:
self.dataSelectionAppletBatch.configure_operator_with_parsed_args( self._batch_input_args )
# Configure the data export operator.
if self._batch_export_args:
self.batchExportApplet.configure_operator_with_parsed_args( self._batch_export_args )
self.opBatchClassify.BlockShape3dDict.disconnect()
# For each BATCH lane...
for lane_index, opBatchClassifyView in enumerate(self.opBatchClassify):
# Force the block size to be the same as image size (1 big block)
tagged_shape = opBatchClassifyView.RawImage.meta.getTaggedShape()
try:
tagged_shape.pop('t')
except KeyError:
pass
try:
tagged_shape.pop('c')
except KeyError:
pass
opBatchClassifyView.BlockShape3dDict.setValue( tagged_shape )
# For now, we force the entire result to be computed as one big block.
# Force the batch classify op to create an internal pipeline for our block.
opBatchClassifyView._ensurePipelineExists( (0,0,0,0,0) )
opSingleBlockClassify = opBatchClassifyView._blockPipelines[(0,0,0,0,0)]
# Export the images (if any)
if self.input_types == 'raw':
# If pixel probabilities need export, do that first.
# (They are needed by the other outputs, anyway)
if self._export_args.export_pixel_probability_img:
self._export_batch_image( lane_index, EXPORT_SELECTION_PIXEL_PROBABILITIES, 'pixel-probability-img' )
if self._export_args.export_object_prediction_img:
self._export_batch_image( lane_index, EXPORT_SELECTION_PREDICTIONS, 'object-prediction-img' )
if self._export_args.export_object_probability_img:
self._export_batch_image( lane_index, EXPORT_SELECTION_PROBABILITIES, 'object-probability-img' )
# Export the CSV
csv_filename = self._export_args.table_filename
if csv_filename:
feature_table = opSingleBlockClassify._opPredict.createExportTable([])
if len(self.opBatchClassify) > 1:
base, ext = os.path.splitext( csv_filename )
csv_filename = base + '-' + str(lane_index) + ext
print "Exporting object table for image #{}:\n{}".format( lane_index, csv_filename )
self.record_array_to_csv(feature_table, csv_filename)
print "FINISHED."
def _export_batch_image(self, lane_index, selection_index, selection_name):
opBatchExport = self.batchExportApplet.topLevelOperator
opBatchExport.InputSelection.setValue(selection_index)
opBatchExportView = opBatchExport.getLane(lane_index)
# Remember this so we can restore it later
default_output_path = opBatchExport.OutputFilenameFormat.value
export_path = opBatchExportView.ExportPath.value
path_comp = PathComponents( export_path, os.getcwd() )
path_comp.filenameBase += '-' + selection_name
opBatchExport.OutputFilenameFormat.setValue( path_comp.externalPath )
logger.info( "Exporting {} for image #{} to {}"
.format(selection_name, lane_index+1, opBatchExportView.ExportPath.value) )
sys.stdout.write( "Result {}/{} Progress: "
.format( lane_index+1, len( self.opBatchClassify ) ) )
sys.stdout.flush()
def print_progress( progress ):
sys.stdout.write( "{} ".format( progress ) )
sys.stdout.flush()
# If the operator provides a progress signal, use it.
slotProgressSignal = opBatchExportView.progressSignal
slotProgressSignal.subscribe( print_progress )
opBatchExportView.run_export()
# Finished.
sys.stdout.write("\n")
# Restore original format
opBatchExport.OutputFilenameFormat.setValue( default_output_path )
def record_array_to_csv(self, record_array, filename):
"""
Save the given record array to a CSV file.
"""
# Sort by offset
with open(filename, 'w') as csv_file:
sorted_fields = sorted( record_array.dtype.fields.items(), key=lambda (k,v): v[1] )
field_names = map( lambda (k,v): k, sorted_fields )
for name in field_names:
# Remove any commas in the header (this is csv, after all)
name = name.replace(',', '/')
csv_file.write(name + ',')
csv_file.write('\n')
for row in record_array:
for name in field_names:
csv_file.write(str(row[name]) + ',')
csv_file.write('\n')
def getHeadlessOutputSlot(self, slotId):
if slotId == "BatchPredictionImage":
return self.opBatchClassify.PredictionImage
raise Exception("Unknown headless output slot")
def postprocessClusterSubResult(self, roi, result, blockwise_fileset):
"""
"""
# TODO: Here, we hard-code to select from the first lane only.
opBatchClassify = self.opBatchClassify[0]
from lazyflow.utility.io.blockwiseFileset import vectorized_pickle_dumps
# Assume that roi always starts as a multiple of the blockshape
block_shape = opBatchClassify.get_blockshape()
assert all(block_shape == blockwise_fileset.description.sub_block_shape), "block shapes don't match"
assert all((roi[0] % block_shape) == 0), "Sub-blocks must exactly correspond to the blockwise object classification blockshape"
sub_block_index = roi[0] / blockwise_fileset.description.sub_block_shape
sub_block_start = sub_block_index
sub_block_stop = sub_block_start + 1
sub_block_roi = (sub_block_start, sub_block_stop)
# FIRST, remove all objects that lie outside the block (i.e. remove the ones in the halo)
region_features = opBatchClassify.BlockwiseRegionFeatures( *sub_block_roi ).wait()
region_features_dict = region_features.flat[0]
region_centers = region_features_dict['Default features']['RegionCenter']
opBlockPipeline = opBatchClassify._blockPipelines[ tuple(roi[0]) ]
# Compute the block offset within the image coordinates
halo_roi = opBlockPipeline._halo_roi
translated_region_centers = region_centers + halo_roi[0][1:-1]
# TODO: If this is too slow, vectorize this
mask = numpy.zeros( region_centers.shape[0], dtype=numpy.bool_ )
for index, translated_region_center in enumerate(translated_region_centers):
# FIXME: Here we assume t=0 and c=0
mask[index] = opBatchClassify.is_in_block( roi[0], (0,) + tuple(translated_region_center) + (0,) )
# Always exclude the first object (it's the background??)
mask[0] = False
# Remove all 'negative' predictions, emit only 'positive' predictions
# FIXME: Don't hardcode this?
POSITIVE_LABEL = 2
objectwise_predictions = opBlockPipeline.ObjectwisePredictions([]).wait()[0]
assert objectwise_predictions.shape == mask.shape
mask[objectwise_predictions != POSITIVE_LABEL] = False
filtered_features = {}
for feature_group, feature_dict in region_features_dict.items():
filtered_group = filtered_features[feature_group] = {}
for feature_name, feature_array in feature_dict.items():
filtered_group[feature_name] = feature_array[mask]
# SECOND, translate from block-local coordinates to global (file) coordinates.
# Unfortunately, we've got multiple translations to perform here:
# Coordinates in the region features are relative to their own block INCLUDING HALO,
# so we need to add the start of the block-with-halo as an offset.
# BUT the image itself may be offset relative to the BlockwiseFileset coordinates
# (due to the view_origin setting), so we also need to add an offset for that, too
# Get the image offset relative to the file coordinates
image_offset = blockwise_fileset.description.view_origin
total_offset_5d = halo_roi[0] + image_offset
total_offset_3d = total_offset_5d[1:-1]
filtered_features["Default features"]["RegionCenter"] += total_offset_3d
filtered_features["Default features"]["Coord<Minimum>"] += total_offset_3d
filtered_features["Default features"]["Coord<Maximum>"] += total_offset_3d
# Finally, write the features to hdf5
h5File = blockwise_fileset.getOpenHdf5FileForBlock( roi[0] )
if 'pickled_region_features' in h5File:
del h5File['pickled_region_features']
# Must use str dtype
dtype = h5py.new_vlen(str)
dataset = h5File.create_dataset( 'pickled_region_features', shape=(1,), dtype=dtype )
pickled_features = vectorized_pickle_dumps(numpy.array((filtered_features,)))
dataset[0] = pickled_features
object_centers_xyz = filtered_features["Default features"]["RegionCenter"].astype(int)
object_min_coords_xyz = filtered_features["Default features"]["Coord<Minimum>"].astype(int)
object_max_coords_xyz = filtered_features["Default features"]["Coord<Maximum>"].astype(int)
object_sizes = filtered_features["Default features"]["Count"][:,0].astype(int)
# Also, write out selected features as a 'point cloud' csv file.
# (Store the csv file next to this block's h5 file.)
dataset_directory = blockwise_fileset.getDatasetDirectory(roi[0])
pointcloud_path = os.path.join( dataset_directory, "block-pointcloud.csv" )
logger.info("Writing to csv: {}".format( pointcloud_path ))
with open(pointcloud_path, "w") as fout:
csv_writer = csv.DictWriter(fout, OUTPUT_COLUMNS, **CSV_FORMAT)
csv_writer.writeheader()
for obj_id in range(len(object_sizes)):
fields = {}
fields["x_px"], fields["y_px"], fields["z_px"], = object_centers_xyz[obj_id]
fields["min_x_px"], fields["min_y_px"], fields["min_z_px"], = object_min_coords_xyz[obj_id]
fields["max_x_px"], fields["max_y_px"], fields["max_z_px"], = object_max_coords_xyz[obj_id]
fields["size_px"] = object_sizes[obj_id]
csv_writer.writerow( fields )
#fout.flush()
logger.info("FINISHED csv export")
def handleAppletStateUpdateRequested(self, upstream_ready=False):
"""
Overridden from Workflow base class
Called when an applet has fired the :py:attr:`Applet.appletStateUpdateRequested`
This method will be called by the child classes with the result of their
own applet readyness findings as keyword argument.
"""
# all workflows have these applets in common:
# object feature selection
# object classification
# object prediction export
# blockwise classification
# batch input
# batch prediction export
cumulated_readyness = upstream_ready
self._shell.setAppletEnabled(self.objectExtractionApplet, cumulated_readyness)
if len(self.objectExtractionApplet.topLevelOperator.ComputedFeatureNames) == 0:
object_features_ready = False
else:
object_features_ready = True
for slot in self.objectExtractionApplet.topLevelOperator.ComputedFeatureNames:
object_features_ready = object_features_ready and len(slot.value) > 0
#object_features_ready = self.objectExtractionApplet.topLevelOperator.RegionFeatures.ready()
cumulated_readyness = cumulated_readyness and object_features_ready
self._shell.setAppletEnabled(self.objectClassificationApplet, cumulated_readyness)
object_classification_ready = \
self.objectClassificationApplet.predict_enabled
cumulated_readyness = cumulated_readyness and object_classification_ready
self._shell.setAppletEnabled(self.dataExportApplet, cumulated_readyness)
if self.batch:
object_prediction_ready = True # TODO is that so?
cumulated_readyness = cumulated_readyness and object_prediction_ready
self._shell.setAppletEnabled(self.blockwiseObjectClassificationApplet, cumulated_readyness)
self._shell.setAppletEnabled(self.dataSelectionAppletBatch, cumulated_readyness)
self._shell.setAppletEnabled(self.batchExportApplet, cumulated_readyness)
# Lastly, check for certain "busy" conditions, during which we
# should prevent the shell from closing the project.
#TODO implement
busy = False
self._shell.enableProjectChanges( not busy )
def _inputReady(self, nRoles):
slot = self.dataSelectionApplet.topLevelOperator.ImageGroup
if len(slot) > 0:
input_ready = True
for sub in slot:
input_ready = input_ready and \
all([sub[i].ready() for i in range(nRoles)])
else:
input_ready = False
return input_ready
class DataConversionWorkflow(Workflow):
"""
Simple workflow for converting data between formats. Has only two applets: Data Selection and Data Export.
Also supports a command-line interface for headless mode.
For example:
.. code-block:: bash
python ilastik.py --headless --new_project=NewTemporaryProject.ilp --workflow=DataConversionWorkflow --output_format="png sequence" ~/input1.h5 ~/input2.h5
Or if you have an existing project with input files already selected and configured:
.. code-block:: bash
python ilastik.py --headless --project=MyProject.ilp --output_format=jpeg
.. note:: Beware of issues related to absolute vs. relative paths. Relative links are stored relative to the project file.
To avoid this issue entirely, either
(1) use only absolute filepaths
or (2) cd into your project file's directory before launching ilastik.
"""
def __init__(self, shell, headless, workflow_cmdline_args, project_creation_args, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super(DataConversionWorkflow, self).__init__(shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs)
self._applets = []
# Create applets
self.dataSelectionApplet = DataSelectionApplet(self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False,
force5d=False)
opDataSelection = self.dataSelectionApplet.topLevelOperator
role_names = ["Input Data"]
opDataSelection.DatasetRoles.setValue( role_names )
self.dataExportApplet = DataExportApplet(self, "Data Export")
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.WorkingDirectory.connect( opDataSelection.WorkingDirectory )
opDataExport.SelectionNames.setValue( ["Input"] )
self._applets.append( self.dataSelectionApplet )
self._applets.append( self.dataExportApplet )
# Parse command-line arguments
# Command-line args are applied in onProjectLoaded(), below.
self._workflow_cmdline_args = workflow_cmdline_args
self._data_input_args = None
self._data_export_args = None
if workflow_cmdline_args:
self._data_export_args, unused_args = self.dataExportApplet.parse_known_cmdline_args( unused_args )
self._data_input_args, unused_args = self.dataSelectionApplet.parse_known_cmdline_args( workflow_cmdline_args, role_names )
if unused_args:
logger.warn("Unused command-line args: {}".format( unused_args ))
def onProjectLoaded(self, projectManager):
"""
Overridden from Workflow base class. Called by the Project Manager.
If the user provided command-line arguments, use them to configure
the workflow inputs and output settings.
"""
# Configure the batch data selection operator.
if self._data_input_args and self._data_input_args.input_files:
self.dataSelectionApplet.configure_operator_with_parsed_args( self._data_input_args )
# Configure the data export operator.
if self._data_export_args:
self.dataExportApplet.configure_operator_with_parsed_args( self._data_export_args )
if self._headless and self._data_input_args and self._data_export_args:
# Now run the export and report progress....
opDataExport = self.dataExportApplet.topLevelOperator
for i, opExportDataLaneView in enumerate(opDataExport):
logger.info( "Exporting file #{} to {}".format(i, opExportDataLaneView.ExportPath.value) )
sys.stdout.write( "Result #{}/{} Progress: ".format( i, len( opDataExport ) ) )
def print_progress( progress ):
sys.stdout.write( "{} ".format( progress ) )
# If the operator provides a progress signal, use it.
slotProgressSignal = opExportDataLaneView.progressSignal
slotProgressSignal.subscribe( print_progress )
opExportDataLaneView.run_export()
# Finished.
sys.stdout.write("\n")
def connectLane(self, laneIndex):
opDataSelectionView = self.dataSelectionApplet.topLevelOperator.getLane(laneIndex)
opDataExportView = self.dataExportApplet.topLevelOperator.getLane(laneIndex)
opDataExportView.RawDatasetInfo.connect( opDataSelectionView.DatasetGroup[0] )
opDataExportView.Inputs.resize( 1 )
opDataExportView.Inputs[0].connect( opDataSelectionView.ImageGroup[0] )
# There is no special "raw" display layer in this workflow.
#opDataExportView.RawData.connect( opDataSelectionView.ImageGroup[0] )
@property
def applets(self):
return self._applets
@property
def imageNameListSlot(self):
return self.dataSelectionApplet.topLevelOperator.ImageName
def handleAppletStateUpdateRequested(self):
"""
Overridden from Workflow base class
Called when an applet has fired the :py:attr:`Applet.statusUpdateSignal`
"""
opDataSelection = self.dataSelectionApplet.topLevelOperator
input_ready = len(opDataSelection.ImageGroup) > 0
opDataExport = self.dataExportApplet.topLevelOperator
export_data_ready = input_ready and \
len(opDataExport.Inputs[0]) > 0 and \
opDataExport.Inputs[0][0].ready() and \
(TinyVector(opDataExport.Inputs[0][0].meta.shape) > 0).all()
self._shell.setAppletEnabled(self.dataExportApplet, export_data_ready)
# Lastly, check for certain "busy" conditions, during which we
# should prevent the shell from closing the project.
busy = False
busy |= self.dataSelectionApplet.busy
busy |= self.dataExportApplet.busy
self._shell.enableProjectChanges( not busy )
def __init__(self, shell, headless,
workflow_cmdline_args,
project_creation_args,
*args, **kwargs):
graph = kwargs['graph'] if 'graph' in kwargs else Graph()
if 'graph' in kwargs:
del kwargs['graph']
super(ObjectClassificationWorkflow, self).__init__(shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs)
# Parse workflow-specific command-line args
parser = argparse.ArgumentParser()
parser.add_argument('--fillmissing', help="use 'fill missing' applet with chosen detection method", choices=['classic', 'svm', 'none'], default='none')
parser.add_argument('--filter', help="pixel feature filter implementation.", choices=['Original', 'Refactored', 'Interpolated'], default='Original')
parser.add_argument('--nobatch', help="do not append batch applets", action='store_true', default=False)
parsed_creation_args, unused_args = parser.parse_known_args(project_creation_args)
self.fillMissing = parsed_creation_args.fillmissing
self.filter_implementation = parsed_creation_args.filter
parsed_args, unused_args = parser.parse_known_args(workflow_cmdline_args)
if parsed_args.fillmissing != 'none' and parsed_creation_args.fillmissing != parsed_args.fillmissing:
logger.error( "Ignoring --fillmissing cmdline arg. Can't specify a different fillmissing setting after the project has already been created." )
if parsed_args.filter != 'Original' and parsed_creation_args.filter != parsed_args.filter:
logger.error( "Ignoring --filter cmdline arg. Can't specify a different filter setting after the project has already been created." )
self.batch = not parsed_args.nobatch
self._applets = []
self.projectMetadataApplet = ProjectMetadataApplet()
self._applets.append(self.projectMetadataApplet)
self.setupInputs()
if self.fillMissing != 'none':
self.fillMissingSlicesApplet = FillMissingSlicesApplet(
self, "Fill Missing Slices", "Fill Missing Slices", self.fillMissing)
self._applets.append(self.fillMissingSlicesApplet)
if isinstance(self, ObjectClassificationWorkflowPixel):
self.input_types = 'raw'
elif isinstance(self, ObjectClassificationWorkflowBinary):
self.input_types = 'raw+binary'
elif isinstance( self, ObjectClassificationWorkflowPrediction ):
self.input_types = 'raw+pmaps'
# our main applets
self.objectExtractionApplet = ObjectExtractionApplet(workflow=self, name = "Object Feature Selection")
self.objectClassificationApplet = ObjectClassificationApplet(workflow=self)
self.dataExportApplet = ObjectClassificationDataExportApplet(self, "Object Information Export")
self.dataExportApplet.set_exporting_operator(self.objectClassificationApplet.topLevelOperator)
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.WorkingDirectory.connect( self.dataSelectionApplet.topLevelOperator.WorkingDirectory )
# See EXPORT_SELECTION_PREDICTIONS and EXPORT_SELECTION_PROBABILITIES, above
opDataExport.SelectionNames.setValue( ['Object Predictions', 'Object Probabilities'] )
if self.input_types == 'raw':
# Re-configure to add the pixel probabilities option
# See EXPORT_SELECTION_PIXEL_PROBABILITIES, above
opDataExport.SelectionNames.setValue( ['Object Predictions', 'Object Probabilities', 'Pixel Probabilities'] )
self._applets.append(self.objectExtractionApplet)
self._applets.append(self.objectClassificationApplet)
self._applets.append(self.dataExportApplet)
if self.batch:
self.dataSelectionAppletBatch = DataSelectionApplet(
self, "Batch Inputs", "Batch Inputs", batchDataGui=True)
self.opDataSelectionBatch = self.dataSelectionAppletBatch.topLevelOperator
if self.input_types == 'raw':
self.opDataSelectionBatch.DatasetRoles.setValue(['Raw Data'])
elif self.input_types == 'raw+binary':
self.opDataSelectionBatch.DatasetRoles.setValue(['Raw Data', 'Binary Data'])
elif self.input_types == 'raw+pmaps':
self.opDataSelectionBatch.DatasetRoles.setValue(['Raw Data', 'Prediction Maps'])
else:
assert False, "Unknown object classification subclass type."
self.blockwiseObjectClassificationApplet = BlockwiseObjectClassificationApplet(
self, "Blockwise Object Classification", "Blockwise Object Classification")
self._applets.append(self.blockwiseObjectClassificationApplet)
self.batchExportApplet = ObjectClassificationDataExportApplet(
self, "Batch Object Prediction Export", isBatch=True)
opBatchDataExport = self.batchExportApplet.topLevelOperator
opBatchDataExport.WorkingDirectory.connect( self.dataSelectionApplet.topLevelOperator.WorkingDirectory )
self._applets.append(self.dataSelectionAppletBatch)
self._applets.append(self.batchExportApplet)
self._initBatchWorkflow()
self._batch_export_args = None
self._batch_input_args = None
if unused_args:
# Additional export args (specific to the object classification workflow)
export_arg_parser = argparse.ArgumentParser()
export_arg_parser.add_argument( "--table_filename", help="The location to export the object feature/prediction CSV file.", required=False )
export_arg_parser.add_argument( "--export_object_prediction_img", action="store_true" )
export_arg_parser.add_argument( "--export_object_probability_img", action="store_true" )
# TODO: Support this, too, someday?
#export_arg_parser.add_argument( "--export_object_label_img", action="store_true" )
if self.input_types == 'raw':
export_arg_parser.add_argument( "--export_pixel_probability_img", action="store_true" )
self._export_args, unused_args = export_arg_parser.parse_known_args(unused_args)
# We parse the export setting args first. All remaining args are considered input files by the input applet.
self._batch_export_args, unused_args = self.batchExportApplet.parse_known_cmdline_args( unused_args )
self._batch_input_args, unused_args = self.dataSelectionAppletBatch.parse_known_cmdline_args( unused_args )
if unused_args:
warnings.warn("Unused command-line args: {}".format( unused_args ))
class WsdtWorkflow(Workflow):
workflowName = "Watershed Over Distance Transform"
workflowDescription = "A bare-bones workflow for using the WSDT applet"
defaultAppletIndex = 0 # show DataSelection by default
DATA_ROLE_RAW = 0
DATA_ROLE_PROBABILITIES = 1
ROLE_NAMES = ['Raw Data', 'Probabilities']
EXPORT_NAMES = ['Watershed']
@property
def applets(self):
return self._applets
@property
def imageNameListSlot(self):
return self.dataSelectionApplet.topLevelOperator.ImageName
def __init__(self, shell, headless, workflow_cmdline_args, project_creation_workflow, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super(WsdtWorkflow, self).__init__( shell, headless, workflow_cmdline_args, project_creation_workflow, graph=graph, *args, **kwargs)
self._applets = []
# -- DataSelection applet
#
self.dataSelectionApplet = DataSelectionApplet(self, "Input Data", "Input Data")
# Dataset inputs
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue( self.ROLE_NAMES )
# -- Wsdt applet
#
self.wsdtApplet = WsdtApplet(self, "Watershed", "Wsdt Watershed")
# -- DataExport applet
#
self.dataExportApplet = DataExportApplet(self, "Data Export")
# Configure global DataExport settings
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.WorkingDirectory.connect( opDataSelection.WorkingDirectory )
opDataExport.SelectionNames.setValue( self.EXPORT_NAMES )
# -- BatchProcessing applet
#
self.batchProcessingApplet = BatchProcessingApplet(self,
"Batch Processing",
self.dataSelectionApplet,
self.dataExportApplet)
# -- Expose applets to shell
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.wsdtApplet)
self._applets.append(self.dataExportApplet)
self._applets.append(self.batchProcessingApplet)
# -- Parse command-line arguments
# (Command-line args are applied in onProjectLoaded(), below.)
if workflow_cmdline_args:
self._data_export_args, unused_args = self.dataExportApplet.parse_known_cmdline_args( workflow_cmdline_args )
self._batch_input_args, unused_args = self.dataSelectionApplet.parse_known_cmdline_args( unused_args, role_names )
else:
unused_args = None
self._batch_input_args = None
self._data_export_args = None
if unused_args:
logger.warning("Unused command-line args: {}".format( unused_args ))
def connectLane(self, laneIndex):
"""
Override from base class.
"""
opDataSelection = self.dataSelectionApplet.topLevelOperator.getLane(laneIndex)
opWsdt = self.wsdtApplet.topLevelOperator.getLane(laneIndex)
opDataExport = self.dataExportApplet.topLevelOperator.getLane(laneIndex)
# watershed inputs
opWsdt.RawData.connect( opDataSelection.ImageGroup[self.DATA_ROLE_RAW] )
opWsdt.Input.connect( opDataSelection.ImageGroup[self.DATA_ROLE_PROBABILITIES] )
# DataExport inputs
opDataExport.RawData.connect( opDataSelection.ImageGroup[self.DATA_ROLE_RAW] )
opDataExport.RawDatasetInfo.connect( opDataSelection.DatasetGroup[self.DATA_ROLE_RAW] )
opDataExport.Inputs.resize( len(self.EXPORT_NAMES) )
opDataExport.Inputs[0].connect( opWsdt.Superpixels )
for slot in opDataExport.Inputs:
assert slot.partner is not None
def onProjectLoaded(self, projectManager):
"""
Overridden from Workflow base class. Called by the Project Manager.
If the user provided command-line arguments, use them to configure
the workflow inputs and output settings.
"""
# Configure the data export operator.
if self._data_export_args:
self.dataExportApplet.configure_operator_with_parsed_args( self._data_export_args )
if self._headless and self._batch_input_args and self._data_export_args:
logger.info("Beginning Batch Processing")
self.batchProcessingApplet.run_export_from_parsed_args(self._batch_input_args)
logger.info("Completed Batch Processing")
def handleAppletStateUpdateRequested(self):
"""
Overridden from Workflow base class
Called when an applet has fired the :py:attr:`Applet.appletStateUpdateRequested`
"""
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataExport = self.dataExportApplet.topLevelOperator
opWsdt = self.wsdtApplet.topLevelOperator
# If no data, nothing else is ready.
input_ready = len(opDataSelection.ImageGroup) > 0 and not self.dataSelectionApplet.busy
# The user isn't allowed to touch anything while batch processing is running.
batch_processing_busy = self.batchProcessingApplet.busy
self._shell.setAppletEnabled( self.dataSelectionApplet, not batch_processing_busy )
self._shell.setAppletEnabled( self.wsdtApplet, not batch_processing_busy and input_ready )
self._shell.setAppletEnabled( self.dataExportApplet, not batch_processing_busy and input_ready and opWsdt.Superpixels.ready())
self._shell.setAppletEnabled( self.batchProcessingApplet, not batch_processing_busy and input_ready )
# Lastly, check for certain "busy" conditions, during which we
# should prevent the shell from closing the project.
busy = False
busy |= self.dataSelectionApplet.busy
busy |= self.wsdtApplet.busy
busy |= self.dataExportApplet.busy
busy |= self.batchProcessingApplet.busy
self._shell.enableProjectChanges( not busy )
def _initBatchWorkflow(self):
"""
Connect the batch-mode top-level operators to the training workflow and to eachother.
"""
# Access applet operators from the training workflow
opTrainingDataSelection = self.dataSelectionApplet.topLevelOperator
opTrainingFeatures = self.featureSelectionApplet.topLevelOperator
opClassify = self.countingApplet.topLevelOperator
opSelectFirstLane = OperatorWrapper( OpSelectSubslot, parent=self )
opSelectFirstLane.Inputs.connect( opTrainingDataSelection.ImageGroup )
opSelectFirstLane.SubslotIndex.setValue(0)
opSelectFirstRole = OpSelectSubslot( parent=self )
opSelectFirstRole.Inputs.connect( opSelectFirstLane.Output )
opSelectFirstRole.SubslotIndex.setValue(0)
## Create additional batch workflow operators
opBatchFeatures = OperatorWrapper( OpFeatureSelection, operator_kwargs={'filter_implementation':'Original'}, parent=self, promotedSlotNames=['InputImage'] )
opBatchPredictionPipeline = OperatorWrapper( OpPredictionPipeline, parent=self )
# Create applets for batch workflow
self.batchInputApplet = DataSelectionApplet(self, "Batch Prediction Input Selections", "BatchDataSelection", supportIlastik05Import=False, batchDataGui=True)
self.batchResultsApplet = CountingDataExportApplet(self, "Batch Prediction Output Locations", opBatchPredictionPipeline, isBatch=True)
# Expose in shell
self._applets.append(self.batchInputApplet)
self._applets.append(self.batchResultsApplet)
opBatchInputs = self.batchInputApplet.topLevelOperator
opBatchResults = self.batchResultsApplet.topLevelOperator
opBatchInputs.DatasetRoles.connect( opTrainingDataSelection.DatasetRoles )
opBatchResults.ConstraintDataset.connect( opSelectFirstRole.Output )
## Connect Operators ##
opTranspose = OpTransposeSlots( parent=self )
opTranspose.OutputLength.setValue(1)
opTranspose.Inputs.connect( opBatchInputs.DatasetGroup )
# Provide dataset paths from data selection applet to the batch export applet
opBatchResults.RawDatasetInfo.connect( opTranspose.Outputs[0] )
opBatchResults.WorkingDirectory.connect( opBatchInputs.WorkingDirectory )
# Connect (clone) the feature operator inputs from
# the interactive workflow's features operator (which gets them from the GUI)
opBatchFeatures.Scales.connect( opTrainingFeatures.Scales )
opBatchFeatures.FeatureIds.connect( opTrainingFeatures.FeatureIds )
opBatchFeatures.SelectionMatrix.connect( opTrainingFeatures.SelectionMatrix )
# Classifier and LabelsCount are provided by the interactive workflow
opBatchPredictionPipeline.Classifier.connect( opClassify.Classifier )
opBatchPredictionPipeline.MaxLabel.connect( opClassify.MaxLabelValue )
opBatchPredictionPipeline.FreezePredictions.setValue( False )
# Provide these for the gui
opBatchResults.RawData.connect( opBatchInputs.Image )
opBatchResults.PmapColors.connect( opClassify.PmapColors )
opBatchResults.LabelNames.connect( opClassify.LabelNames )
opBatchResults.UpperBound.connect( opClassify.UpperBound )
# Connect Image pathway:
# Input Image -> Features Op -> Prediction Op -> Export
opBatchFeatures.InputImage.connect( opBatchInputs.Image )
opBatchPredictionPipeline.FeatureImages.connect( opBatchFeatures.OutputImage )
opBatchResults.SelectionNames.setValue( ['Probabilities'] )
# opBatchResults.Inputs is indexed by [lane][selection],
# Use OpTranspose to allow connection.
opTransposeBatchInputs = OpTransposeSlots( parent=self )
opTransposeBatchInputs.OutputLength.setValue(0)
opTransposeBatchInputs.Inputs.resize(1)
opTransposeBatchInputs.Inputs[0].connect( opBatchPredictionPipeline.HeadlessPredictionProbabilities ) # selection 0
# Now opTransposeBatchInputs.Outputs is level-2 indexed by [lane][selection]
opBatchResults.Inputs.connect( opTransposeBatchInputs.Outputs )
# We don't actually need the cached path in the batch pipeline.
# Just connect the uncached features here to satisfy the operator.
opBatchPredictionPipeline.CachedFeatureImages.connect( opBatchFeatures.OutputImage )
self.opBatchPredictionPipeline = opBatchPredictionPipeline
class CountingWorkflow(Workflow):
workflowName = "Cell Density Counting"
workflowDescription = "This is obviously self-explanatory."
defaultAppletIndex = 1 # show DataSelection by default
def __init__(self, shell, headless, workflow_cmdline_args, project_creation_args, appendBatchOperators=True, *args, **kwargs):
graph = kwargs['graph'] if 'graph' in kwargs else Graph()
if 'graph' in kwargs: del kwargs['graph']
super( CountingWorkflow, self ).__init__( shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs )
# Parse workflow-specific command-line args
parser = argparse.ArgumentParser()
parser.add_argument("--csv-export-file", help="Instead of exporting prediction density images, export total counts to the given csv path.")
self.parsed_counting_workflow_args, unused_args = parser.parse_known_args(workflow_cmdline_args)
######################
# Interactive workflow
######################
self.projectMetadataApplet = ProjectMetadataApplet()
self.dataSelectionApplet = DataSelectionApplet(self,
"Input Data",
"Input Data",
batchDataGui=False,
force5d=False
)
opDataSelection = self.dataSelectionApplet.topLevelOperator
role_names = ['Raw Data']
opDataSelection.DatasetRoles.setValue( role_names )
self.featureSelectionApplet = FeatureSelectionApplet(self,
"Feature Selection",
"FeatureSelections")
#self.pcApplet = PixelClassificationApplet(self, "PixelClassification")
self.countingApplet = CountingApplet(workflow=self)
opCounting = self.countingApplet.topLevelOperator
self.dataExportApplet = CountingDataExportApplet(self, "Density Export", opCounting)
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.PmapColors.connect(opCounting.PmapColors)
opDataExport.LabelNames.connect(opCounting.LabelNames)
opDataExport.UpperBound.connect(opCounting.UpperBound)
opDataExport.WorkingDirectory.connect(opDataSelection.WorkingDirectory)
opDataExport.SelectionNames.setValue( ['Probabilities'] )
self._applets = []
self._applets.append(self.projectMetadataApplet)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.featureSelectionApplet)
self._applets.append(self.countingApplet)
self._applets.append(self.dataExportApplet)
self._batch_input_args = None
self._batch_export_args = None
self.batchInputApplet = None
self.batchResultsApplet = None
if appendBatchOperators:
# Connect batch workflow (NOT lane-based)
self._initBatchWorkflow()
if unused_args:
# We parse the export setting args first.
# All remaining args are considered input files by the input applet.
self._batch_export_args, unused_args = self.batchResultsApplet.parse_known_cmdline_args( unused_args )
self._batch_input_args, unused_args = self.batchInputApplet.parse_known_cmdline_args( unused_args, role_names )
@property
def applets(self):
return self._applets
@property
def imageNameListSlot(self):
return self.dataSelectionApplet.topLevelOperator.ImageName
def connectLane(self, laneIndex):
## Access applet operators
opData = self.dataSelectionApplet.topLevelOperator.getLane(laneIndex)
opTrainingFeatures = self.featureSelectionApplet.topLevelOperator.getLane(laneIndex)
opCounting = self.countingApplet.topLevelOperator.getLane(laneIndex)
opDataExport = self.dataExportApplet.topLevelOperator.getLane(laneIndex)
#### connect input image
opTrainingFeatures.InputImage.connect(opData.Image)
opCounting.InputImages.connect(opData.Image)
opCounting.FeatureImages.connect(opTrainingFeatures.OutputImage)
opCounting.LabelsAllowedFlags.connect(opData.AllowLabels)
opCounting.CachedFeatureImages.connect( opTrainingFeatures.CachedOutputImage )
#opCounting.UserLabels.connect(opClassify.LabelImages)
#opCounting.ForegroundLabels.connect(opObjExtraction.LabelImage)
opDataExport.Inputs.resize(1)
opDataExport.Inputs[0].connect( opCounting.HeadlessPredictionProbabilities )
opDataExport.RawData.connect( opData.ImageGroup[0] )
opDataExport.RawDatasetInfo.connect( opData.DatasetGroup[0] )
opDataExport.ConstraintDataset.connect( opData.ImageGroup[0] )
def _initBatchWorkflow(self):
"""
Connect the batch-mode top-level operators to the training workflow and to eachother.
"""
# Access applet operators from the training workflow
opTrainingDataSelection = self.dataSelectionApplet.topLevelOperator
opTrainingFeatures = self.featureSelectionApplet.topLevelOperator
opClassify = self.countingApplet.topLevelOperator
opSelectFirstLane = OperatorWrapper( OpSelectSubslot, parent=self )
opSelectFirstLane.Inputs.connect( opTrainingDataSelection.ImageGroup )
opSelectFirstLane.SubslotIndex.setValue(0)
opSelectFirstRole = OpSelectSubslot( parent=self )
opSelectFirstRole.Inputs.connect( opSelectFirstLane.Output )
opSelectFirstRole.SubslotIndex.setValue(0)
## Create additional batch workflow operators
opBatchFeatures = OperatorWrapper( OpFeatureSelection, operator_kwargs={'filter_implementation':'Original'}, parent=self, promotedSlotNames=['InputImage'] )
opBatchPredictionPipeline = OperatorWrapper( OpPredictionPipeline, parent=self )
# Create applets for batch workflow
self.batchInputApplet = DataSelectionApplet(self, "Batch Prediction Input Selections", "BatchDataSelection", supportIlastik05Import=False, batchDataGui=True)
self.batchResultsApplet = CountingDataExportApplet(self, "Batch Prediction Output Locations", opBatchPredictionPipeline, isBatch=True)
# Expose in shell
self._applets.append(self.batchInputApplet)
self._applets.append(self.batchResultsApplet)
opBatchInputs = self.batchInputApplet.topLevelOperator
opBatchResults = self.batchResultsApplet.topLevelOperator
opBatchInputs.DatasetRoles.connect( opTrainingDataSelection.DatasetRoles )
opBatchResults.ConstraintDataset.connect( opSelectFirstRole.Output )
## Connect Operators ##
opTranspose = OpTransposeSlots( parent=self )
opTranspose.OutputLength.setValue(1)
opTranspose.Inputs.connect( opBatchInputs.DatasetGroup )
# Provide dataset paths from data selection applet to the batch export applet
opBatchResults.RawDatasetInfo.connect( opTranspose.Outputs[0] )
opBatchResults.WorkingDirectory.connect( opBatchInputs.WorkingDirectory )
# Connect (clone) the feature operator inputs from
# the interactive workflow's features operator (which gets them from the GUI)
opBatchFeatures.Scales.connect( opTrainingFeatures.Scales )
opBatchFeatures.FeatureIds.connect( opTrainingFeatures.FeatureIds )
opBatchFeatures.SelectionMatrix.connect( opTrainingFeatures.SelectionMatrix )
# Classifier and LabelsCount are provided by the interactive workflow
opBatchPredictionPipeline.Classifier.connect( opClassify.Classifier )
opBatchPredictionPipeline.MaxLabel.connect( opClassify.MaxLabelValue )
opBatchPredictionPipeline.FreezePredictions.setValue( False )
# Provide these for the gui
opBatchResults.RawData.connect( opBatchInputs.Image )
opBatchResults.PmapColors.connect( opClassify.PmapColors )
opBatchResults.LabelNames.connect( opClassify.LabelNames )
opBatchResults.UpperBound.connect( opClassify.UpperBound )
# Connect Image pathway:
# Input Image -> Features Op -> Prediction Op -> Export
opBatchFeatures.InputImage.connect( opBatchInputs.Image )
opBatchPredictionPipeline.FeatureImages.connect( opBatchFeatures.OutputImage )
opBatchResults.SelectionNames.setValue( ['Probabilities'] )
# opBatchResults.Inputs is indexed by [lane][selection],
# Use OpTranspose to allow connection.
opTransposeBatchInputs = OpTransposeSlots( parent=self )
opTransposeBatchInputs.OutputLength.setValue(0)
opTransposeBatchInputs.Inputs.resize(1)
opTransposeBatchInputs.Inputs[0].connect( opBatchPredictionPipeline.HeadlessPredictionProbabilities ) # selection 0
# Now opTransposeBatchInputs.Outputs is level-2 indexed by [lane][selection]
opBatchResults.Inputs.connect( opTransposeBatchInputs.Outputs )
# We don't actually need the cached path in the batch pipeline.
# Just connect the uncached features here to satisfy the operator.
opBatchPredictionPipeline.CachedFeatureImages.connect( opBatchFeatures.OutputImage )
self.opBatchPredictionPipeline = opBatchPredictionPipeline
def onProjectLoaded(self, projectManager):
"""
Overridden from Workflow base class. Called by the Project Manager.
If the user provided command-line arguments, use them to configure
the workflow for batch mode and export all results.
(This workflow's headless mode supports only batch mode for now.)
"""
# Configure the batch data selection operator.
if self._batch_input_args and (self._batch_input_args.input_files or self._batch_input_args.raw_data):
self.batchInputApplet.configure_operator_with_parsed_args( self._batch_input_args )
# Configure the data export operator.
if self._batch_export_args:
self.batchResultsApplet.configure_operator_with_parsed_args( self._batch_export_args )
if self._batch_input_args and self.countingApplet.topLevelOperator.classifier_cache._dirty:
logger.warn("Your project file has no classifier. "
"A new classifier will be trained for this run.")
if self._headless:
# In headless mode, let's see the messages from the training operator.
logging.getLogger("lazyflow.operators.classifierOperators").setLevel(logging.DEBUG)
if self._headless and self._batch_input_args and self._batch_export_args:
# Make sure we're using the up-to-date classifier.
self.countingApplet.topLevelOperator.FreezePredictions.setValue(False)
csv_path = self.parsed_counting_workflow_args.csv_export_file
if csv_path:
logger.info( "Exporting Object Counts to {}".format(csv_path) )
sys.stdout.write("Progress: ")
sys.stdout.flush()
def print_progress( progress ):
sys.stdout.write( "{:.1f} ".format( progress ) )
sys.stdout.flush()
self.batchResultsApplet.progressSignal.connect(print_progress)
req = self.batchResultsApplet.prepareExportObjectCountsToCsv( csv_path )
req.wait()
# Finished.
sys.stdout.write("\n")
sys.stdout.flush()
else:
# Now run the batch export and report progress....
opBatchDataExport = self.batchResultsApplet.topLevelOperator
for i, opExportDataLaneView in enumerate(opBatchDataExport):
logger.info( "Exporting object density image {} to {}".format(i, opExportDataLaneView.ExportPath.value) )
sys.stdout.write( "Result {}/{} Progress: ".format( i, len( opBatchDataExport ) ) )
sys.stdout.flush()
def print_progress( progress ):
sys.stdout.write( "{:.1f} ".format( progress ) )
sys.stdout.flush()
# If the operator provides a progress signal, use it.
slotProgressSignal = opExportDataLaneView.progressSignal
slotProgressSignal.subscribe( print_progress )
opExportDataLaneView.run_export()
# Finished.
sys.stdout.write("\n")
def handleAppletStateUpdateRequested(self):
"""
Overridden from Workflow base class
Called when an applet has fired the :py:attr:`Applet.statusUpdateSignal`
"""
# If no data, nothing else is ready.
opDataSelection = self.dataSelectionApplet.topLevelOperator
input_ready = len(opDataSelection.ImageGroup) > 0 and not self.dataSelectionApplet.busy
opFeatureSelection = self.featureSelectionApplet.topLevelOperator
featureOutput = opFeatureSelection.OutputImage
features_ready = input_ready and \
len(featureOutput) > 0 and \
featureOutput[0].ready() and \
(TinyVector(featureOutput[0].meta.shape) > 0).all()
opDataExport = self.dataExportApplet.topLevelOperator
predictions_ready = features_ready and \
len(opDataExport.Inputs) > 0 and \
opDataExport.Inputs[0][0].ready() and \
(TinyVector(opDataExport.Inputs[0][0].meta.shape) > 0).all()
self._shell.setAppletEnabled(self.featureSelectionApplet, input_ready)
self._shell.setAppletEnabled(self.countingApplet, features_ready)
self._shell.setAppletEnabled(self.dataExportApplet, predictions_ready and not self.dataExportApplet.busy)
# Training workflow must be fully configured before batch can be used
self._shell.setAppletEnabled(self.batchInputApplet, predictions_ready)
opBatchDataSelection = self.batchInputApplet.topLevelOperator
batch_input_ready = predictions_ready and \
len(opBatchDataSelection.ImageGroup) > 0
self._shell.setAppletEnabled(self.batchResultsApplet, batch_input_ready)
# Lastly, check for certain "busy" conditions, during which we
# should prevent the shell from closing the project.
busy = False
busy |= self.dataSelectionApplet.busy
busy |= self.featureSelectionApplet.busy
busy |= self.dataExportApplet.busy
self._shell.enableProjectChanges( not busy )
class CarvingWorkflow(Workflow):
workflowName = "Carving"
defaultAppletIndex = 1 # show DataSelection by default
@property
def applets(self):
return self._applets
@property
def imageNameListSlot(self):
return self.dataSelectionApplet.topLevelOperator.ImageName
def __init__(self, shell, headless, workflow_cmdline_args, project_creation_args, hintoverlayFile=None, pmapoverlayFile=None, *args, **kwargs):
if hintoverlayFile is not None:
assert isinstance(hintoverlayFile, str), "hintoverlayFile should be a string, not '%s'" % type(hintoverlayFile)
if pmapoverlayFile is not None:
assert isinstance(pmapoverlayFile, str), "pmapoverlayFile should be a string, not '%s'" % type(pmapoverlayFile)
graph = Graph()
super(CarvingWorkflow, self).__init__(shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs)
self.workflow_cmdline_args = workflow_cmdline_args
data_instructions = "Select your input data using the 'Raw Data' tab shown on the right"
## Create applets
self.projectMetadataApplet = ProjectMetadataApplet()
self.dataSelectionApplet = DataSelectionApplet( self,
"Input Data",
"Input Data",
supportIlastik05Import=True,
batchDataGui=False,
instructionText=data_instructions,
max_lanes=1 )
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue( DATA_ROLES )
self.preprocessingApplet = PreprocessingApplet(workflow=self,
title = "Preprocessing",
projectFileGroupName="preprocessing")
self.carvingApplet = CarvingApplet(workflow=self,
projectFileGroupName="carving",
hintOverlayFile=hintoverlayFile,
pmapOverlayFile=pmapoverlayFile)
#self.carvingApplet.topLevelOperator.MST.connect(self.preprocessingApplet.topLevelOperator.PreprocessedData)
# Expose to shell
self._applets = []
self._applets.append(self.projectMetadataApplet)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.preprocessingApplet)
self._applets.append(self.carvingApplet)
def connectLane(self, laneIndex):
## Access applet operators
opData = self.dataSelectionApplet.topLevelOperator.getLane(laneIndex)
opPreprocessing = self.preprocessingApplet.topLevelOperator.getLane(laneIndex)
opCarvingLane = self.carvingApplet.topLevelOperator.getLane(laneIndex)
opCarvingLane.connectToPreprocessingApplet(self.preprocessingApplet)
op5 = OpReorderAxes(parent=self)
op5.AxisOrder.setValue("txyzc")
op5.Input.connect(opData.Image)
## Connect operators
opPreprocessing.InputData.connect(op5.Output)
#opCarvingTopLevel.RawData.connect(op5.output)
opCarvingLane.InputData.connect(op5.Output)
opCarvingLane.FilteredInputData.connect(opPreprocessing.FilteredImage)
opCarvingLane.MST.connect(opPreprocessing.PreprocessedData)
opCarvingLane.UncertaintyType.setValue("none")
# Special input-input connection: WriteSeeds metadata must mirror the input data
opCarvingLane.WriteSeeds.connect( opCarvingLane.InputData )
self.preprocessingApplet.enableDownstream(False)
def handleAppletStateUpdateRequested(self):
# If no data, nothing else is ready.
opDataSelection = self.dataSelectionApplet.topLevelOperator
input_ready = len(opDataSelection.ImageGroup) > 0
# If preprocessing isn't configured yet, don't allow carving
preprocessed_data_ready = input_ready and self.preprocessingApplet._enabledDS
# Enable each applet as appropriate
self._shell.setAppletEnabled(self.preprocessingApplet, input_ready)
self._shell.setAppletEnabled(self.carvingApplet, preprocessed_data_ready)
def onProjectLoaded(self, projectManager):
"""
Overridden from Workflow base class. Called by the Project Manager.
If the user provided command-line arguments, apply them to the workflow operators.
Currently, we support command-line configuration of:
- DataSelection
- Preprocessing, in which case preprocessing is immediately executed
"""
# If input data files were provided on the command line, configure the DataSelection applet now.
# (Otherwise, we assume the project already had a dataset selected.)
input_data_args, unused_args = DataSelectionApplet.parse_known_cmdline_args(self.workflow_cmdline_args, DATA_ROLES)
if input_data_args.raw_data:
self.dataSelectionApplet.configure_operator_with_parsed_args(input_data_args)
#
# Parse the remaining cmd-line arguments
#
filter_indexes = { 'bright-lines' : OpFilter.HESSIAN_BRIGHT,
'dark-lines' : OpFilter.HESSIAN_DARK,
'step-edges' : OpFilter.STEP_EDGES,
'original' : OpFilter.RAW,
'inverted' : OpFilter.RAW_INVERTED }
parser = argparse.ArgumentParser()
parser.add_argument('--run-preprocessing', action='store_true')
parser.add_argument('--preprocessing-sigma', type=float, required=False)
parser.add_argument('--preprocessing-filter', required=False, type=str.lower,
choices=filter_indexes.keys())
parsed_args, unused_args = parser.parse_known_args(unused_args)
if unused_args:
logger.warn("Did not use the following command-line arguments: {}".format(unused_args))
# Execute pre-processing.
if parsed_args.run_preprocessing:
if len(self.preprocessingApplet.topLevelOperator) != 1:
raise RuntimeError("Can't run preprocessing on a project with no images.")
opPreprocessing = self.preprocessingApplet.topLevelOperator.getLane(0) # Carving has only one 'lane'
# If user provided parameters, override the defaults.
if parsed_args.preprocessing_sigma is not None:
opPreprocessing.Sigma.setValue(parsed_args.preprocessing_sigma)
if parsed_args.preprocessing_filter:
filter_index = filter_indexes[parsed_args.preprocessing_filter]
opPreprocessing.Filter.setValue(filter_index)
logger.info("Running Preprocessing...")
opPreprocessing.PreprocessedData[:].wait()
logger.info("FINISHED Preprocessing...")
logger.info("Saving project...")
self._shell.projectManager.saveProject()
logger.info("Done saving.")
def parse_known_cmdline_args(self, cmdline_args):
# We use the same parser as the DataSelectionApplet
role_names = self.dataSelectionApplet.topLevelOperator.DatasetRoles.value
parsed_args, unused_args = DataSelectionApplet.parse_known_cmdline_args(cmdline_args, role_names)
return parsed_args, unused_args
class DataConversionWorkflow(Workflow):
"""
Simple workflow for converting data between formats.
Has only two 'interactive' applets (Data Selection and Data Export), plus the BatchProcessing applet.
Supports headless mode. For example:
.. code-block::
python ilastik.py --headless
--new_project=NewTemporaryProject.ilp
--workflow=DataConversionWorkflow
--output_format="png sequence"
~/input1.h5
~/input2.h5
.. note:: Beware of issues related to absolute vs. relative paths.
Relative links are stored relative to the project file.
To avoid this issue entirely, either
(1) use only absolute filepaths
or (2) cd into your project file's directory before launching ilastik.
"""
def __init__(self, shell, headless, workflow_cmdline_args, project_creation_args, *args, **kwargs):
# Create a graph to be shared by all operators
graph = Graph()
super(DataConversionWorkflow, self).__init__(shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs)
self._applets = []
# Instantiate DataSelection applet
self.dataSelectionApplet = DataSelectionApplet(self,
"Input Data",
"Input Data",
supportIlastik05Import=True)
# Configure global DataSelection settings
role_names = ["Input Data"]
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue( role_names )
# Instantiate DataExport applet
self.dataExportApplet = DataExportApplet(self, "Data Export")
# Configure global DataExport settings
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.WorkingDirectory.connect( opDataSelection.WorkingDirectory )
opDataExport.SelectionNames.setValue( ["Input"] )
# No special data pre/post processing necessary in this workflow,
# but this is where we'd hook it up if we needed it.
#
#self.dataExportApplet.prepare_for_entire_export = self.prepare_for_entire_export
#self.dataExportApplet.prepare_lane_for_export = self.prepare_lane_for_export
#self.dataExportApplet.post_process_lane_export = self.post_process_lane_export
#self.dataExportApplet.post_process_entire_export = self.post_process_entire_export
# Instantiate BatchProcessing applet
self.batchProcessingApplet = BatchProcessingApplet(self,
"Batch Processing",
self.dataSelectionApplet,
self.dataExportApplet)
# Expose our applets in a list (for the shell to use)
self._applets.append( self.dataSelectionApplet )
self._applets.append( self.dataExportApplet )
self._applets.append(self.batchProcessingApplet)
# Parse command-line arguments
# Command-line args are applied in onProjectLoaded(), below.
if workflow_cmdline_args:
self._data_export_args, unused_args = self.dataExportApplet.parse_known_cmdline_args( workflow_cmdline_args )
self._batch_input_args, unused_args = self.dataSelectionApplet.parse_known_cmdline_args( unused_args, role_names )
else:
unused_args = None
self._batch_input_args = None
self._data_export_args = None
if unused_args:
logger.warn("Unused command-line args: {}".format( unused_args ))
@property
def applets(self):
"""
Overridden from Workflow base class.
"""
return self._applets
@property
def imageNameListSlot(self):
"""
Overridden from Workflow base class.
"""
return self.dataSelectionApplet.topLevelOperator.ImageName
def prepareForNewLane(self, laneIndex):
"""
Overridden from Workflow base class.
Called immediately before connectLane()
"""
# No preparation necessary.
pass
def connectLane(self, laneIndex):
"""
Overridden from Workflow base class.
"""
# Get a *view* of each top-level operator, specific to the current lane.
opDataSelectionView = self.dataSelectionApplet.topLevelOperator.getLane(laneIndex)
opDataExportView = self.dataExportApplet.topLevelOperator.getLane(laneIndex)
# Now connect the operators together for this lane.
# Most workflows would have more to do here, but this workflow is super simple:
# We just connect input to export
opDataExportView.RawDatasetInfo.connect( opDataSelectionView.DatasetGroup[RAW_DATA_ROLE_INDEX] )
opDataExportView.Inputs.resize( 1 )
opDataExportView.Inputs[RAW_DATA_ROLE_INDEX].connect( opDataSelectionView.ImageGroup[RAW_DATA_ROLE_INDEX] )
# There is no special "raw" display layer in this workflow.
#opDataExportView.RawData.connect( opDataSelectionView.ImageGroup[0] )
def handleNewLanesAdded(self):
"""
Overridden from Workflow base class.
Called immediately AFTER connectLane() and the dataset is loaded into the workflow.
"""
# No special handling required.
pass
def onProjectLoaded(self, projectManager):
"""
Overridden from Workflow base class. Called by the Project Manager.
If the user provided command-line arguments, use them to configure
the workflow inputs and output settings.
"""
# Configure the data export operator.
if self._data_export_args:
self.dataExportApplet.configure_operator_with_parsed_args( self._data_export_args )
if self._headless and self._batch_input_args and self._data_export_args:
logger.info("Beginning Batch Processing")
self.batchProcessingApplet.run_export_from_parsed_args(self._batch_input_args)
logger.info("Completed Batch Processing")
def handleAppletStateUpdateRequested(self):
"""
Overridden from Workflow base class
Called when an applet has fired the :py:attr:`Applet.statusUpdateSignal`
"""
opDataSelection = self.dataSelectionApplet.topLevelOperator
input_ready = len(opDataSelection.ImageGroup) > 0
opDataExport = self.dataExportApplet.topLevelOperator
export_data_ready = input_ready and \
len(opDataExport.Inputs[0]) > 0 and \
opDataExport.Inputs[0][0].ready() and \
(TinyVector(opDataExport.Inputs[0][0].meta.shape) > 0).all()
self._shell.setAppletEnabled(self.dataSelectionApplet, not self.batchProcessingApplet.busy)
self._shell.setAppletEnabled(self.dataExportApplet, export_data_ready and not self.batchProcessingApplet.busy)
self._shell.setAppletEnabled(self.batchProcessingApplet, export_data_ready)
# Lastly, check for certain "busy" conditions, during which we
# should prevent the shell from closing the project.
busy = False
busy |= self.dataSelectionApplet.busy
busy |= self.dataExportApplet.busy
busy |= self.batchProcessingApplet.busy
self._shell.enableProjectChanges( not busy )
