def __init__(self, shell, headless, workflow_cmdline_args, project_creation_args, *args, **kwargs):
graph = kwargs["graph"] if "graph" in kwargs else Graph()
if "graph" in kwargs:
del kwargs["graph"]
super(CountingWorkflow, self).__init__(
shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs
)
self.stored_classifier = None
# Parse workflow-specific command-line args
parser = argparse.ArgumentParser()
parser.add_argument(
"--csv-export-file",
help="Instead of exporting prediction density images, export total counts to the given csv path.",
)
self.parsed_counting_workflow_args, unused_args = parser.parse_known_args(workflow_cmdline_args)
######################
# Interactive workflow
######################
allowed_axis_orders = []
for space in itertools.permutations("xyz", 2):
allowed_axis_orders.append("".join(space) + "c")
self.dataSelectionApplet = DataSelectionApplet(
self, "Input Data", "Input Data", forceAxisOrder=allowed_axis_orders
)
opDataSelection = self.dataSelectionApplet.topLevelOperator
role_names = ["Raw Data"]
opDataSelection.DatasetRoles.setValue(role_names)
self.featureSelectionApplet = FeatureSelectionApplet(self, "Feature Selection", "FeatureSelections")
self.countingApplet = CountingApplet(workflow=self)
opCounting = self.countingApplet.topLevelOperator
opCounting.WorkingDirectory.connect(opDataSelection.WorkingDirectory)
self.dataExportApplet = CountingDataExportApplet(self, "Density Export", opCounting)
# Customization hooks
self.dataExportApplet.prepare_for_entire_export = self.prepare_for_entire_export
self.dataExportApplet.post_process_lane_export = self.post_process_lane_export
self.dataExportApplet.post_process_entire_export = self.post_process_entire_export
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.PmapColors.connect(opCounting.PmapColors)
opDataExport.LabelNames.connect(opCounting.LabelNames)
opDataExport.UpperBound.connect(opCounting.UpperBound)
opDataExport.WorkingDirectory.connect(opDataSelection.WorkingDirectory)
opDataExport.SelectionNames.setValue(["Probabilities"])
self._applets = []
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.featureSelectionApplet)
self._applets.append(self.countingApplet)
self._applets.append(self.dataExportApplet)
self.batchProcessingApplet = BatchProcessingApplet(
self, "Batch Processing", self.dataSelectionApplet, self.dataExportApplet
)
self._applets.append(self.batchProcessingApplet)
if unused_args:
# We parse the export setting args first. All remaining args are considered input files by the input applet.
self._batch_export_args, unused_args = self.dataExportApplet.parse_known_cmdline_args(unused_args)
self._batch_input_args, unused_args = self.batchProcessingApplet.parse_known_cmdline_args(unused_args)
else:
self._batch_input_args = None
self._batch_export_args = None
if unused_args:
logger.warning("Unused command-line args: {}".format(unused_args))
def __init__(self, shell, headless, workflow_cmdline_args, project_creation_args, appendBatchOperators=True, *args, **kwargs):
graph = kwargs['graph'] if 'graph' in kwargs else Graph()
if 'graph' in kwargs: del kwargs['graph']
super( CountingWorkflow, self ).__init__( shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs )
# Parse workflow-specific command-line args
parser = argparse.ArgumentParser()
parser.add_argument("--csv-export-file", help="Instead of exporting prediction density images, export total counts to the given csv path.")
self.parsed_counting_workflow_args, unused_args = parser.parse_known_args(workflow_cmdline_args)
######################
# Interactive workflow
######################
self.projectMetadataApplet = ProjectMetadataApplet()
self.dataSelectionApplet = DataSelectionApplet(self,
"Input Data",
"Input Data",
batchDataGui=False,
force5d=False
)
opDataSelection = self.dataSelectionApplet.topLevelOperator
role_names = ['Raw Data']
opDataSelection.DatasetRoles.setValue( role_names )
self.featureSelectionApplet = FeatureSelectionApplet(self,
"Feature Selection",
"FeatureSelections")
#self.pcApplet = PixelClassificationApplet(self, "PixelClassification")
self.countingApplet = CountingApplet(workflow=self)
opCounting = self.countingApplet.topLevelOperator
self.dataExportApplet = CountingDataExportApplet(self, "Density Export", opCounting)
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.PmapColors.connect(opCounting.PmapColors)
opDataExport.LabelNames.connect(opCounting.LabelNames)
opDataExport.UpperBound.connect(opCounting.UpperBound)
opDataExport.WorkingDirectory.connect(opDataSelection.WorkingDirectory)
opDataExport.SelectionNames.setValue( ['Probabilities'] )
self._applets = []
self._applets.append(self.projectMetadataApplet)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.featureSelectionApplet)
self._applets.append(self.countingApplet)
self._applets.append(self.dataExportApplet)
self._batch_input_args = None
self._batch_export_args = None
self.batchInputApplet = None
self.batchResultsApplet = None
if appendBatchOperators:
# Connect batch workflow (NOT lane-based)
self._initBatchWorkflow()
if unused_args:
# We parse the export setting args first.
# All remaining args are considered input files by the input applet.
self._batch_export_args, unused_args = self.batchResultsApplet.parse_known_cmdline_args( unused_args )
self._batch_input_args, unused_args = self.batchInputApplet.parse_known_cmdline_args( unused_args, role_names )
class CountingWorkflow(Workflow):
workflowName = "Cell Density Counting"
workflowDescription = "This is obviously self-explanatory."
defaultAppletIndex = 0 # show DataSelection by default
def __init__(self, shell, headless, workflow_cmdline_args, project_creation_args, *args, **kwargs):
graph = kwargs["graph"] if "graph" in kwargs else Graph()
if "graph" in kwargs:
del kwargs["graph"]
super(CountingWorkflow, self).__init__(
shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs
)
self.stored_classifier = None
# Parse workflow-specific command-line args
parser = argparse.ArgumentParser()
parser.add_argument(
"--csv-export-file",
help="Instead of exporting prediction density images, export total counts to the given csv path.",
)
self.parsed_counting_workflow_args, unused_args = parser.parse_known_args(workflow_cmdline_args)
######################
# Interactive workflow
######################
allowed_axis_orders = []
for space in itertools.permutations("xyz", 2):
allowed_axis_orders.append("".join(space) + "c")
self.dataSelectionApplet = DataSelectionApplet(
self, "Input Data", "Input Data", forceAxisOrder=allowed_axis_orders
)
opDataSelection = self.dataSelectionApplet.topLevelOperator
role_names = ["Raw Data"]
opDataSelection.DatasetRoles.setValue(role_names)
self.featureSelectionApplet = FeatureSelectionApplet(self, "Feature Selection", "FeatureSelections")
self.countingApplet = CountingApplet(workflow=self)
opCounting = self.countingApplet.topLevelOperator
opCounting.WorkingDirectory.connect(opDataSelection.WorkingDirectory)
self.dataExportApplet = CountingDataExportApplet(self, "Density Export", opCounting)
# Customization hooks
self.dataExportApplet.prepare_for_entire_export = self.prepare_for_entire_export
self.dataExportApplet.post_process_lane_export = self.post_process_lane_export
self.dataExportApplet.post_process_entire_export = self.post_process_entire_export
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.PmapColors.connect(opCounting.PmapColors)
opDataExport.LabelNames.connect(opCounting.LabelNames)
opDataExport.UpperBound.connect(opCounting.UpperBound)
opDataExport.WorkingDirectory.connect(opDataSelection.WorkingDirectory)
opDataExport.SelectionNames.setValue(["Probabilities"])
self._applets = []
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.featureSelectionApplet)
self._applets.append(self.countingApplet)
self._applets.append(self.dataExportApplet)
self.batchProcessingApplet = BatchProcessingApplet(
self, "Batch Processing", self.dataSelectionApplet, self.dataExportApplet
)
self._applets.append(self.batchProcessingApplet)
if unused_args:
# We parse the export setting args first. All remaining args are considered input files by the input applet.
self._batch_export_args, unused_args = self.dataExportApplet.parse_known_cmdline_args(unused_args)
self._batch_input_args, unused_args = self.batchProcessingApplet.parse_known_cmdline_args(unused_args)
else:
self._batch_input_args = None
self._batch_export_args = None
if unused_args:
logger.warning("Unused command-line args: {}".format(unused_args))
@property
def applets(self):
return self._applets
@property
def imageNameListSlot(self):
return self.dataSelectionApplet.topLevelOperator.ImageName
def prepareForNewLane(self, laneIndex):
"""
Overridden from Workflow base class.
Called immediately before a new lane is added to the workflow.
"""
# When the new lane is added, dirty notifications will propagate throughout the entire graph.
# This means the classifier will be marked 'dirty' even though it is still usable.
# Before that happens, let's store the classifier, so we can restore it at the end of connectLane(), below.
opCounting = self.countingApplet.topLevelOperator
if opCounting.classifier_cache.Output.ready() and not opCounting.classifier_cache._dirty:
self.stored_classifier = opCounting.classifier_cache.Output.value
else:
self.stored_classifier = None
def handleNewLanesAdded(self):
"""
Overridden from Workflow base class.
Called immediately after a new lane is added to the workflow and initialized.
"""
# Restore classifier we saved in prepareForNewLane() (if any)
if self.stored_classifier is not None:
self.countingApplet.topLevelOperator.classifier_cache.forceValue(self.stored_classifier)
# Release reference
self.stored_classifier = None
def connectLane(self, laneIndex):
## Access applet operators
opData = self.dataSelectionApplet.topLevelOperator.getLane(laneIndex)
opTrainingFeatures = self.featureSelectionApplet.topLevelOperator.getLane(laneIndex)
opCounting = self.countingApplet.topLevelOperator.getLane(laneIndex)
opDataExport = self.dataExportApplet.topLevelOperator.getLane(laneIndex)
#### connect input image
opTrainingFeatures.InputImage.connect(opData.Image)
opCounting.InputImages.connect(opData.Image)
opCounting.FeatureImages.connect(opTrainingFeatures.OutputImage)
opCounting.CachedFeatureImages.connect(opTrainingFeatures.CachedOutputImage)
# opCounting.UserLabels.connect(opClassify.LabelImages)
# opCounting.ForegroundLabels.connect(opObjExtraction.LabelImage)
opDataExport.Inputs.resize(1)
opDataExport.Inputs[0].connect(opCounting.HeadlessPredictionProbabilities)
opDataExport.RawData.connect(opData.ImageGroup[0])
opDataExport.RawDatasetInfo.connect(opData.DatasetGroup[0])
def onProjectLoaded(self, projectManager):
"""
Overridden from Workflow base class. Called by the Project Manager.
If the user provided command-line arguments, use them to configure
the workflow for batch mode and export all results.
(This workflow's headless mode supports only batch mode for now.)
"""
# Headless batch mode.
if self._headless and self._batch_input_args and self._batch_export_args:
self.dataExportApplet.configure_operator_with_parsed_args(self._batch_export_args)
# If the user provided a csv_path via the command line,
# overwrite the setting in the counting export operator.
csv_path = self.parsed_counting_workflow_args.csv_export_file
if csv_path:
self.dataExportApplet.topLevelOperator.CsvFilepath.setValue(csv_path)
if self.countingApplet.topLevelOperator.classifier_cache._dirty:
logger.warning(
"Your project file has no classifier. " "A new classifier will be trained for this run."
)
logger.info("Beginning Batch Processing")
self.batchProcessingApplet.run_export_from_parsed_args(self._batch_input_args)
logger.info("Completed Batch Processing")
def prepare_for_entire_export(self):
"""
Customization hook for data export (including batch mode).
"""
self.freeze_status = self.countingApplet.topLevelOperator.FreezePredictions.value
self.countingApplet.topLevelOperator.FreezePredictions.setValue(False)
# Create a new CSV file to write object counts into.
self.csv_export_file = None
if self.dataExportApplet.topLevelOperator.CsvFilepath.ready():
csv_path = self.dataExportApplet.topLevelOperator.CsvFilepath.value
logger.info("Exporting object counts to CSV: " + csv_path)
self.csv_export_file = open(csv_path, "w")
def post_process_lane_export(self, lane_index):
"""
Customization hook for data export (including batch mode).
"""
# Write the object counts for this lane as a line in the CSV file.
if self.csv_export_file:
self.dataExportApplet.write_csv_results(self.csv_export_file, lane_index)
def post_process_entire_export(self):
"""
Customization hook for data export (including batch mode).
"""
self.countingApplet.topLevelOperator.FreezePredictions.setValue(self.freeze_status)
if self.csv_export_file:
self.csv_export_file.close()
def handleAppletStateUpdateRequested(self):
"""
Overridden from Workflow base class
Called when an applet has fired the :py:attr:`Applet.statusUpdateSignal`
"""
# If no data, nothing else is ready.
opDataSelection = self.dataSelectionApplet.topLevelOperator
input_ready = len(opDataSelection.ImageGroup) > 0 and not self.dataSelectionApplet.busy
opFeatureSelection = self.featureSelectionApplet.topLevelOperator
featureOutput = opFeatureSelection.OutputImage
features_ready = (
input_ready
and len(featureOutput) > 0
and featureOutput[0].ready()
and (TinyVector(featureOutput[0].meta.shape) > 0).all()
)
opDataExport = self.dataExportApplet.topLevelOperator
predictions_ready = (
features_ready
and len(opDataExport.Inputs) > 0
and opDataExport.Inputs[0][0].ready()
and (TinyVector(opDataExport.Inputs[0][0].meta.shape) > 0).all()
)
self._shell.setAppletEnabled(self.featureSelectionApplet, input_ready)
self._shell.setAppletEnabled(self.countingApplet, features_ready)
self._shell.setAppletEnabled(self.dataExportApplet, predictions_ready and not self.dataExportApplet.busy)
self._shell.setAppletEnabled(
self.batchProcessingApplet, predictions_ready and not self.batchProcessingApplet.busy
)
# Lastly, check for certain "busy" conditions, during which we
# should prevent the shell from closing the project.
busy = False
busy |= self.dataSelectionApplet.busy
busy |= self.featureSelectionApplet.busy
busy |= self.dataExportApplet.busy
busy |= self.batchProcessingApplet.busy
self._shell.enableProjectChanges(not busy)
class CountingWorkflow(Workflow):
workflowName = "Cell Density Counting"
workflowDescription = "This is obviously self-explanatory."
defaultAppletIndex = 1 # show DataSelection by default
def __init__(self, shell, headless, workflow_cmdline_args, project_creation_args, appendBatchOperators=True, *args, **kwargs):
graph = kwargs['graph'] if 'graph' in kwargs else Graph()
if 'graph' in kwargs: del kwargs['graph']
super( CountingWorkflow, self ).__init__( shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs )
# Parse workflow-specific command-line args
parser = argparse.ArgumentParser()
parser.add_argument("--csv-export-file", help="Instead of exporting prediction density images, export total counts to the given csv path.")
self.parsed_counting_workflow_args, unused_args = parser.parse_known_args(workflow_cmdline_args)
######################
# Interactive workflow
######################
self.projectMetadataApplet = ProjectMetadataApplet()
self.dataSelectionApplet = DataSelectionApplet(self,
"Input Data",
"Input Data",
batchDataGui=False,
force5d=False
)
opDataSelection = self.dataSelectionApplet.topLevelOperator
role_names = ['Raw Data']
opDataSelection.DatasetRoles.setValue( role_names )
self.featureSelectionApplet = FeatureSelectionApplet(self,
"Feature Selection",
"FeatureSelections")
#self.pcApplet = PixelClassificationApplet(self, "PixelClassification")
self.countingApplet = CountingApplet(workflow=self)
opCounting = self.countingApplet.topLevelOperator
self.dataExportApplet = CountingDataExportApplet(self, "Density Export", opCounting)
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.PmapColors.connect(opCounting.PmapColors)
opDataExport.LabelNames.connect(opCounting.LabelNames)
opDataExport.UpperBound.connect(opCounting.UpperBound)
opDataExport.WorkingDirectory.connect(opDataSelection.WorkingDirectory)
opDataExport.SelectionNames.setValue( ['Probabilities'] )
self._applets = []
self._applets.append(self.projectMetadataApplet)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.featureSelectionApplet)
self._applets.append(self.countingApplet)
self._applets.append(self.dataExportApplet)
self._batch_input_args = None
self._batch_export_args = None
self.batchInputApplet = None
self.batchResultsApplet = None
if appendBatchOperators:
# Connect batch workflow (NOT lane-based)
self._initBatchWorkflow()
if unused_args:
# We parse the export setting args first.
# All remaining args are considered input files by the input applet.
self._batch_export_args, unused_args = self.batchResultsApplet.parse_known_cmdline_args( unused_args )
self._batch_input_args, unused_args = self.batchInputApplet.parse_known_cmdline_args( unused_args, role_names )
@property
def applets(self):
return self._applets
@property
def imageNameListSlot(self):
return self.dataSelectionApplet.topLevelOperator.ImageName
def connectLane(self, laneIndex):
## Access applet operators
opData = self.dataSelectionApplet.topLevelOperator.getLane(laneIndex)
opTrainingFeatures = self.featureSelectionApplet.topLevelOperator.getLane(laneIndex)
opCounting = self.countingApplet.topLevelOperator.getLane(laneIndex)
opDataExport = self.dataExportApplet.topLevelOperator.getLane(laneIndex)
#### connect input image
opTrainingFeatures.InputImage.connect(opData.Image)
opCounting.InputImages.connect(opData.Image)
opCounting.FeatureImages.connect(opTrainingFeatures.OutputImage)
opCounting.LabelsAllowedFlags.connect(opData.AllowLabels)
opCounting.CachedFeatureImages.connect( opTrainingFeatures.CachedOutputImage )
#opCounting.UserLabels.connect(opClassify.LabelImages)
#opCounting.ForegroundLabels.connect(opObjExtraction.LabelImage)
opDataExport.Inputs.resize(1)
opDataExport.Inputs[0].connect( opCounting.HeadlessPredictionProbabilities )
opDataExport.RawData.connect( opData.ImageGroup[0] )
opDataExport.RawDatasetInfo.connect( opData.DatasetGroup[0] )
opDataExport.ConstraintDataset.connect( opData.ImageGroup[0] )
def _initBatchWorkflow(self):
"""
Connect the batch-mode top-level operators to the training workflow and to eachother.
"""
# Access applet operators from the training workflow
opTrainingDataSelection = self.dataSelectionApplet.topLevelOperator
opTrainingFeatures = self.featureSelectionApplet.topLevelOperator
opClassify = self.countingApplet.topLevelOperator
opSelectFirstLane = OperatorWrapper( OpSelectSubslot, parent=self )
opSelectFirstLane.Inputs.connect( opTrainingDataSelection.ImageGroup )
opSelectFirstLane.SubslotIndex.setValue(0)
opSelectFirstRole = OpSelectSubslot( parent=self )
opSelectFirstRole.Inputs.connect( opSelectFirstLane.Output )
opSelectFirstRole.SubslotIndex.setValue(0)
## Create additional batch workflow operators
opBatchFeatures = OperatorWrapper( OpFeatureSelection, operator_kwargs={'filter_implementation':'Original'}, parent=self, promotedSlotNames=['InputImage'] )
opBatchPredictionPipeline = OperatorWrapper( OpPredictionPipeline, parent=self )
# Create applets for batch workflow
self.batchInputApplet = DataSelectionApplet(self, "Batch Prediction Input Selections", "BatchDataSelection", supportIlastik05Import=False, batchDataGui=True)
self.batchResultsApplet = CountingDataExportApplet(self, "Batch Prediction Output Locations", opBatchPredictionPipeline, isBatch=True)
# Expose in shell
self._applets.append(self.batchInputApplet)
self._applets.append(self.batchResultsApplet)
opBatchInputs = self.batchInputApplet.topLevelOperator
opBatchResults = self.batchResultsApplet.topLevelOperator
opBatchInputs.DatasetRoles.connect( opTrainingDataSelection.DatasetRoles )
opBatchResults.ConstraintDataset.connect( opSelectFirstRole.Output )
## Connect Operators ##
opTranspose = OpTransposeSlots( parent=self )
opTranspose.OutputLength.setValue(1)
opTranspose.Inputs.connect( opBatchInputs.DatasetGroup )
# Provide dataset paths from data selection applet to the batch export applet
opBatchResults.RawDatasetInfo.connect( opTranspose.Outputs[0] )
opBatchResults.WorkingDirectory.connect( opBatchInputs.WorkingDirectory )
# Connect (clone) the feature operator inputs from
# the interactive workflow's features operator (which gets them from the GUI)
opBatchFeatures.Scales.connect( opTrainingFeatures.Scales )
opBatchFeatures.FeatureIds.connect( opTrainingFeatures.FeatureIds )
opBatchFeatures.SelectionMatrix.connect( opTrainingFeatures.SelectionMatrix )
# Classifier and LabelsCount are provided by the interactive workflow
opBatchPredictionPipeline.Classifier.connect( opClassify.Classifier )
opBatchPredictionPipeline.MaxLabel.connect( opClassify.MaxLabelValue )
opBatchPredictionPipeline.FreezePredictions.setValue( False )
# Provide these for the gui
opBatchResults.RawData.connect( opBatchInputs.Image )
opBatchResults.PmapColors.connect( opClassify.PmapColors )
opBatchResults.LabelNames.connect( opClassify.LabelNames )
opBatchResults.UpperBound.connect( opClassify.UpperBound )
# Connect Image pathway:
# Input Image -> Features Op -> Prediction Op -> Export
opBatchFeatures.InputImage.connect( opBatchInputs.Image )
opBatchPredictionPipeline.FeatureImages.connect( opBatchFeatures.OutputImage )
opBatchResults.SelectionNames.setValue( ['Probabilities'] )
# opBatchResults.Inputs is indexed by [lane][selection],
# Use OpTranspose to allow connection.
opTransposeBatchInputs = OpTransposeSlots( parent=self )
opTransposeBatchInputs.OutputLength.setValue(0)
opTransposeBatchInputs.Inputs.resize(1)
opTransposeBatchInputs.Inputs[0].connect( opBatchPredictionPipeline.HeadlessPredictionProbabilities ) # selection 0
# Now opTransposeBatchInputs.Outputs is level-2 indexed by [lane][selection]
opBatchResults.Inputs.connect( opTransposeBatchInputs.Outputs )
# We don't actually need the cached path in the batch pipeline.
# Just connect the uncached features here to satisfy the operator.
opBatchPredictionPipeline.CachedFeatureImages.connect( opBatchFeatures.OutputImage )
self.opBatchPredictionPipeline = opBatchPredictionPipeline
def onProjectLoaded(self, projectManager):
"""
Overridden from Workflow base class. Called by the Project Manager.
If the user provided command-line arguments, use them to configure
the workflow for batch mode and export all results.
(This workflow's headless mode supports only batch mode for now.)
"""
# Configure the batch data selection operator.
if self._batch_input_args and (self._batch_input_args.input_files or self._batch_input_args.raw_data):
self.batchInputApplet.configure_operator_with_parsed_args( self._batch_input_args )
# Configure the data export operator.
if self._batch_export_args:
self.batchResultsApplet.configure_operator_with_parsed_args( self._batch_export_args )
if self._batch_input_args and self.countingApplet.topLevelOperator.classifier_cache._dirty:
logger.warn("Your project file has no classifier. "
"A new classifier will be trained for this run.")
if self._headless:
# In headless mode, let's see the messages from the training operator.
logging.getLogger("lazyflow.operators.classifierOperators").setLevel(logging.DEBUG)
if self._headless and self._batch_input_args and self._batch_export_args:
# Make sure we're using the up-to-date classifier.
self.countingApplet.topLevelOperator.FreezePredictions.setValue(False)
csv_path = self.parsed_counting_workflow_args.csv_export_file
if csv_path:
logger.info( "Exporting Object Counts to {}".format(csv_path) )
sys.stdout.write("Progress: ")
sys.stdout.flush()
def print_progress( progress ):
sys.stdout.write( "{:.1f} ".format( progress ) )
sys.stdout.flush()
self.batchResultsApplet.progressSignal.connect(print_progress)
req = self.batchResultsApplet.prepareExportObjectCountsToCsv( csv_path )
req.wait()
# Finished.
sys.stdout.write("\n")
sys.stdout.flush()
else:
# Now run the batch export and report progress....
opBatchDataExport = self.batchResultsApplet.topLevelOperator
for i, opExportDataLaneView in enumerate(opBatchDataExport):
logger.info( "Exporting object density image {} to {}".format(i, opExportDataLaneView.ExportPath.value) )
sys.stdout.write( "Result {}/{} Progress: ".format( i, len( opBatchDataExport ) ) )
sys.stdout.flush()
def print_progress( progress ):
sys.stdout.write( "{:.1f} ".format( progress ) )
sys.stdout.flush()
# If the operator provides a progress signal, use it.
slotProgressSignal = opExportDataLaneView.progressSignal
slotProgressSignal.subscribe( print_progress )
opExportDataLaneView.run_export()
# Finished.
sys.stdout.write("\n")
def handleAppletStateUpdateRequested(self):
"""
Overridden from Workflow base class
Called when an applet has fired the :py:attr:`Applet.statusUpdateSignal`
"""
# If no data, nothing else is ready.
opDataSelection = self.dataSelectionApplet.topLevelOperator
input_ready = len(opDataSelection.ImageGroup) > 0 and not self.dataSelectionApplet.busy
opFeatureSelection = self.featureSelectionApplet.topLevelOperator
featureOutput = opFeatureSelection.OutputImage
features_ready = input_ready and \
len(featureOutput) > 0 and \
featureOutput[0].ready() and \
(TinyVector(featureOutput[0].meta.shape) > 0).all()
opDataExport = self.dataExportApplet.topLevelOperator
predictions_ready = features_ready and \
len(opDataExport.Inputs) > 0 and \
opDataExport.Inputs[0][0].ready() and \
(TinyVector(opDataExport.Inputs[0][0].meta.shape) > 0).all()
self._shell.setAppletEnabled(self.featureSelectionApplet, input_ready)
self._shell.setAppletEnabled(self.countingApplet, features_ready)
self._shell.setAppletEnabled(self.dataExportApplet, predictions_ready and not self.dataExportApplet.busy)
# Training workflow must be fully configured before batch can be used
self._shell.setAppletEnabled(self.batchInputApplet, predictions_ready)
opBatchDataSelection = self.batchInputApplet.topLevelOperator
batch_input_ready = predictions_ready and \
len(opBatchDataSelection.ImageGroup) > 0
self._shell.setAppletEnabled(self.batchResultsApplet, batch_input_ready)
# Lastly, check for certain "busy" conditions, during which we
# should prevent the shell from closing the project.
busy = False
busy |= self.dataSelectionApplet.busy
busy |= self.featureSelectionApplet.busy
busy |= self.dataExportApplet.busy
self._shell.enableProjectChanges( not busy )
def _initBatchWorkflow(self):
"""
Connect the batch-mode top-level operators to the training workflow and to eachother.
"""
# Access applet operators from the training workflow
opTrainingDataSelection = self.dataSelectionApplet.topLevelOperator
opTrainingFeatures = self.featureSelectionApplet.topLevelOperator
opClassify = self.countingApplet.topLevelOperator
opSelectFirstLane = OperatorWrapper( OpSelectSubslot, parent=self )
opSelectFirstLane.Inputs.connect( opTrainingDataSelection.ImageGroup )
opSelectFirstLane.SubslotIndex.setValue(0)
opSelectFirstRole = OpSelectSubslot( parent=self )
opSelectFirstRole.Inputs.connect( opSelectFirstLane.Output )
opSelectFirstRole.SubslotIndex.setValue(0)
## Create additional batch workflow operators
opBatchFeatures = OperatorWrapper( OpFeatureSelection, operator_kwargs={'filter_implementation':'Original'}, parent=self, promotedSlotNames=['InputImage'] )
opBatchPredictionPipeline = OperatorWrapper( OpPredictionPipeline, parent=self )
# Create applets for batch workflow
self.batchInputApplet = DataSelectionApplet(self, "Batch Prediction Input Selections", "BatchDataSelection", supportIlastik05Import=False, batchDataGui=True)
self.batchResultsApplet = CountingDataExportApplet(self, "Batch Prediction Output Locations", opBatchPredictionPipeline, isBatch=True)
# Expose in shell
self._applets.append(self.batchInputApplet)
self._applets.append(self.batchResultsApplet)
opBatchInputs = self.batchInputApplet.topLevelOperator
opBatchResults = self.batchResultsApplet.topLevelOperator
opBatchInputs.DatasetRoles.connect( opTrainingDataSelection.DatasetRoles )
opBatchResults.ConstraintDataset.connect( opSelectFirstRole.Output )
## Connect Operators ##
opTranspose = OpTransposeSlots( parent=self )
opTranspose.OutputLength.setValue(1)
opTranspose.Inputs.connect( opBatchInputs.DatasetGroup )
# Provide dataset paths from data selection applet to the batch export applet
opBatchResults.RawDatasetInfo.connect( opTranspose.Outputs[0] )
opBatchResults.WorkingDirectory.connect( opBatchInputs.WorkingDirectory )
# Connect (clone) the feature operator inputs from
# the interactive workflow's features operator (which gets them from the GUI)
opBatchFeatures.Scales.connect( opTrainingFeatures.Scales )
opBatchFeatures.FeatureIds.connect( opTrainingFeatures.FeatureIds )
opBatchFeatures.SelectionMatrix.connect( opTrainingFeatures.SelectionMatrix )
# Classifier and LabelsCount are provided by the interactive workflow
opBatchPredictionPipeline.Classifier.connect( opClassify.Classifier )
opBatchPredictionPipeline.MaxLabel.connect( opClassify.MaxLabelValue )
opBatchPredictionPipeline.FreezePredictions.setValue( False )
# Provide these for the gui
opBatchResults.RawData.connect( opBatchInputs.Image )
opBatchResults.PmapColors.connect( opClassify.PmapColors )
opBatchResults.LabelNames.connect( opClassify.LabelNames )
opBatchResults.UpperBound.connect( opClassify.UpperBound )
# Connect Image pathway:
# Input Image -> Features Op -> Prediction Op -> Export
opBatchFeatures.InputImage.connect( opBatchInputs.Image )
opBatchPredictionPipeline.FeatureImages.connect( opBatchFeatures.OutputImage )
opBatchResults.SelectionNames.setValue( ['Probabilities'] )
# opBatchResults.Inputs is indexed by [lane][selection],
# Use OpTranspose to allow connection.
opTransposeBatchInputs = OpTransposeSlots( parent=self )
opTransposeBatchInputs.OutputLength.setValue(0)
opTransposeBatchInputs.Inputs.resize(1)
opTransposeBatchInputs.Inputs[0].connect( opBatchPredictionPipeline.HeadlessPredictionProbabilities ) # selection 0
# Now opTransposeBatchInputs.Outputs is level-2 indexed by [lane][selection]
opBatchResults.Inputs.connect( opTransposeBatchInputs.Outputs )
# We don't actually need the cached path in the batch pipeline.
# Just connect the uncached features here to satisfy the operator.
opBatchPredictionPipeline.CachedFeatureImages.connect( opBatchFeatures.OutputImage )
self.opBatchPredictionPipeline = opBatchPredictionPipeline
def __init__(self, shell, headless, workflow_cmdline_args, project_creation_args, appendBatchOperators=True, *args, **kwargs):
graph = kwargs['graph'] if 'graph' in kwargs else Graph()
if 'graph' in kwargs: del kwargs['graph']
super( CountingWorkflow, self ).__init__( shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs )
self.stored_classifer = None
# Parse workflow-specific command-line args
parser = argparse.ArgumentParser()
parser.add_argument("--csv-export-file", help="Instead of exporting prediction density images, export total counts to the given csv path.")
self.parsed_counting_workflow_args, unused_args = parser.parse_known_args(workflow_cmdline_args)
######################
# Interactive workflow
######################
self.projectMetadataApplet = ProjectMetadataApplet()
self.dataSelectionApplet = DataSelectionApplet(self,
"Input Data",
"Input Data" )
opDataSelection = self.dataSelectionApplet.topLevelOperator
role_names = ['Raw Data']
opDataSelection.DatasetRoles.setValue( role_names )
self.featureSelectionApplet = FeatureSelectionApplet(self,
"Feature Selection",
"FeatureSelections")
self.countingApplet = CountingApplet(workflow=self)
opCounting = self.countingApplet.topLevelOperator
self.dataExportApplet = CountingDataExportApplet(self, "Density Export", opCounting)
# Customization hooks
self.dataExportApplet.prepare_for_entire_export = self.prepare_for_entire_export
self.dataExportApplet.post_process_lane_export = self.post_process_lane_export
self.dataExportApplet.post_process_entire_export = self.post_process_entire_export
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.PmapColors.connect(opCounting.PmapColors)
opDataExport.LabelNames.connect(opCounting.LabelNames)
opDataExport.UpperBound.connect(opCounting.UpperBound)
opDataExport.WorkingDirectory.connect(opDataSelection.WorkingDirectory)
opDataExport.SelectionNames.setValue( ['Probabilities'] )
self._applets = []
self._applets.append(self.projectMetadataApplet)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.featureSelectionApplet)
self._applets.append(self.countingApplet)
self._applets.append(self.dataExportApplet)
self._batch_input_args = None
self._batch_export_args = None
if appendBatchOperators:
self.batchProcessingApplet = BatchProcessingApplet( self,
"Batch Processing",
self.dataSelectionApplet,
self.dataExportApplet )
self._applets.append(self.batchProcessingApplet)
if unused_args:
# We parse the export setting args first. All remaining args are considered input files by the input applet.
self._batch_export_args, unused_args = self.dataExportApplet.parse_known_cmdline_args( unused_args )
self._batch_input_args, unused_args = self.batchProcessingApplet.parse_known_cmdline_args( unused_args )
if unused_args:
logger.warn("Unused command-line args: {}".format( unused_args ))
class CountingWorkflow(Workflow):
workflowName = "Cell Density Counting"
workflowDescription = "This is obviously self-explanatory."
defaultAppletIndex = 1 # show DataSelection by default
def __init__(self, shell, headless, workflow_cmdline_args, project_creation_args, appendBatchOperators=True, *args, **kwargs):
graph = kwargs['graph'] if 'graph' in kwargs else Graph()
if 'graph' in kwargs: del kwargs['graph']
super( CountingWorkflow, self ).__init__( shell, headless, workflow_cmdline_args, project_creation_args, graph=graph, *args, **kwargs )
self.stored_classifer = None
# Parse workflow-specific command-line args
parser = argparse.ArgumentParser()
parser.add_argument("--csv-export-file", help="Instead of exporting prediction density images, export total counts to the given csv path.")
self.parsed_counting_workflow_args, unused_args = parser.parse_known_args(workflow_cmdline_args)
######################
# Interactive workflow
######################
self.projectMetadataApplet = ProjectMetadataApplet()
self.dataSelectionApplet = DataSelectionApplet(self,
"Input Data",
"Input Data" )
opDataSelection = self.dataSelectionApplet.topLevelOperator
role_names = ['Raw Data']
opDataSelection.DatasetRoles.setValue( role_names )
self.featureSelectionApplet = FeatureSelectionApplet(self,
"Feature Selection",
"FeatureSelections")
self.countingApplet = CountingApplet(workflow=self)
opCounting = self.countingApplet.topLevelOperator
self.dataExportApplet = CountingDataExportApplet(self, "Density Export", opCounting)
# Customization hooks
self.dataExportApplet.prepare_for_entire_export = self.prepare_for_entire_export
self.dataExportApplet.post_process_lane_export = self.post_process_lane_export
self.dataExportApplet.post_process_entire_export = self.post_process_entire_export
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.PmapColors.connect(opCounting.PmapColors)
opDataExport.LabelNames.connect(opCounting.LabelNames)
opDataExport.UpperBound.connect(opCounting.UpperBound)
opDataExport.WorkingDirectory.connect(opDataSelection.WorkingDirectory)
opDataExport.SelectionNames.setValue( ['Probabilities'] )
self._applets = []
self._applets.append(self.projectMetadataApplet)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.featureSelectionApplet)
self._applets.append(self.countingApplet)
self._applets.append(self.dataExportApplet)
self._batch_input_args = None
self._batch_export_args = None
if appendBatchOperators:
self.batchProcessingApplet = BatchProcessingApplet( self,
"Batch Processing",
self.dataSelectionApplet,
self.dataExportApplet )
self._applets.append(self.batchProcessingApplet)
if unused_args:
# We parse the export setting args first. All remaining args are considered input files by the input applet.
self._batch_export_args, unused_args = self.dataExportApplet.parse_known_cmdline_args( unused_args )
self._batch_input_args, unused_args = self.batchProcessingApplet.parse_known_cmdline_args( unused_args )
if unused_args:
logger.warn("Unused command-line args: {}".format( unused_args ))
@property
def applets(self):
return self._applets
@property
def imageNameListSlot(self):
return self.dataSelectionApplet.topLevelOperator.ImageName
def prepareForNewLane(self, laneIndex):
"""
Overridden from Workflow base class.
Called immediately before a new lane is added to the workflow.
"""
# When the new lane is added, dirty notifications will propagate throughout the entire graph.
# This means the classifier will be marked 'dirty' even though it is still usable.
# Before that happens, let's store the classifier, so we can restore it at the end of connectLane(), below.
opCounting = self.countingApplet.topLevelOperator
if opCounting.classifier_cache.Output.ready() and \
not opCounting.classifier_cache._dirty:
self.stored_classifer = opCounting.classifier_cache.Output.value
else:
self.stored_classifer = None
def handleNewLanesAdded(self):
"""
Overridden from Workflow base class.
Called immediately after a new lane is added to the workflow and initialized.
"""
# Restore classifier we saved in prepareForNewLane() (if any)
if self.stored_classifer is not None:
self.countingApplet.topLevelOperator.classifier_cache.forceValue(self.stored_classifer)
# Release reference
self.stored_classifer = None
def connectLane(self, laneIndex):
## Access applet operators
opData = self.dataSelectionApplet.topLevelOperator.getLane(laneIndex)
opTrainingFeatures = self.featureSelectionApplet.topLevelOperator.getLane(laneIndex)
opCounting = self.countingApplet.topLevelOperator.getLane(laneIndex)
opDataExport = self.dataExportApplet.topLevelOperator.getLane(laneIndex)
#### connect input image
opTrainingFeatures.InputImage.connect(opData.Image)
opCounting.InputImages.connect(opData.Image)
opCounting.FeatureImages.connect(opTrainingFeatures.OutputImage)
opCounting.LabelsAllowedFlags.connect(opData.AllowLabels)
opCounting.CachedFeatureImages.connect( opTrainingFeatures.CachedOutputImage )
#opCounting.UserLabels.connect(opClassify.LabelImages)
#opCounting.ForegroundLabels.connect(opObjExtraction.LabelImage)
opDataExport.Inputs.resize(1)
opDataExport.Inputs[0].connect( opCounting.HeadlessPredictionProbabilities )
opDataExport.RawData.connect( opData.ImageGroup[0] )
opDataExport.RawDatasetInfo.connect( opData.DatasetGroup[0] )
def onProjectLoaded(self, projectManager):
"""
Overridden from Workflow base class. Called by the Project Manager.
If the user provided command-line arguments, use them to configure
the workflow for batch mode and export all results.
(This workflow's headless mode supports only batch mode for now.)
"""
# Headless batch mode.
if self._headless and self._batch_input_args and self._batch_export_args:
self.dataExportApplet.configure_operator_with_parsed_args( self._batch_export_args )
# If the user provided a csv_path via the command line,
# overwrite the setting in the counting export operator.
csv_path = self.parsed_counting_workflow_args.csv_export_file
if csv_path:
self.dataExportApplet.topLevelOperator.CsvFilepath.setValue(csv_path)
if self.countingApplet.topLevelOperator.classifier_cache._dirty:
logger.warn("Your project file has no classifier. "
"A new classifier will be trained for this run.")
logger.info("Beginning Batch Processing")
self.batchProcessingApplet.run_export_from_parsed_args(self._batch_input_args)
logger.info("Completed Batch Processing")
def prepare_for_entire_export(self):
"""
Customization hook for data export (including batch mode).
"""
self.freeze_status = self.countingApplet.topLevelOperator.FreezePredictions.value
self.countingApplet.topLevelOperator.FreezePredictions.setValue(False)
# Create a new CSV file to write object counts into.
self.csv_export_file = None
if self.dataExportApplet.topLevelOperator.CsvFilepath.ready():
csv_path = self.dataExportApplet.topLevelOperator.CsvFilepath.value
logger.info("Exporting object counts to CSV: " + csv_path)
self.csv_export_file = open(csv_path, 'w')
def post_process_lane_export(self, lane_index):
"""
Customization hook for data export (including batch mode).
"""
# Write the object counts for this lane as a line in the CSV file.
if self.csv_export_file:
self.dataExportApplet.write_csv_results(self.csv_export_file, lane_index)
def post_process_entire_export(self):
"""
Customization hook for data export (including batch mode).
"""
self.countingApplet.topLevelOperator.FreezePredictions.setValue(self.freeze_status)
if self.csv_export_file:
self.csv_export_file.close()
def handleAppletStateUpdateRequested(self):
"""
Overridden from Workflow base class
Called when an applet has fired the :py:attr:`Applet.statusUpdateSignal`
"""
# If no data, nothing else is ready.
opDataSelection = self.dataSelectionApplet.topLevelOperator
input_ready = len(opDataSelection.ImageGroup) > 0 and not self.dataSelectionApplet.busy
opFeatureSelection = self.featureSelectionApplet.topLevelOperator
featureOutput = opFeatureSelection.OutputImage
features_ready = input_ready and \
len(featureOutput) > 0 and \
featureOutput[0].ready() and \
(TinyVector(featureOutput[0].meta.shape) > 0).all()
opDataExport = self.dataExportApplet.topLevelOperator
predictions_ready = features_ready and \
len(opDataExport.Inputs) > 0 and \
opDataExport.Inputs[0][0].ready() and \
(TinyVector(opDataExport.Inputs[0][0].meta.shape) > 0).all()
self._shell.setAppletEnabled(self.featureSelectionApplet, input_ready)
self._shell.setAppletEnabled(self.countingApplet, features_ready)
self._shell.setAppletEnabled(self.dataExportApplet, predictions_ready and not self.dataExportApplet.busy)
self._shell.setAppletEnabled(self.batchProcessingApplet, predictions_ready and not self.batchProcessingApplet.busy)
# Lastly, check for certain "busy" conditions, during which we
# should prevent the shell from closing the project.
busy = False
busy |= self.dataSelectionApplet.busy
busy |= self.featureSelectionApplet.busy
busy |= self.dataExportApplet.busy
busy |= self.batchProcessingApplet.busy
self._shell.enableProjectChanges( not busy )
def __init__(self,
shell,
headless,
workflow_cmdline_args,
project_creation_args,
appendBatchOperators=True,
*args,
**kwargs):
graph = kwargs['graph'] if 'graph' in kwargs else Graph()
if 'graph' in kwargs: del kwargs['graph']
super(CountingWorkflow, self).__init__(shell,
headless,
workflow_cmdline_args,
project_creation_args,
graph=graph,
*args,
**kwargs)
######################
# Interactive workflow
######################
self.projectMetadataApplet = ProjectMetadataApplet()
self.dataSelectionApplet = DataSelectionApplet(self,
"Input Data",
"Input Data",
batchDataGui=False,
force5d=False)
opDataSelection = self.dataSelectionApplet.topLevelOperator
opDataSelection.DatasetRoles.setValue(['Raw Data'])
self.featureSelectionApplet = FeatureSelectionApplet(
self, "Feature Selection", "FeatureSelections")
#self.pcApplet = PixelClassificationApplet(self, "PixelClassification")
self.countingApplet = CountingApplet(workflow=self)
opCounting = self.countingApplet.topLevelOperator
self.dataExportApplet = CountingDataExportApplet(
self, "Density Export")
opDataExport = self.dataExportApplet.topLevelOperator
opDataExport.PmapColors.connect(opCounting.PmapColors)
opDataExport.LabelNames.connect(opCounting.LabelNames)
opDataExport.UpperBound.connect(opCounting.UpperBound)
opDataExport.WorkingDirectory.connect(opDataSelection.WorkingDirectory)
opDataExport.SelectionNames.setValue(['Probabilities'])
self._applets = []
self._applets.append(self.projectMetadataApplet)
self._applets.append(self.dataSelectionApplet)
self._applets.append(self.featureSelectionApplet)
self._applets.append(self.countingApplet)
self._applets.append(self.dataExportApplet)
if appendBatchOperators:
# Create applets for batch workflow
self.batchInputApplet = DataSelectionApplet(
self,
"Batch Prediction Input Selections",
"BatchDataSelection",
supportIlastik05Import=False,
batchDataGui=True)
self.batchResultsApplet = CountingDataExportApplet(
self, "Batch Prediction Output Locations", isBatch=True)
# Expose in shell
self._applets.append(self.batchInputApplet)
self._applets.append(self.batchResultsApplet)
# Connect batch workflow (NOT lane-based)
self._initBatchWorkflow()