def predict_probabilities(self, X):
logger.debug( "Predicting with parallel vigra RF" )
X = numpy.asarray(X, dtype=numpy.float32)
# As each forest completes, aggregate results in a shared array.
# (Must put in a list so we can update it in this closure.)
total_predictions = [None]
prediction_lock = RequestLock()
def update_predictions(forest, forest_predictions):
forest_predictions *= forest.treeCount()
with prediction_lock:
if total_predictions[0] is None:
total_predictions[0] = forest_predictions
else:
total_predictions[0] += forest_predictions
# Create a request for each forest
pool = RequestPool()
for forest in self._forests:
req = Request( partial( forest.predictProbabilities, X ) )
req.notify_finished( partial(update_predictions, forest) )
pool.add( req )
del req
pool.wait()
total_predictions[0] /= self._num_trees
return total_predictions[0]
def create_and_train(self, X, y):
logger.debug( "Training parallel vigra RF" )
# Save for future reference
known_labels = numpy.unique(y)
X = numpy.asarray(X, numpy.float32)
y = numpy.asarray(y, numpy.uint32)
if y.ndim == 1:
y = y[:, numpy.newaxis]
assert X.ndim == 2
assert len(X) == len(y)
# Create N forests
forests = []
for _ in range(self._num_forests):
forest = vigra.learning.RandomForest(self._trees_per_forest, **self._kwargs)
forests.append( forest )
# Train them all in parallel
oobs = [None] * len(forests)
pool = RequestPool()
for i, forest in enumerate(forests):
req = Request( partial(forest.learnRF, X, y) )
# save the oobs
req.notify_finished( partial( oobs.__setitem__, i ) )
pool.add( req )
pool.wait()
return ParallelVigraRfLazyflowClassifier( forests, oobs, known_labels )
def predict_probabilities(self, X):
logger.debug( "Predicting with parallel vigra RF" )
X = numpy.asarray(X, dtype=numpy.float32)
assert X.ndim == 2
if self._feature_names is not None:
# For some reason, vigra doesn't seem to check this for us...
assert X.shape[1] == len(self._feature_names), \
"Feature count ({}) doesn't match the training feature count ({}).\n"\
"Expected features: {}".format( X.shape[1], len(self._feature_names), self._feature_names )
# As each forest completes, aggregate results in a shared array.
# (Must put in a list so we can update it in this closure.)
total_predictions = [None]
prediction_lock = RequestLock()
def update_predictions(forest, forest_predictions):
forest_predictions *= forest.treeCount()
with prediction_lock:
if total_predictions[0] is None:
total_predictions[0] = forest_predictions
else:
total_predictions[0] += forest_predictions
# Create a request for each forest
pool = RequestPool()
for forest in self._forests:
req = Request( partial( forest.predictProbabilities, X ) )
req.notify_finished( partial(update_predictions, forest) )
pool.add( req )
del req
pool.wait()
total_predictions[0] /= self._num_trees
return total_predictions[0]
def _triggerTableUpdate(self):
# Check that object area is included in selected features
featureNames = self.topLevelOperatorView.SelectedFeatures.value
if 'Standard Object Features' not in featureNames or 'Count' not in featureNames[
'Standard Object Features']:
box = QMessageBox(
QMessageBox.Warning, 'Warning',
'Object area is not a selected feature. Please select this feature on: \"Standard Object Features > Shape > Size in pixels\"',
QMessageBox.NoButton, self)
box.show()
return
# Clear table
self.table.clearContents()
self.table.setRowCount(0)
self.table.setSortingEnabled(False)
self.progressBar.show()
self.computeButton.setEnabled(False)
# Compute object features and number of labels per frame
def compute_features():
features = self.topLevelOperatorView.ObjectFeatures([]).wait()
labels = self.topLevelOperatorView.LabelInputs([]).wait()
return features, labels
req = Request(compute_features)
req.notify_finished(self._populateTable)
req.submit()
def run_export(self):
role_names = self.parentApplet.dataSelectionApplet.topLevelOperator.DatasetRoles.value
# Prepare file lists in an OrderedDict
role_path_dict = OrderedDict()
role_path_dict[0] = BatchProcessingGui.get_all_item_strings(
self.list_widgets[0])
num_datasets = len(role_path_dict[0])
for role_index, list_widget in enumerate(self.list_widgets[1:],
start=1):
role_path_dict[
role_index] = BatchProcessingGui.get_all_item_strings(
self.list_widgets[role_index])
assert len(role_path_dict[role_index]) <= num_datasets, \
"Too many files given for role: '{}'".format( role_names[role_index] )
if len(role_path_dict[role_index]) < num_datasets:
role_path_dict[role_index] += [None] * (
num_datasets - len(role_path_dict[role_index]))
# Run the export in a separate thread
export_req = Request(
partial(self.parentApplet.run_export, role_path_dict))
export_req.notify_failed(self.handle_batch_processing_failure)
export_req.notify_finished(self.handle_batch_processing_finished)
export_req.notify_cancelled(self.handle_batch_processing_cancelled)
self.export_req = export_req
self.parentApplet.busy = True
self.parentApplet.appletStateUpdateRequested.emit()
self.cancel_button.setVisible(True)
self.run_button.setEnabled(False)
# Start the export
export_req.submit()
def run_export(self):
role_names = self.parentApplet.dataSelectionApplet.role_names
# Prepare file lists in an OrderedDict
role_inputs = {
role_name: self._data_role_widgets[role_name].filepaths
for role_name in role_names
}
if all(len(role_inp) == 0 for role_inp in role_inputs.values()):
return
# Run the export in a separate thread
lane_configs = self.parentApplet.dataSelectionApplet.create_lane_configs(
role_inputs=role_inputs)
export_req = Request(
partial(self.parentApplet.run_export, lane_configs=lane_configs))
export_req.notify_failed(self.handle_batch_processing_failure)
export_req.notify_finished(self.handle_batch_processing_finished)
export_req.notify_cancelled(self.handle_batch_processing_cancelled)
self.export_req = export_req
self.parentApplet.busy = True
self.parentApplet.appletStateUpdateRequested()
self.cancel_button.setVisible(True)
self.run_button.setEnabled(False)
# Start the export
export_req.submit()
def export_object_data(self, settings, selected_features, gui=None):
"""
Initialize progress displays and start the actual export in a new thread using the lazyflow.request framework
:param settings: the settings from the GUI export dialog
:type settings: dict
:param selected_features: the features to export from the GUI dialog
:type selected_features: list
:param gui: the Progress bar and callbacks for finish/fail/cancel see ExportingGui.show_export_dialog
:type gui: dict
"""
self.save_export_progress_dialog(None)
if gui is None or "dialog" not in gui:
progress_display = ProgressPrinter("Export Progress", xrange(100, -1, -5), 2)
else:
progress_display = gui["dialog"]
self.save_export_progress_dialog(progress_display)
export = partial(self.do_export, settings, selected_features, progress_display) # ();return
request = Request(export)
request.notify_failed(gui["fail"] if gui is not None and "fail" in gui else self.export_failed)
request.notify_failed(self.export_failed)
request.notify_finished(gui["ok"] if gui is not None and "ok" in gui else self.export_finished)
request.notify_cancelled(gui["cancel"] if gui is not None and "cancel" in gui else self.export_cancelled)
request.submit()
if gui is not None and "dialog" in gui:
progress_display.cancel.connect(request.cancel)
def run_export(self):
role_names = self.parentApplet.dataSelectionApplet.topLevelOperator.DatasetRoles.value
# Prepare file lists in an OrderedDict
role_path_dict = OrderedDict()
role_path_dict[0] = BatchProcessingGui.get_all_item_strings(self.list_widgets[0])
num_datasets = len(role_path_dict[0])
for role_index, list_widget in enumerate(self.list_widgets[1:], start=1):
role_path_dict[role_index] = BatchProcessingGui.get_all_item_strings(self.list_widgets[role_index])
assert len(role_path_dict[role_index]) <= num_datasets, \
"Too many files given for role: '{}'".format( role_names[role_index] )
if len(role_path_dict[role_index]) < num_datasets:
role_path_dict[role_index] += [None] * (num_datasets-len(role_path_dict[role_index]))
# Run the export in a separate thread
export_req = Request(partial(self.parentApplet.run_export, role_path_dict))
export_req.notify_failed(self.handle_batch_processing_failure)
export_req.notify_finished(self.handle_batch_processing_finished)
export_req.notify_cancelled(self.handle_batch_processing_cancelled)
self.export_req = export_req
self.parentApplet.busy = True
self.parentApplet.appletStateUpdateRequested()
self.cancel_button.setVisible(True)
self.run_button.setEnabled(False)
# Start the export
export_req.submit()
def predict_probabilities(self, X):
logger.debug("Predicting with parallel vigra RF")
X = numpy.asarray(X, dtype=numpy.float32)
assert X.ndim == 2
if self._feature_names is not None:
# For some reason, vigra doesn't seem to check this for us...
assert X.shape[1] == len(self._feature_names), \
"Feature count doesn't match the training data."
# As each forest completes, aggregate results in a shared array.
# (Must put in a list so we can update it in this closure.)
total_predictions = [None]
prediction_lock = RequestLock()
def update_predictions(forest, forest_predictions):
forest_predictions *= forest.treeCount()
with prediction_lock:
if total_predictions[0] is None:
total_predictions[0] = forest_predictions
else:
total_predictions[0] += forest_predictions
# Create a request for each forest
pool = RequestPool()
for forest in self._forests:
req = Request(partial(forest.predictProbabilities, X))
req.notify_finished(partial(update_predictions, forest))
pool.add(req)
del req
pool.wait()
total_predictions[0] /= self._num_trees
return total_predictions[0]
def _triggerTableUpdate(self):
# Check that object area is included in selected features
featureNames = self.topLevelOperatorView.SelectedFeatures.value
if 'Standard Object Features' not in featureNames or 'Count' not in featureNames[
'Standard Object Features']:
box = QMessageBox(
QMessageBox.Warning, 'Warning',
'Object area is not a selected feature. Please select this feature on: \"Standard Object Features > Shape > Size in pixels\"',
QMessageBox.NoButton, self)
box.show()
return
# Clear table
self.table.clearContents()
self.table.setRowCount(0)
self.table.setSortingEnabled(False)
self.progressBar.show()
self.computeButton.setEnabled(False)
def compute_features_for_frame(tIndex, t, features):
# Compute features and labels (called in parallel from request pool)
roi = [
slice(None) for i in range(
len(self.topLevelOperatorView.LabelImages.meta.shape))
]
roi[tIndex] = slice(t, t + 1)
roi = tuple(roi)
frame = self.topLevelOperatorView.SegmentationImages(roi).wait()
frame = frame.squeeze().astype(numpy.uint32, copy=False)
# Dirty trick: We don't care what we're passing here for the 'image' parameter,
# but vigra insists that we pass *something*, so we'll cast the label image as float32.
features[t] = vigra.analysis.extractRegionFeatures(
frame.view(numpy.float32), frame, ['Count'], ignoreLabel=0)
tIndex = self.topLevelOperatorView.SegmentationImages.meta.axistags.index(
't')
tMax = self.topLevelOperatorView.SegmentationImages.meta.shape[tIndex]
features = {}
labels = {}
def compute_all_features():
# Compute features in parallel
pool = RequestPool()
for t in range(tMax):
pool.add(
Request(
partial(compute_features_for_frame, tIndex, t,
features)))
pool.wait()
# Compute labels
labels = self.topLevelOperatorView.LabelInputs([]).wait()
req = Request(compute_all_features)
req.notify_finished(partial(self._populateTable, features, labels))
req.submit()
def predict_probabilities(self, X):
logger.debug( "Predicting with parallel vigra RF" )
X = numpy.asarray(X, dtype=numpy.float32)
# As each forest completes, aggregate results in a shared array.
# (Must put in a list so we can update it in this closure.)
total_predictions = [None]
prediction_lock = RequestLock()
def update_predictions(forest, forest_predictions):
forest_predictions *= forest.treeCount()
with prediction_lock:
if total_predictions[0] is None:
total_predictions[0] = forest_predictions
else:
total_predictions[0] += forest_predictions
# Create a request for each forest
pool = RequestPool()
for forest in self._forests:
req = Request( partial( forest.predictProbabilities, X ) )
req.notify_finished( partial(update_predictions, forest) )
pool.add( req )
del req
pool.wait()
total_predictions[0] /= self._num_trees
return total_predictions[0]
def addStack(self, roleIndex, laneIndex):
"""
The user clicked the "Import Stack Files" button.
"""
stackDlg = StackFileSelectionWidget(self)
stackDlg.exec_()
if stackDlg.result() != QDialog.Accepted :
return
files = stackDlg.selectedFiles
sequence_axis = stackDlg.sequence_axis
if len(files) == 0:
return
info = DatasetInfo()
info.filePath = os.path.pathsep.join( files )
prefix = os.path.commonprefix(files)
info.nickname = PathComponents(prefix).filenameBase
# Add an underscore for each wildcard digit
num_wildcards = len(files[-1]) - len(prefix) - len( os.path.splitext(files[-1])[1] )
info.nickname += "_"*num_wildcards
# Allow labels by default if this gui isn't being used for batch data.
info.allowLabels = ( self.guiMode == GuiMode.Normal )
info.fromstack = True
originalNumLanes = len(self.topLevelOperator.DatasetGroup)
if laneIndex is None or laneIndex == -1:
laneIndex = len(self.topLevelOperator.DatasetGroup)
if len(self.topLevelOperator.DatasetGroup) < laneIndex+1:
self.topLevelOperator.DatasetGroup.resize(laneIndex+1)
def importStack():
self.parentApplet.busy = True
self.parentApplet.appletStateUpdateRequested.emit()
# Serializer will update the operator for us, which will propagate to the GUI.
try:
self.serializer.importStackAsLocalDataset( info, sequence_axis )
try:
self.topLevelOperator.DatasetGroup[laneIndex][roleIndex].setValue(info)
except DatasetConstraintError as ex:
# Give the user a chance to repair the problem.
filename = files[0] + "\n...\n" + files[-1]
return_val = [False]
self.handleDatasetConstraintError( info, filename, ex, roleIndex, laneIndex, return_val )
if not return_val[0]:
# Not successfully repaired. Roll back the changes and give up.
self.topLevelOperator.DatasetGroup.resize(originalNumLanes)
finally:
self.parentApplet.busy = False
self.parentApplet.appletStateUpdateRequested.emit()
req = Request( importStack )
req.notify_finished( lambda result: self.showDataset(laneIndex, roleIndex) )
req.notify_failed( partial(self.handleFailedStackLoad, files, originalNumLanes ) )
req.submit()
def addStack(self, roleIndex, laneIndex):
"""
The user clicked the "Import Stack Files" button.
"""
stackDlg = StackFileSelectionWidget(self)
stackDlg.exec_()
if stackDlg.result() != QDialog.Accepted :
return
files = stackDlg.selectedFiles
sequence_axis = stackDlg.sequence_axis
if len(files) == 0:
return
info = DatasetInfo()
info.filePath = os.path.pathsep.join( files )
prefix = os.path.commonprefix(files)
info.nickname = PathComponents(prefix).filenameBase
# Add an underscore for each wildcard digit
num_wildcards = len(files[-1]) - len(prefix) - len( os.path.splitext(files[-1])[1] )
info.nickname += "_"*num_wildcards
# Allow labels by default if this gui isn't being used for batch data.
info.allowLabels = ( self.guiMode == GuiMode.Normal )
info.fromstack = True
originalNumLanes = len(self.topLevelOperator.DatasetGroup)
if laneIndex is None or laneIndex == -1:
laneIndex = len(self.topLevelOperator.DatasetGroup)
if len(self.topLevelOperator.DatasetGroup) < laneIndex+1:
self.topLevelOperator.DatasetGroup.resize(laneIndex+1)
def importStack():
self.parentApplet.busy = True
self.parentApplet.appletStateUpdateRequested.emit()
# Serializer will update the operator for us, which will propagate to the GUI.
try:
self.serializer.importStackAsLocalDataset( info, sequence_axis )
try:
self.topLevelOperator.DatasetGroup[laneIndex][roleIndex].setValue(info)
except DatasetConstraintError as ex:
# Give the user a chance to repair the problem.
filename = files[0] + "\n...\n" + files[-1]
return_val = [False]
self.handleDatasetConstraintError( info, filename, ex, roleIndex, laneIndex, return_val )
if not return_val[0]:
# Not successfully repaired. Roll back the changes and give up.
self.topLevelOperator.DatasetGroup.resize(originalNumLanes)
finally:
self.parentApplet.busy = False
self.parentApplet.appletStateUpdateRequested.emit()
req = Request( importStack )
req.notify_finished( lambda result: self.showDataset(laneIndex, roleIndex) )
req.notify_failed( partial(self.handleFailedStackLoad, files, originalNumLanes ) )
req.submit()
def addStack(self, roleIndex, laneIndex):
"""
The user clicked the "Import Stack Files" button.
"""
stackDlg = StackFileSelectionWidget(self)
stackDlg.exec_()
if stackDlg.result() != QDialog.Accepted:
return
files = stackDlg.selectedFiles
sequence_axis = stackDlg.sequence_axis
if len(files) == 0:
return
cwd = self.topLevelOperator.WorkingDirectory.value
info = DatasetInfo(os.path.pathsep.join(files), cwd=cwd)
originalNumLanes = len(self.topLevelOperator.DatasetGroup)
if laneIndex is None or laneIndex == -1:
laneIndex = len(self.topLevelOperator.DatasetGroup)
if len(self.topLevelOperator.DatasetGroup) < laneIndex + 1:
self.topLevelOperator.DatasetGroup.resize(laneIndex + 1)
def importStack():
self.parentApplet.busy = True
self.parentApplet.appletStateUpdateRequested()
# Serializer will update the operator for us, which will propagate to the GUI.
try:
self.serializer.importStackAsLocalDataset(info, sequence_axis)
try:
self.topLevelOperator.DatasetGroup[laneIndex][
roleIndex].setValue(info)
except DatasetConstraintError as ex:
# Give the user a chance to repair the problem.
filename = files[0] + "\n...\n" + files[-1]
return_val = [False]
self.parentApplet.busy = False # required for possible fixing dialogs from DatasetConstraintError
self.parentApplet.appletStateUpdateRequested()
self.handleDatasetConstraintError(info, filename, ex,
roleIndex, laneIndex,
return_val)
if not return_val[0]:
# Not successfully repaired. Roll back the changes and give up.
self.topLevelOperator.DatasetGroup.resize(
originalNumLanes)
finally:
self.parentApplet.busy = False
self.parentApplet.appletStateUpdateRequested()
req = Request(importStack)
req.notify_finished(
lambda result: self.showDataset(laneIndex, roleIndex))
req.notify_failed(
partial(self.handleFailedStackLoad, files, originalNumLanes))
req.submit()
def _triggerTableUpdate(self):
# Check that object area is included in selected features
featureNames = self.topLevelOperatorView.SelectedFeatures.value
if 'Standard Object Features' not in featureNames or 'Count' not in featureNames['Standard Object Features']:
box = QMessageBox(QMessageBox.Warning,
'Warning',
'Object area is not a selected feature. Please select this feature on: \"Standard Object Features > Shape > Size in pixels\"',
QMessageBox.NoButton,
self)
box.show()
return
# Clear table
self.table.clearContents()
self.table.setRowCount(0)
self.table.setSortingEnabled(False)
self.progressBar.show()
self.computeButton.setEnabled(False)
def compute_features_for_frame(tIndex, t, features):
# Compute features and labels (called in parallel from request pool)
roi = [slice(None) for i in range(len(self.topLevelOperatorView.LabelImages.meta.shape))]
roi[tIndex] = slice(t, t+1)
roi = tuple(roi)
frame = self.topLevelOperatorView.SegmentationImages(roi).wait()
frame = frame.squeeze().astype(numpy.uint32, copy=False)
# Dirty trick: We don't care what we're passing here for the 'image' parameter,
# but vigra insists that we pass *something*, so we'll cast the label image as float32.
features[t] = vigra.analysis.extractRegionFeatures(frame.view(numpy.float32),
frame,
['Count'],
ignoreLabel=0)
tIndex = self.topLevelOperatorView.SegmentationImages.meta.axistags.index('t')
tMax = self.topLevelOperatorView.SegmentationImages.meta.shape[tIndex]
features = {}
labels = {}
def compute_all_features():
# Compute features in parallel
pool = RequestPool()
for t in range(tMax):
pool.add( Request( partial(compute_features_for_frame, tIndex, t, features) ) )
pool.wait()
# Compute labels
labels = self.topLevelOperatorView.LabelInputs([]).wait()
req = Request(compute_all_features)
req.notify_finished( partial(self._populateTable, features, labels) )
req.submit()
def exportFinalSegmentation(self,
outputPath,
axisorder,
progressCallback=None):
assert self.FinalSegmentation.ready(
), "Can't export yet: The final segmentation isn't ready!"
logger.info("Starting Final Segmentation Export...")
opTranspose = OpReorderAxes(parent=self)
opTranspose.AxisOrder.setValue(axisorder)
opTranspose.Input.connect(self.FinalSegmentation)
f = h5py.File(outputPath, 'w')
opExporter = OpH5WriterBigDataset(parent=self)
opExporter.hdf5File.setValue(f)
opExporter.hdf5Path.setValue('split_result')
opExporter.Image.connect(opTranspose.Output)
if progressCallback is not None:
opExporter.progressSignal.subscribe(progressCallback)
req = Request(partial(self._runExporter, opExporter))
def cleanOps():
opExporter.cleanUp()
opTranspose.cleanUp()
def handleFailed(exc, exc_info):
cleanOps()
f.close()
import traceback
traceback.print_tb(exc_info[2])
msg = "Final Segmentation export FAILED due to the following error:\n{}".format(
exc)
logger.error(msg)
def handleFinished(result):
try:
cleanOps()
logger.info("FINISHED Final Segmentation Export")
finally:
f.close()
def handleCancelled():
cleanOps()
f.close()
logger.info("Final Segmentation export was cancelled!")
req.notify_failed(handleFailed)
req.notify_finished(handleFinished)
req.notify_cancelled(handleCancelled)
req.submit()
return req # Returned in case the user wants to cancel it.
def _train_forests_with_feature_importance(forests,
X,
y,
feature_names,
export_path=None):
"""
Train all RFs (in parallel) and compute feature importances while doing so.
The importances table will be logged as INFO, and also exported to a file if export_path is given.
Returns: oobs and importances
"""
oobs = [None] * len(forests)
importances = [None] * len(forests)
def store_training_results(i, training_results):
oob, importance_results = training_results
oobs[i] = oob
importances[i] = importance_results
with Timer() as train_timer:
pool = RequestPool()
for i, forest in enumerate(forests):
req = Request(partial(forest.learnRFWithFeatureSelection, X,
y))
# save the training results
req.notify_finished(partial(store_training_results, i))
pool.add(req)
pool.wait()
logger.info("Training took, {} seconds".format(train_timer.seconds()))
# Forests may have different numbers of trees,
# so take a weighted average of their importances
tree_counts = [f.treeCount() for f in forests]
weights = numpy.array(tree_counts).astype(float)
weights /= weights.sum()
named_importances = collections.OrderedDict(
list(
zip(feature_names,
numpy.average(importances, weights=weights, axis=0))))
importance_table = generate_importance_table(named_importances,
sort="overall",
export_path=export_path)
logger.info(
"Feature importance measurements during training: \n{}".format(
importance_table))
return oobs, named_importances
def addStack(self, roleIndex, laneIndex):
"""
The user clicked the "Import Stack Files" button.
"""
stackDlg = StackFileSelectionWidget(self)
stackDlg.exec_()
if stackDlg.result() != QDialog.Accepted :
return
files = stackDlg.selectedFiles
sequence_axis = stackDlg.sequence_axis
if len(files) == 0:
return
cwd = self.topLevelOperator.WorkingDirectory.value
info = DatasetInfo(os.path.pathsep.join(files), cwd=cwd)
originalNumLanes = len(self.topLevelOperator.DatasetGroup)
if laneIndex is None or laneIndex == -1:
laneIndex = len(self.topLevelOperator.DatasetGroup)
if len(self.topLevelOperator.DatasetGroup) < laneIndex+1:
self.topLevelOperator.DatasetGroup.resize(laneIndex+1)
def importStack():
self.parentApplet.busy = True
self.parentApplet.appletStateUpdateRequested()
# Serializer will update the operator for us, which will propagate to the GUI.
try:
self.serializer.importStackAsLocalDataset( info, sequence_axis )
try:
self.topLevelOperator.DatasetGroup[laneIndex][roleIndex].setValue(info)
except DatasetConstraintError as ex:
# Give the user a chance to repair the problem.
filename = files[0] + "\n...\n" + files[-1]
return_val = [False]
self.parentApplet.busy = False # required for possible fixing dialogs from DatasetConstraintError
self.parentApplet.appletStateUpdateRequested()
self.handleDatasetConstraintError( info, filename, ex, roleIndex, laneIndex, return_val )
if not return_val[0]:
# Not successfully repaired. Roll back the changes and give up.
self.topLevelOperator.DatasetGroup.resize(originalNumLanes)
finally:
self.parentApplet.busy = False
self.parentApplet.appletStateUpdateRequested()
req = Request( importStack )
req.notify_finished( lambda result: self.showDataset(laneIndex, roleIndex) )
req.notify_failed( partial(self.handleFailedStackLoad, files, originalNumLanes ) )
req.submit()
def exportFinalSegmentation(self, outputPath, axisorder, progressCallback=None):
assert self.FinalSegmentation.ready(), "Can't export yet: The final segmentation isn't ready!"
logger.info("Starting Final Segmentation Export...")
opTranspose = OpReorderAxes( parent=self )
opTranspose.AxisOrder.setValue( axisorder )
opTranspose.Input.connect( self.FinalSegmentation )
f = h5py.File(outputPath, 'w')
opExporter = OpH5WriterBigDataset(parent=self)
opExporter.hdf5File.setValue( f )
opExporter.hdf5Path.setValue( 'split_result' )
opExporter.Image.connect( opTranspose.Output )
if progressCallback is not None:
opExporter.progressSignal.subscribe( progressCallback )
req = Request( partial(self._runExporter, opExporter) )
def cleanOps():
opExporter.cleanUp()
opTranspose.cleanUp()
def handleFailed( exc, exc_info ):
cleanOps()
f.close()
import traceback
traceback.print_tb(exc_info[2])
msg = "Final Segmentation export FAILED due to the following error:\n{}".format( exc )
logger.error( msg )
def handleFinished( result ):
try:
cleanOps()
logger.info("FINISHED Final Segmentation Export")
finally:
f.close()
def handleCancelled():
cleanOps()
f.close()
logger.info( "Final Segmentation export was cancelled!" )
req.notify_failed( handleFailed )
req.notify_finished( handleFinished )
req.notify_cancelled( handleCancelled )
req.submit()
return req # Returned in case the user wants to cancel it.
def _train_forests(forests, X, y):
"""
Train all RFs (in parallel), and return the oobs.
"""
oobs = [None] * len(forests)
def store_oob_results(i, oob):
oobs[i] = oob
with Timer() as train_timer:
pool = RequestPool()
for i, forest in enumerate(forests):
req = Request( partial(forest.learnRF, X, y) )
# save the oob results
req.notify_finished( partial( store_oob_results, i ) )
pool.add( req )
pool.wait()
logger.info("Training took, {} seconds".format( train_timer.seconds() ) )
return oobs
def run(self, opExport):
"""
Start the export and return immediately (after showing the progress dialog).
:param opExport: The export object to execute.
It must have a 'run_export()' method and a 'progressSignal' member.
"""
progressDlg = MultiStepProgressDialog(parent=self.parent())
progressDlg.setNumberOfSteps(1)
def _forwardProgressToGui(progress):
self._forwardingSignal.emit(
partial(progressDlg.setStepProgress, progress))
opExport.progressSignal.subscribe(_forwardProgressToGui)
def _onFinishExport(*args): # Also called on cancel
self._forwardingSignal.emit(progressDlg.finishStep)
def _onFail(exc, exc_info):
import traceback
traceback.print_tb(exc_info[2])
msg = "Failed to export layer due to the following error:\n{}".format(
exc)
self._forwardingSignal.emit(
partial(QMessageBox.critical, self.parent(), "Export Failed",
msg))
self._forwardingSignal.emit(progressDlg.setFailed)
# Use a request to execute in the background
req = Request(opExport.run_export)
req.notify_cancelled(_onFinishExport)
req.notify_finished(_onFinishExport)
req.notify_failed(_onFail)
# Allow cancel.
progressDlg.rejected.connect(req.cancel)
# Start the export
req.submit()
# Execute the progress dialog
# (We can block the thread here because the QDialog spins up its own event loop.)
progressDlg.exec_()
def create_and_train(self, X, y, feature_names=None):
# Distribute trees as evenly as possible
tree_counts = numpy.array([self._num_trees // self._num_forests] *
self._num_forests)
tree_counts[:self._num_trees % self._num_forests] += 1
assert tree_counts.sum() == self._num_trees
tree_counts = map(int, tree_counts)
tree_counts[:] = (tree_count for tree_count in tree_counts
if tree_count != 0)
logger.debug("Training parallel vigra RF")
# Save for future reference
known_labels = numpy.unique(y)
X = numpy.asarray(X, numpy.float32)
y = numpy.asarray(y, numpy.uint32)
if y.ndim == 1:
y = y[:, numpy.newaxis]
assert X.ndim == 2
assert len(X) == len(y)
# Create N forests
forests = []
for tree_count in tree_counts:
forests.append(
vigra.learning.RandomForest(tree_count, **self._kwargs))
# Train them all in parallel
oobs = [None] * len(forests)
pool = RequestPool()
for i, forest in enumerate(forests):
req = Request(partial(forest.learnRF, X, y))
# save the oobs
req.notify_finished(partial(oobs.__setitem__, i))
pool.add(req)
with Timer() as timer:
pool.wait()
logger.info("Training completed in {} seconds. Average OOB: {}".format(
timer.seconds(), numpy.average(oobs)))
return ParallelVigraRfLazyflowClassifier(forests, oobs, known_labels,
feature_names)
def execute(self, slot, subindex, roi, result):
assert slot == self.LabelAndFeatureMatrix
self.progressSignal(0.0)
# Technically, this could result in strange progress reporting if execute()
# is called by multiple threads in parallel.
# This could be fixed with some fancier progress state, but
# (1) We don't expect that to by typical, and
# (2) progress reporting is merely informational.
num_dirty_blocks = len(self._dirty_blocks)
def update_progress(result):
remaining_dirty = len(self._dirty_blocks)
percent_complete = 95.0 * (num_dirty_blocks -
remaining_dirty) / num_dirty_blocks
self.progressSignal(percent_complete)
# Update all dirty blocks in the cache
logger.debug("Updating {} dirty blocks".format(num_dirty_blocks))
pool = RequestPool()
for block_start in self._dirty_blocks:
req = Request(partial(self._update_block, block_start))
req.notify_finished(update_progress)
pool.add(req)
pool.wait()
# Concatenate the all blockwise results
if self._blockwise_feature_matrices:
total_feature_matrix = numpy.concatenate(
self._blockwise_feature_matrices.values(), axis=0)
else:
# No label points at all.
# Return an empty label&feature matrix (of the correct shape)
num_feature_channels = self.FeatureImage.meta.shape[-1]
total_feature_matrix = numpy.ndarray(shape=(0, 1 +
num_feature_channels),
dtype=numpy.float)
self.progressSignal(100.0)
logger.debug("After update, there are {} clean blocks".format(
len(self._blockwise_feature_matrices)))
result[0] = total_feature_matrix
def test_cancellation_behavior():
"""
If a request is cancelled while it was waiting on a lock,
it should raise the CancellationException.
"""
lock = RequestLock()
lock.acquire()
def f():
try:
with lock:
assert False
except Request.CancellationException:
pass
else:
assert False
finished = [False]
cancelled = [False]
failed = [False]
def handle_finished(result):
finished[0] = True
def handle_cancelled():
cancelled[0] = True
def handle_failed(*args):
failed[0] = True
req = Request(f)
req.notify_finished(handle_finished)
req.notify_failed(handle_failed)
req.notify_cancelled(handle_cancelled)
req.submit()
req.cancel()
time.sleep(0.1)
lock.release()
time.sleep(0.1)
assert not finished[0] and not failed[0] and cancelled[0]
def test_cancellation_behavior():
"""
If a request is cancelled while it was waiting on a lock,
it should raise the CancellationException.
"""
lock = RequestLock()
lock.acquire()
def f():
try:
with lock:
assert False
except Request.CancellationException:
pass
else:
assert False
finished = [False]
cancelled = [False]
failed = [False]
def handle_finished(result):
finished[0] = True
def handle_cancelled():
cancelled[0] = True
def handle_failed(*args):
failed[0] = True
req = Request(f)
req.notify_finished(handle_finished)
req.notify_failed(handle_failed)
req.notify_cancelled(handle_cancelled)
req.submit()
req.cancel()
time.sleep(0.1)
lock.release()
time.sleep(0.1)
assert not finished[0] and not failed[0] and cancelled[0]
def _train_forests_with_feature_importance(forests, X, y, feature_names, export_path=None):
"""
Train all RFs (in parallel) and compute feature importances while doing so.
The importances table will be logged as INFO, and also exported to a file if export_path is given.
Returns: oobs and importances
"""
oobs = [None] * len(forests)
importances = [None] * len(forests)
def store_training_results(i, training_results):
oob, importance_results = training_results
oobs[i] = oob
importances[i] = importance_results
with Timer() as train_timer:
pool = RequestPool()
for i, forest in enumerate(forests):
req = Request(partial(forest.learnRFWithFeatureSelection, X, y))
# save the training results
req.notify_finished(partial(store_training_results, i))
pool.add(req)
pool.wait()
logger.info("Training took, {} seconds".format(train_timer.seconds()))
# Forests may have different numbers of trees,
# so take a weighted average of their importances
tree_counts = [f.treeCount() for f in forests]
weights = numpy.array(tree_counts).astype(float)
weights /= weights.sum()
named_importances = collections.OrderedDict(
list(zip(feature_names, numpy.average(importances, weights=weights, axis=0)))
)
importance_table = generate_importance_table(named_importances, sort="overall", export_path=export_path)
logger.info("Feature importance measurements during training: \n{}".format(importance_table))
return oobs, named_importances
def run_export(self):
role_names = self.parentApplet.dataSelectionApplet.topLevelOperator.DatasetRoles.value
# Prepare file lists in an OrderedDict
role_path_dict = OrderedDict(
(role_name, self._data_role_widgets[role_name].filepaths)
for role_name
in role_names
)
dominant_role_name = role_names[0]
num_paths = len(role_path_dict[dominant_role_name])
if num_paths == 0:
return
for role_name in role_names[1:]:
paths = role_path_dict[role_name]
if len(paths) == 0:
role_path_dict[role_name] = [None] * num_paths
if len(role_path_dict[role_name]) != num_paths:
raise BatchProcessingDataConstraintException(
f"Number of files for '{role_name!r}' does not match! "
f"Exptected {num_paths} files."
)
# Run the export in a separate thread
export_req = Request(partial(self.parentApplet.run_export, role_path_dict))
export_req.notify_failed(self.handle_batch_processing_failure)
export_req.notify_finished(self.handle_batch_processing_finished)
export_req.notify_cancelled(self.handle_batch_processing_cancelled)
self.export_req = export_req
self.parentApplet.busy = True
self.parentApplet.appletStateUpdateRequested()
self.cancel_button.setVisible(True)
self.run_button.setEnabled(False)
# Start the export
export_req.submit()
def create_and_train(self, X, y, feature_names=None):
# Distribute trees as evenly as possible
tree_counts = numpy.array( [self._num_trees // self._num_forests] * self._num_forests )
tree_counts[:self._num_trees % self._num_forests] += 1
assert tree_counts.sum() == self._num_trees
tree_counts = map(int, tree_counts)
tree_counts[:] = (tree_count for tree_count in tree_counts if tree_count != 0)
logger.debug( "Training parallel vigra RF" )
# Save for future reference
known_labels = numpy.unique(y)
X = numpy.asarray(X, numpy.float32)
y = numpy.asarray(y, numpy.uint32)
if y.ndim == 1:
y = y[:, numpy.newaxis]
assert X.ndim == 2
assert len(X) == len(y)
# Create N forests
forests = []
for tree_count in tree_counts:
forests.append( vigra.learning.RandomForest(tree_count, **self._kwargs) ) # HERE <--- this links to C++ library
# Train them all in parallel
oobs = [None] * len(forests)
pool = RequestPool()
for i, forest in enumerate(forests):
req = Request( partial(forest.learnRF, X, y) )
# save the oobs
req.notify_finished( partial( oobs.__setitem__, i ) )
pool.add( req )
with Timer() as timer:
pool.wait()
logger.info( "Training completed in {} seconds. Average OOB: {}".format( timer.seconds(), numpy.average(oobs) ) )
return ParallelVigraRfLazyflowClassifier( forests, oobs, known_labels, feature_names )
def run(self, opExport):
"""
Start the export and return immediately (after showing the progress dialog).
:param opExport: The export object to execute.
It must have a 'run_export()' method and a 'progressSignal' member.
"""
progressDlg = MultiStepProgressDialog(parent=self.parent())
progressDlg.setNumberOfSteps(1)
def _forwardProgressToGui(progress):
self._forwardingSignal.emit( partial( progressDlg.setStepProgress, progress ) )
opExport.progressSignal.subscribe( _forwardProgressToGui )
def _onFinishExport( *args ): # Also called on cancel
self._forwardingSignal.emit( progressDlg.finishStep )
def _onFail( exc, exc_info ):
import traceback
traceback.print_tb(exc_info[2])
msg = "Failed to export layer due to the following error:\n{}".format( exc )
self._forwardingSignal.emit( partial(QMessageBox.critical, self.parent(), "Export Failed", msg) )
self._forwardingSignal.emit( progressDlg.setFailed )
# Use a request to execute in the background
req = Request( opExport.run_export )
req.notify_cancelled( _onFinishExport )
req.notify_finished( _onFinishExport )
req.notify_failed( _onFail )
# Allow cancel.
progressDlg.rejected.connect( req.cancel )
# Start the export
req.submit()
# Execute the progress dialog
# (We can block the thread here because the QDialog spins up its own event loop.)
progressDlg.exec_()
def export_object_data(self, lane_index, show_gui=False, filename_suffix=""):
"""
Initialize progress displays and start the actual export in a new thread using the lazyflow.request framework
:param settings: the settings from the GUI export dialog
:type settings: dict
:param selected_features: the features to export from the GUI dialog
:type selected_features: list
:param gui: the Progress bar and callbacks for finish/fail/cancel see ExportingGui.show_export_dialog
:type gui: dict
"""
settings, selected_features = self.get_table_export_settings()
self.save_export_progress_dialog(None)
if not show_gui:
progress_display = ProgressPrinter("Export Progress", xrange(100, -1, -5), 2)
gui = None
else:
from ilastik.widgets.progressDialog import ProgressDialog
progress = ProgressDialog(["Feature Data", "Labeling Rois", "Raw Image", "Exporting"])
progress.set_busy(True)
progress.show()
gui = {
"dialog": progress,
"ok": partial(progress.safe_popup, "information", "Information", "Export successful!"),
"cancel": partial(progress.safe_popup, "information", "Information", "Export cancelled!"),
"fail": partial(progress.safe_popup, "critical", "Critical", "Export failed!"),
"unlock": self.unlock_gui,
"lock": self.lock_gui,
}
progress_display = gui["dialog"]
self.save_export_progress_dialog(progress_display)
export = partial(self.do_export, settings, selected_features, progress_display, lane_index, filename_suffix)
request = Request(export)
if gui is not None:
if "fail" in gui:
request.notify_failed(gui["fail"])
if "ok" in gui:
request.notify_finished(gui["ok"])
if "cancel" in gui:
request.notify_cancelled(gui["cancel"])
if "unlock" in gui:
request.notify_cancelled(gui["unlock"])
request.notify_failed(gui["unlock"])
request.notify_finished(gui["unlock"])
if "lock" in gui:
lock = gui["lock"]
lock()
request.notify_failed(self.export_failed)
request.notify_finished(self.export_finished)
request.notify_cancelled(self.export_cancelled)
request.submit()
if gui is not None and "dialog" in gui:
progress_display.cancel.connect(request.cancel)
return request
def execute(self, slot, subindex, roi, result):
assert slot == self.LabelAndFeatureMatrix
self.progressSignal(0.0)
# Technically, this could result in strange progress reporting if execute()
# is called by multiple threads in parallel.
# This could be fixed with some fancier progress state, but
# (1) We don't expect that to by typical, and
# (2) progress reporting is merely informational.
num_dirty_blocks = len( self._dirty_blocks )
def update_progress( result ):
remaining_dirty = len( self._dirty_blocks )
percent_complete = 95.0*(num_dirty_blocks - remaining_dirty)/num_dirty_blocks
self.progressSignal( percent_complete )
# Update all dirty blocks in the cache
logger.debug( "Updating {} dirty blocks".format(num_dirty_blocks) )
pool = RequestPool()
for block_start in self._dirty_blocks:
req = Request( partial(self._update_block, block_start ) )
req.notify_finished( update_progress )
pool.add( req )
pool.wait()
# Concatenate the all blockwise results
if self._blockwise_feature_matrices:
total_feature_matrix = numpy.concatenate( self._blockwise_feature_matrices.values(), axis=0 )
else:
# No label points at all.
# Return an empty label&feature matrix (of the correct shape)
num_feature_channels = self.FeatureImage.meta.shape[-1]
total_feature_matrix = numpy.ndarray( shape=(0, 1 + num_feature_channels), dtype=numpy.float )
self.progressSignal(100.0)
logger.debug( "After update, there are {} clean blocks".format( len(self._blockwise_feature_matrices) ) )
result[0] = total_feature_matrix
def export_object_data(self, settings, selected_features, gui=None):
"""
Initialize progress displays and start the actual export in a new thread using the lazyflow.request framework
:param settings: the settings from the GUI export dialog
:type settings: dict
:param selected_features: the features to export from the GUI dialog
:type selected_features: list
:param gui: the Progress bar and callbacks for finish/fail/cancel see ExportingGui.show_export_dialog
:type gui: dict
"""
self.save_export_progress_dialog(None)
if gui is None or "dialog" not in gui:
progress_display = ProgressPrinter("Export Progress", xrange(100, -1, -5), 2)
else:
progress_display = gui["dialog"]
self.save_export_progress_dialog(progress_display)
export = partial(self.do_export, settings, selected_features, progress_display)
request = Request(export)
if gui is not None:
if "fail" in gui:
request.notify_failed(gui["fail"])
if "ok" in gui:
request.notify_finished(gui["ok"])
if "cancel" in gui:
request.notify_cancelled(gui["cancel"])
if "unlock" in gui:
request.notify_cancelled(gui["unlock"])
request.notify_failed(gui["unlock"])
request.notify_finished(gui["unlock"])
if "lock" in gui:
lock = gui["lock"]
lock()
request.notify_failed(self.export_failed)
request.notify_finished(self.export_finished)
request.notify_cancelled(self.export_cancelled)
request.submit()
if gui is not None and "dialog" in gui:
progress_display.cancel.connect(request.cancel)
def _onTrackButtonPressed(self):
if not self.mainOperator.ObjectFeatures.ready():
self._criticalMessage("You have to compute object features first.")
return
def _track():
self.applet.busy = True
self.applet.appletStateUpdateRequested.emit()
app = self._drawer.appSpinBox.value()
dis = self._drawer.disSpinBox.value()
opp = self._drawer.oppSpinBox.value()
noiserate = self._drawer.noiseRateSpinBox.value()
noiseweight = self._drawer.noiseWeightSpinBox.value()
epGap = self._drawer.epGapSpinBox.value()
n_neighbors = self._drawer.nNeighborsSpinBox.value()
with_div = self._drawer.withDivisionsBox.isChecked()
cplex_timeout = None
if len(str(self._drawer.timeoutBox.text())):
cplex_timeout = int(self._drawer.timeoutBox.text())
from_t = self._drawer.from_time.value()
to_t = self._drawer.to_time.value()
from_x = self._drawer.from_x.value()
to_x = self._drawer.to_x.value()
from_y = self._drawer.from_y.value()
to_y = self._drawer.to_y.value()
from_z = self._drawer.from_z.value()
to_z = self._drawer.to_z.value()
from_size = self._drawer.from_size.value()
to_size = self._drawer.to_size.value()
try:
self.mainOperator.track(time_range=range(from_t, to_t + 1),
x_range=(from_x, to_x + 1),
y_range=(from_y, to_y + 1),
z_range=(from_z, to_z + 1),
size_range=(from_size, to_size + 1),
x_scale=self._drawer.x_scale.value(),
y_scale=self._drawer.y_scale.value(),
z_scale=self._drawer.z_scale.value(),
app=app,
dis=dis,
noiserate=noiserate,
noiseweight=noiseweight,
opp=opp,
ep_gap=epGap,
n_neighbors=n_neighbors,
with_div=with_div,
cplex_timeout=cplex_timeout)
except Exception:
ex_type, ex, tb = sys.exc_info()
log_exception(logger)
self._criticalMessage("Exception(" + str(ex_type) + "): " +
str(ex))
return
def _handle_finished(*args):
self.applet.progressSignal.emit(100)
self._drawer.TrackButton.setEnabled(True)
self._drawer.exportButton.setEnabled(True)
self._drawer.exportTifButton.setEnabled(True)
self._setLayerVisible("Objects", False)
self.applet.busy = False
self.applet.appletStateUpdateRequested.emit()
def _handle_failure(exc, exc_info):
self.applet.progressSignal.emit(100)
msg = "Exception raised during tracking. See traceback above.\n"
log_exception(logger, msg, exc_info)
self._drawer.TrackButton.setEnabled(True)
self.applet.busy = False
self.applet.appletStateUpdateRequested.emit()
self._drawer.TrackButton.setEnabled(False)
self.applet.progressSignal.emit(0)
self.applet.progressSignal.emit(-1)
req = Request(_track)
req.notify_failed(_handle_failure)
req.notify_finished(_handle_finished)
req.submit()
def export_object_data(self,
lane_index,
show_gui=False,
filename_suffix=""):
"""
Initialize progress displays and start the actual export in a new thread
using the lazyflow.request framework
Args:
lane_index (int): Index of the lane to be exported
show_gui (bool, optional): boolean to determine whether or not to
show gui
filename_suffix (str, optional): If provided, appended to the
filename (before the extension)
Returns:
lazyflow.request.Request: Request object from which the result can
be obtained.
"""
settings, selected_features = self.get_table_export_settings()
if not settings:
return Request.with_value(None)
self.save_export_progress_dialog(None)
if not show_gui:
progress_display = ProgressPrinter("Export Progress",
range(100, -1, -5), 2)
gui = None
else:
from ilastik.widgets.progressDialog import ProgressDialog
progress = ProgressDialog(
["Feature Data", "Labeling Rois", "Raw Image", "Exporting"])
progress.set_busy(True)
progress.show()
gui = {
"dialog":
progress,
"ok":
partial(progress.safe_popup, "information", "Information",
"Export successful!"),
"cancel":
partial(progress.safe_popup, "information", "Information",
"Export cancelled!"),
"fail":
partial(progress.safe_popup, "critical", "Critical",
"Export failed!"),
"unlock":
self.unlock_gui,
"lock":
self.lock_gui,
}
progress_display = gui["dialog"]
self.save_export_progress_dialog(progress_display)
export = partial(self.do_export, settings, selected_features,
progress_display, lane_index, filename_suffix)
request = Request(export)
if gui is not None:
if "fail" in gui:
request.notify_failed(gui["fail"])
if "ok" in gui:
request.notify_finished(gui["ok"])
if "cancel" in gui:
request.notify_cancelled(gui["cancel"])
if "unlock" in gui:
request.notify_cancelled(gui["unlock"])
request.notify_failed(gui["unlock"])
request.notify_finished(gui["unlock"])
if "lock" in gui:
lock = gui["lock"]
lock()
request.notify_failed(self.export_failed)
request.notify_finished(self.export_finished)
request.notify_cancelled(self.export_cancelled)
request.submit()
if gui is not None and "dialog" in gui:
progress_display.cancel.connect(request.cancel)
return request
def exportFinalSupervoxels(self, outputPath, axisorder, progressCallback=None):
"""
Executes the export process within a request.
The (already-running) request is returned, in case you want to wait for it or monitor its progress.
"""
assert self.FinalSupervoxels.ready(), "Can't export yet: The final segmentation isn't ready!"
logger.info("Starting Final Segmentation Export...")
opTranspose = OpReorderAxes( parent=self )
opTranspose.AxisOrder.setValue( axisorder )
opTranspose.Input.connect( self.FinalSupervoxels )
f = h5py.File(outputPath, 'w')
opExporter = OpH5WriterBigDataset(parent=self)
opExporter.hdf5File.setValue( f )
opExporter.hdf5Path.setValue( 'stack' )
opExporter.Image.connect( opTranspose.Output )
if progressCallback is not None:
opExporter.progressSignal.subscribe( progressCallback )
req = Request( partial(self._runExporter, opExporter) )
def cleanOps():
opExporter.cleanUp()
opTranspose.cleanUp()
def handleFailed( exc, exc_info ):
cleanOps()
f.close()
import traceback
traceback.print_tb(exc_info[2])
msg = "Final Supervoxel export FAILED due to the following error:\n{}".format( exc )
logger.error( msg )
def handleFinished( result ):
# Generate the mapping transforms dataset
mapping = self._opAccumulateFinalImage.Mapping.value
num_labels = mapping.keys()[-1][1]
transform = numpy.zeros( shape=(num_labels, 2), dtype=numpy.uint32 )
for (start, stop), body_id in mapping.items():
for supervoxel_label in range(start, stop):
transform[supervoxel_label][0] = supervoxel_label
if body_id == -1:
# Special case: -1 means "identity transform" for this supervoxel
# (Which is really an untouched raveler body)
transform[supervoxel_label][1] = supervoxel_label
else:
transform[supervoxel_label][1] = body_id
# Save the transform before closing the file
f.create_dataset('transforms', data=transform)
# Copy all other datasets from the original segmentation file.
ravelerSegmentationInfo = self.DatasetInfos[2].value
pathComponents = PathComponents(ravelerSegmentationInfo.filePath, self.WorkingDirectory.value)
with h5py.File(pathComponents.externalPath, 'r') as originalFile:
for k,dset in originalFile.items():
if k not in ['transforms', 'stack']:
f.copy(dset, k)
try:
cleanOps()
logger.info("FINISHED Final Supervoxel Export")
finally:
f.close()
def handleCancelled():
cleanOps()
f.close()
logger.info( "Final Supervoxel export was cancelled!" )
req.notify_failed( handleFailed )
req.notify_finished( handleFinished )
req.notify_cancelled( handleCancelled )
req.submit()
return req # Returned in case the user wants to cancel it.
def export_object_data(self,
lane_index,
show_gui=False,
filename_suffix=""):
"""
Initialize progress displays and start the actual export in a new thread using the lazyflow.request framework
:param settings: the settings from the GUI export dialog
:type settings: dict
:param selected_features: the features to export from the GUI dialog
:type selected_features: list
:param gui: the Progress bar and callbacks for finish/fail/cancel see ExportingGui.show_export_dialog
:type gui: dict
"""
settings, selected_features = self.get_table_export_settings()
self.save_export_progress_dialog(None)
if not show_gui:
progress_display = ProgressPrinter("Export Progress",
xrange(100, -1, -5), 2)
gui = None
else:
from ilastik.widgets.progressDialog import ProgressDialog
progress = ProgressDialog(
["Feature Data", "Labeling Rois", "Raw Image", "Exporting"])
progress.set_busy(True)
progress.show()
gui = {
"dialog":
progress,
"ok":
partial(progress.safe_popup, "information", "Information",
"Export successful!"),
"cancel":
partial(progress.safe_popup, "information", "Information",
"Export cancelled!"),
"fail":
partial(progress.safe_popup, "critical", "Critical",
"Export failed!"),
"unlock":
self.unlock_gui,
"lock":
self.lock_gui
}
progress_display = gui["dialog"]
self.save_export_progress_dialog(progress_display)
export = partial(self.do_export, settings, selected_features,
progress_display, lane_index, filename_suffix)
request = Request(export)
if gui is not None:
if "fail" in gui:
request.notify_failed(gui["fail"])
if "ok" in gui:
request.notify_finished(gui["ok"])
if "cancel" in gui:
request.notify_cancelled(gui["cancel"])
if "unlock" in gui:
request.notify_cancelled(gui["unlock"])
request.notify_failed(gui["unlock"])
request.notify_finished(gui["unlock"])
if "lock" in gui:
lock = gui["lock"]
lock()
request.notify_failed(self.export_failed)
request.notify_finished(self.export_finished)
request.notify_cancelled(self.export_cancelled)
request.submit()
if gui is not None and "dialog" in gui:
progress_display.cancel.connect(request.cancel)
return request
def _onTrackButtonPressed( self ):
if not self.mainOperator.ObjectFeatures.ready():
self._criticalMessage("You have to compute object features first.")
return
def _track():
self.applet.busy = True
self.applet.appletStateUpdateRequested.emit()
app = self._drawer.appSpinBox.value()
dis = self._drawer.disSpinBox.value()
opp = self._drawer.oppSpinBox.value()
noiserate = self._drawer.noiseRateSpinBox.value()
noiseweight = self._drawer.noiseWeightSpinBox.value()
epGap = self._drawer.epGapSpinBox.value()
n_neighbors = self._drawer.nNeighborsSpinBox.value()
with_div = self._drawer.withDivisionsBox.isChecked()
cplex_timeout = None
if len(str(self._drawer.timeoutBox.text())):
cplex_timeout = int(self._drawer.timeoutBox.text())
from_t = self._drawer.from_time.value()
to_t = self._drawer.to_time.value()
from_x = self._drawer.from_x.value()
to_x = self._drawer.to_x.value()
from_y = self._drawer.from_y.value()
to_y = self._drawer.to_y.value()
from_z = self._drawer.from_z.value()
to_z = self._drawer.to_z.value()
from_size = self._drawer.from_size.value()
to_size = self._drawer.to_size.value()
try:
self.mainOperator.track(
time_range = range(from_t, to_t + 1),
x_range = (from_x, to_x + 1),
y_range = (from_y, to_y + 1),
z_range = (from_z, to_z + 1),
size_range = (from_size, to_size + 1),
x_scale = self._drawer.x_scale.value(),
y_scale = self._drawer.y_scale.value(),
z_scale = self._drawer.z_scale.value(),
app=app,
dis=dis,
noiserate = noiserate,
noiseweight = noiseweight,
opp=opp,
ep_gap=epGap,
n_neighbors=n_neighbors,
with_div=with_div,
cplex_timeout=cplex_timeout)
except Exception:
ex_type, ex, tb = sys.exc_info()
traceback.print_tb(tb)
self._criticalMessage("Exception(" + str(ex_type) + "): " + str(ex))
return
def _handle_finished(*args):
self.applet.progressSignal.emit(100)
self._drawer.TrackButton.setEnabled(True)
self._drawer.exportButton.setEnabled(True)
self._drawer.exportTifButton.setEnabled(True)
self._setLayerVisible("Objects", False)
self.applet.busy = False
self.applet.appletStateUpdateRequested.emit()
def _handle_failure( exc, exc_info ):
self.applet.progressSignal.emit(100)
traceback.print_exception(*exc_info)
sys.stderr.write("Exception raised during tracking. See traceback above.\n")
self._drawer.TrackButton.setEnabled(True)
self.applet.busy = False
self.applet.appletStateUpdateRequested.emit()
self._drawer.TrackButton.setEnabled(False)
self.applet.progressSignal.emit(0)
self.applet.progressSignal.emit(-1)
req = Request( _track )
req.notify_failed( _handle_failure )
req.notify_finished( _handle_finished )
req.submit()
def export_object_data(self, lane_index, show_gui=False, filename_suffix=""):
"""
Initialize progress displays and start the actual export in a new thread
using the lazyflow.request framework
Args:
lane_index (int): Index of the lane to be exported
show_gui (bool, optional): boolean to determine whether or not to
show gui
filename_suffix (str, optional): If provided, appended to the
filename (before the extension)
Returns:
lazyflow.request.Request: Request object from which the result can
be obtained.
"""
settings, selected_features = self.get_table_export_settings()
self.save_export_progress_dialog(None)
if not show_gui:
progress_display = ProgressPrinter("Export Progress", range(100, -1, -5), 2)
gui = None
else:
from ilastik.widgets.progressDialog import ProgressDialog
progress = ProgressDialog(["Feature Data", "Labeling Rois", "Raw Image", "Exporting"])
progress.set_busy(True)
progress.show()
gui = {
"dialog": progress,
"ok": partial(progress.safe_popup, "information", "Information", "Export successful!"),
"cancel": partial(progress.safe_popup, "information", "Information", "Export cancelled!"),
"fail": partial(progress.safe_popup, "critical", "Critical", "Export failed!"),
"unlock": self.unlock_gui,
"lock": self.lock_gui
}
progress_display = gui["dialog"]
self.save_export_progress_dialog(progress_display)
export = partial(self.do_export, settings, selected_features, progress_display, lane_index, filename_suffix)
request = Request(export)
if gui is not None:
if "fail" in gui:
request.notify_failed(gui["fail"])
if "ok" in gui:
request.notify_finished(gui["ok"])
if "cancel" in gui:
request.notify_cancelled(gui["cancel"])
if "unlock" in gui:
request.notify_cancelled(gui["unlock"])
request.notify_failed(gui["unlock"])
request.notify_finished(gui["unlock"])
if "lock" in gui:
lock = gui["lock"]
lock()
request.notify_failed(self.export_failed)
request.notify_finished(self.export_finished)
request.notify_cancelled(self.export_cancelled)
request.submit()
if gui is not None and "dialog" in gui:
progress_display.cancel.connect(request.cancel)
return request
def exportFinalSupervoxels(self,
outputPath,
axisorder,
progressCallback=None):
"""
Executes the export process within a request.
The (already-running) request is returned, in case you want to wait for it or monitor its progress.
"""
assert self.FinalSupervoxels.ready(
), "Can't export yet: The final segmentation isn't ready!"
logger.info("Starting Final Segmentation Export...")
opTranspose = OpReorderAxes(parent=self)
opTranspose.AxisOrder.setValue(axisorder)
opTranspose.Input.connect(self.FinalSupervoxels)
f = h5py.File(outputPath, 'w')
opExporter = OpH5WriterBigDataset(parent=self)
opExporter.hdf5File.setValue(f)
opExporter.hdf5Path.setValue('stack')
opExporter.Image.connect(opTranspose.Output)
if progressCallback is not None:
opExporter.progressSignal.subscribe(progressCallback)
req = Request(partial(self._runExporter, opExporter))
def cleanOps():
opExporter.cleanUp()
opTranspose.cleanUp()
def handleFailed(exc, exc_info):
cleanOps()
f.close()
msg = "Final Supervoxel export FAILED due to the following error:\n{}".format(
exc)
log_exception(logger, msg, exc_info)
def handleFinished(result):
# Generate the mapping transforms dataset
mapping = self._opAccumulateFinalImage.Mapping.value
num_labels = mapping.keys()[-1][1]
transform = numpy.zeros(shape=(num_labels, 2), dtype=numpy.uint32)
for (start, stop), body_id in mapping.items():
for supervoxel_label in range(start, stop):
transform[supervoxel_label][0] = supervoxel_label
if body_id == -1:
# Special case: -1 means "identity transform" for this supervoxel
# (Which is really an untouched raveler body)
transform[supervoxel_label][1] = supervoxel_label
else:
transform[supervoxel_label][1] = body_id
# Save the transform before closing the file
f.create_dataset('transforms', data=transform)
# Copy all other datasets from the original segmentation file.
ravelerSegmentationInfo = self.DatasetInfos[2].value
pathComponents = PathComponents(ravelerSegmentationInfo.filePath,
self.WorkingDirectory.value)
with h5py.File(pathComponents.externalPath, 'r') as originalFile:
for k, dset in originalFile.items():
if k not in ['transforms', 'stack']:
f.copy(dset, k)
try:
cleanOps()
logger.info("FINISHED Final Supervoxel Export")
finally:
f.close()
def handleCancelled():
cleanOps()
f.close()
logger.info("Final Supervoxel export was cancelled!")
req.notify_failed(handleFailed)
req.notify_finished(handleFinished)
req.notify_cancelled(handleCancelled)
req.submit()
return req # Returned in case the user wants to cancel it.
def execute(self, slot, subindex, roi, result):
assert slot == self.LabelAndFeatureMatrix
self.progressSignal(0.0)
# Technically, this could result in strange progress reporting if execute()
# is called by multiple threads in parallel.
# This could be fixed with some fancier progress state, but
# (1) We don't expect that to by typical, and
# (2) progress reporting is merely informational.
num_dirty_blocks = len(self._dirty_blocks)
remaining_dirty = [num_dirty_blocks]
def update_progress(result):
remaining_dirty[0] -= 1
percent_complete = 95.0 * (num_dirty_blocks -
remaining_dirty[0]) / num_dirty_blocks
self.progressSignal(percent_complete)
# Update all dirty blocks in the cache
logger.debug("Updating {} dirty blocks".format(num_dirty_blocks))
# Before updating the blocks, ensure that the necessary block locks exist
# It's better to do this now instead of inside each request
# to avoid contention over self._lock
with self._lock:
for block_start in self._dirty_blocks:
if block_start not in self._block_locks:
self._block_locks[block_start] = RequestLock()
# Update each block in its own request.
pool = RequestPool()
reqs = {}
for block_start in self._dirty_blocks:
req = Request(partial(self._get_features_for_block, block_start))
req.notify_finished(update_progress)
reqs[block_start] = req
pool.add(req)
pool.wait()
# Now store the results we got.
# It's better to store the blocks here -- rather than within each request -- to
# avoid contention over self._lock from within every block's request.
with self._lock:
for block_start, req in reqs.items():
if req.result is None:
# 'None' means the block wasn't dirty. No need to update.
continue
labels_and_features_matrix = req.result
self._dirty_blocks.remove(block_start)
if labels_and_features_matrix.shape[0] > 0:
# Update the block entry with the new matrix.
self._blockwise_feature_matrices[
block_start] = labels_and_features_matrix
else:
# All labels were removed from the block,
# So the new feature matrix is empty.
# Just delete its entry from our list.
try:
del self._blockwise_feature_matrices[block_start]
except KeyError:
pass
# Concatenate the all blockwise results
if self._blockwise_feature_matrices:
total_feature_matrix = numpy.concatenate(
self._blockwise_feature_matrices.values(), axis=0)
else:
# No label points at all.
# Return an empty label&feature matrix (of the correct shape)
num_feature_channels = self.FeatureImage.meta.shape[-1]
total_feature_matrix = numpy.ndarray(shape=(0, 1 +
num_feature_channels),
dtype=numpy.float32)
self.progressSignal(100.0)
logger.debug("After update, there are {} clean blocks".format(
len(self._blockwise_feature_matrices)))
result[0] = total_feature_matrix
def execute(self, slot, subindex, roi, result):
assert slot == self.LabelAndFeatureMatrix
self.progressSignal(0.0)
# Technically, this could result in strange progress reporting if execute()
# is called by multiple threads in parallel.
# This could be fixed with some fancier progress state, but
# (1) We don't expect that to by typical, and
# (2) progress reporting is merely informational.
num_dirty_blocks = len( self._dirty_blocks )
remaining_dirty = [num_dirty_blocks]
def update_progress( result ):
remaining_dirty[0] -= 1
percent_complete = 95.0*(num_dirty_blocks - remaining_dirty[0])/num_dirty_blocks
self.progressSignal( percent_complete )
# Update all dirty blocks in the cache
logger.debug( "Updating {} dirty blocks".format(num_dirty_blocks) )
# Before updating the blocks, ensure that the necessary block locks exist
# It's better to do this now instead of inside each request
# to avoid contention over self._lock
with self._lock:
for block_start in self._dirty_blocks:
if block_start not in self._block_locks:
self._block_locks[block_start] = RequestLock()
# Update each block in its own request.
pool = RequestPool()
reqs = {}
for block_start in self._dirty_blocks:
req = Request( partial(self._get_features_for_block, block_start ) )
req.notify_finished( update_progress )
reqs[block_start] = req
pool.add( req )
pool.wait()
# Now store the results we got.
# It's better to store the blocks here -- rather than within each request -- to
# avoid contention over self._lock from within every block's request.
with self._lock:
for block_start, req in reqs.items():
if req.result is None:
# 'None' means the block wasn't dirty. No need to update.
continue
labels_and_features_matrix = req.result
self._dirty_blocks.remove(block_start)
if labels_and_features_matrix.shape[0] > 0:
# Update the block entry with the new matrix.
self._blockwise_feature_matrices[block_start] = labels_and_features_matrix
else:
# All labels were removed from the block,
# So the new feature matrix is empty.
# Just delete its entry from our list.
try:
del self._blockwise_feature_matrices[block_start]
except KeyError:
pass
# Concatenate the all blockwise results
if self._blockwise_feature_matrices:
total_feature_matrix = numpy.concatenate( self._blockwise_feature_matrices.values(), axis=0 )
else:
# No label points at all.
# Return an empty label&feature matrix (of the correct shape)
num_feature_channels = self.FeatureImage.meta.shape[-1]
total_feature_matrix = numpy.ndarray( shape=(0, 1 + num_feature_channels), dtype=numpy.float32 )
self.progressSignal(100.0)
logger.debug( "After update, there are {} clean blocks".format( len(self._blockwise_feature_matrices) ) )
result[0] = total_feature_matrix
