class OpImageOnDiskProvider(Operator): """ This simply wraps a lazyflow OpInputDataReader, but ensures that the metadata (axistags, drange) on the output matches the metadata from the original data (even if the output file format doesn't support metadata fields). """ TransactionSlot = InputSlot() Input = InputSlot() # Used for dtype and shape only. Data is always provided directly from the file. WorkingDirectory = InputSlot() DatasetPath = InputSlot() # A TOTAL path (possibly including a dataset name, e.g. myfile.h5/volume/data Dirty = InputSlot() Output = OutputSlot() def __init__(self, *args, **kwargs): super( OpImageOnDiskProvider, self ).__init__(*args, **kwargs) self._opReader = None self._opMetadataInjector = None # Block diagram: # # (Input.axistags, Input.drange) # \ # DatasetPath ---> opReader ---> opMetadataInjector --> Output # / # WorkingDirectory def setupOutputs( self ): if self._opReader is not None: self.Output.disconnect() if self._opMetadataInjector: self._opMetadataInjector.cleanUp() self._opMetadataInjector = None self._opReader.cleanUp() self._opReader = None try: # Configure the reader dataReady = True self._opReader = OpInputDataReader( parent=self ) self._opReader.WorkingDirectory.setValue( self.WorkingDirectory.value ) self._opReader.FilePath.setValue( self.DatasetPath.value ) # Since most file formats don't save meta-info, # The reader output's axis order may be incorrect. # (For example, if we export in npy format with zxy order, # the Npy reader op will simply assume xyz order when it reads the data.) # Force the metadata back to the correct state by copying select items from Input.meta metadata = {} metadata['axistags'] = self.Input.meta.axistags metadata['drange'] = self.Input.meta.drange self._opMetadataInjector = OpMetadataInjector( parent=self ) self._opMetadataInjector.Input.connect( self._opReader.Output ) self._opMetadataInjector.Metadata.setValue( metadata ) dataReady &= self._opMetadataInjector.Output.meta.shape == self.Input.meta.shape dataReady &= self._opMetadataInjector.Output.meta.dtype == self.Input.meta.dtype if dataReady: self.Output.connect( self._opMetadataInjector.Output ) else: self._opMetadataInjector.cleanUp() self._opMetadataInjector = None self._opReader.cleanUp() self._opReader = None self.Output.meta.NOTREADY = True except OpInputDataReader.DatasetReadError: # Note: If the data is exported as a 'sequence', then this will always be NOTREADY # because the 'path' (e.g. 'myfile_{slice_index}.png' will be nonexistent. # That's okay because a stack is probably too slow to be of use for a preview anyway. if self._opMetadataInjector: self._opMetadataInjector.cleanUp() self._opMetadataInjector = None self._opReader.cleanUp() self._opReader = None # The dataset doesn't exist yet. self.Output.meta.NOTREADY = True def execute(self, slot, subindex, roi, result): assert False, "Output is supposed to be directly connected to an internal operator." def propagateDirty(self, slot, subindex, roi): if slot == self.Input: self.Output.setDirty( roi ) else: self.Output.setDirty( slice(None) )
def import_labeling_layer(labelLayer, labelingSlots, parent_widget=None): """ Prompt the user for layer import settings, and perform the layer import. :param labelLayer: The top label layer source :param labelingSlots: An instance of LabelingGui.LabelingSlots :param parent_widget: The Qt GUI parent object """ writeSeeds = labelingSlots.labelInput assert isinstance( writeSeeds, lazyflow.graph.Slot), "slot is of type %r" % (type(writeSeeds)) opLabels = writeSeeds.getRealOperator() assert isinstance(opLabels, lazyflow.graph.Operator ), "slot's operator is of type %r" % (type(opLabels)) recentlyImported = PreferencesManager().get('labeling', 'recently imported') mostRecentProjectPath = PreferencesManager().get('shell', 'recently opened') mostRecentImageFile = PreferencesManager().get('DataSelection', 'recent image') if recentlyImported: defaultDirectory = os.path.split(recentlyImported)[0] elif mostRecentProjectPath: defaultDirectory = os.path.split(mostRecentProjectPath)[0] elif mostRecentImageFile: defaultDirectory = os.path.split(mostRecentImageFile)[0] else: defaultDirectory = os.path.expanduser('~') fileNames = DataSelectionGui.getImageFileNamesToOpen( parent_widget, defaultDirectory) fileNames = map(str, fileNames) if not fileNames: return PreferencesManager().set('labeling', 'recently imported', fileNames[0]) try: # Initialize operators opImport = OpInputDataReader(parent=opLabels.parent) opCache = OpArrayCache(parent=opLabels.parent) opMetadataInjector = OpMetadataInjector(parent=opLabels.parent) opReorderAxes = OpReorderAxes(parent=opLabels.parent) # Set up the pipeline as follows: # # opImport --> opCache --> opMetadataInjector --------> opReorderAxes --(inject via setInSlot)--> labelInput # / / # User-specified axisorder labelInput.meta.axistags opImport.WorkingDirectory.setValue(defaultDirectory) opImport.FilePath.setValue(fileNames[0] if len(fileNames) == 1 else os.path.pathsep.join(fileNames)) assert opImport.Output.ready() opCache.blockShape.setValue(opImport.Output.meta.shape) opCache.Input.connect(opImport.Output) assert opCache.Output.ready() opMetadataInjector.Input.connect(opCache.Output) metadata = opCache.Output.meta.copy() opMetadataInjector.Metadata.setValue(metadata) opReorderAxes.Input.connect(opMetadataInjector.Output) # Transpose the axes for assignment to the labeling operator. opReorderAxes.AxisOrder.setValue(writeSeeds.meta.getAxisKeys()) # We'll show a little window with a busy indicator while the data is loading busy_dlg = QProgressDialog(parent=parent_widget) busy_dlg.setLabelText("Importing Label Data...") busy_dlg.setCancelButton(None) busy_dlg.setMinimum(100) busy_dlg.setMaximum(100) def close_busy_dlg(*args): QApplication.postEvent(busy_dlg, QCloseEvent()) # Load the data from file into our cache # When it's done loading, close the progress dialog. req = opCache.Output[:] req.notify_finished(close_busy_dlg) req.notify_failed(close_busy_dlg) req.submit() busy_dlg.exec_() readData = req.result maxLabels = len(labelingSlots.labelNames.value) # Can't use return_counts feature because that requires numpy >= 1.9 #unique_read_labels, readLabelCounts = numpy.unique(readData, return_counts=True) # This does the same as the above, albeit slower, and probably with more ram. unique_read_labels = numpy.unique(readData) readLabelCounts = numpy.bincount(readData.flat)[unique_read_labels] labelInfo = (maxLabels, (unique_read_labels, readLabelCounts)) del readData # Ask the user how to interpret the data. settingsDlg = LabelImportOptionsDlg(parent_widget, fileNames, opMetadataInjector.Output, labelingSlots.labelInput, labelInfo) def handle_updated_axes(): # The user is specifying a new interpretation of the file's axes updated_axisorder = str(settingsDlg.axesEdit.text()) metadata = opMetadataInjector.Metadata.value.copy() metadata.axistags = vigra.defaultAxistags(updated_axisorder) opMetadataInjector.Metadata.setValue(metadata) settingsDlg.axesEdit.editingFinished.connect(handle_updated_axes) dlg_result = settingsDlg.exec_() if dlg_result != LabelImportOptionsDlg.Accepted: return # Get user's chosen label mapping from dlg labelMapping = settingsDlg.labelMapping # Get user's chosen offsets. # Offsets in dlg only include the file axes, not the 5D axes expected by the label input, # so expand them to full 5D axes_5d = opReorderAxes.Output.meta.getAxisKeys() tagged_offsets = collections.OrderedDict( zip(axes_5d, [0] * len(axes_5d))) tagged_offsets.update( dict( zip(opMetadataInjector.Output.meta.getAxisKeys(), settingsDlg.imageOffsets))) imageOffsets = tagged_offsets.values() # Optimization if mapping is identity if labelMapping.keys() == labelMapping.values(): labelMapping = None # This will be fast (it's already cached) label_data = opReorderAxes.Output[:].wait() # Map input labels to output labels if labelMapping: # There are other ways to do a relabeling (e.g skimage.segmentation.relabel_sequential) # But this supports potentially huge values of unique_read_labels (in the billions), # without needing GB of RAM. mapping_indexes = numpy.searchsorted(unique_read_labels, label_data) new_labels = numpy.array( [labelMapping[x] for x in unique_read_labels]) label_data[:] = new_labels[mapping_indexes] label_roi = numpy.array(roiFromShape(opReorderAxes.Output.meta.shape)) label_roi += imageOffsets label_slice = roiToSlice(*label_roi) writeSeeds[label_slice] = label_data finally: opReorderAxes.cleanUp() opMetadataInjector.cleanUp() opCache.cleanUp() opImport.cleanUp()
class OpImageOnDiskProvider(Operator): """ This simply wraps a lazyflow OpInputDataReader, but ensures that the metadata (axistags, drange) on the output matches the metadata from the original data (even if the output file format doesn't support metadata fields). """ TransactionSlot = InputSlot() Input = InputSlot() # Used for dtype and shape only. Data is always provided directly from the file. WorkingDirectory = InputSlot() DatasetPath = InputSlot() # A TOTAL path (possibly including a dataset name, e.g. myfile.h5/volume/data Dirty = InputSlot() Output = OutputSlot() def __init__(self, *args, **kwargs): super( OpImageOnDiskProvider, self ).__init__(*args, **kwargs) self._opReader = None self._opMetadataInjector = None # Block diagram: # # (Input.axistags, Input.drange) # \ # DatasetPath ---> opReader ---> opMetadataInjector --> Output # / # WorkingDirectory def setupOutputs( self ): if self._opReader is not None: self.Output.disconnect() if self._opMetadataInjector: self._opMetadataInjector.cleanUp() self._opMetadataInjector = None self._opReader.cleanUp() self._opReader = None try: # Configure the reader dataReady = True self._opReader = OpInputDataReader( parent=self ) self._opReader.WorkingDirectory.setValue( self.WorkingDirectory.value ) self._opReader.FilePath.setValue( self.DatasetPath.value ) # Since most file formats don't save meta-info, # The reader output's axis order may be incorrect. # (For example, if we export in npy format with zxy order, # the Npy reader op will simply assume xyz order when it reads the data.) # Force the metadata back to the correct state by copying select items from Input.meta metadata = {} metadata['axistags'] = self.Input.meta.axistags metadata['drange'] = self.Input.meta.drange metadata['display_mode'] = self.Input.meta.display_mode self._opMetadataInjector = OpMetadataInjector( parent=self ) self._opMetadataInjector.Input.connect( self._opReader.Output ) self._opMetadataInjector.Metadata.setValue( metadata ) dataReady &= self._opMetadataInjector.Output.meta.shape == self.Input.meta.shape dataReady &= self._opMetadataInjector.Output.meta.dtype == self.Input.meta.dtype if dataReady: self.Output.connect( self._opMetadataInjector.Output ) else: self._opMetadataInjector.cleanUp() self._opMetadataInjector = None self._opReader.cleanUp() self._opReader = None self.Output.meta.NOTREADY = True #except OpInputDataReader.DatasetReadError: except Exception as ex: #logger.debug( "On-disk image can't be read: {}".format(ex) ) # Note: If the data is exported as a 'sequence', then this will always be NOTREADY # because the 'path' (e.g. 'myfile_{slice_index}.png' will be nonexistent. # That's okay because a stack is probably too slow to be of use for a preview anyway. if self._opMetadataInjector: self._opMetadataInjector.cleanUp() self._opMetadataInjector = None self._opReader.cleanUp() self._opReader = None # The dataset doesn't exist yet. self.Output.meta.NOTREADY = True def execute(self, slot, subindex, roi, result): assert False, "Output is supposed to be directly connected to an internal operator." def propagateDirty(self, slot, subindex, roi): if slot == self.Input: self.Output.setDirty( roi ) else: self.Output.setDirty( slice(None) )
def import_labeling_layer(labelLayer, labelingSlots, parent_widget=None): """ Prompt the user for layer import settings, and perform the layer import. :param labelLayer: The top label layer source :param labelingSlots: An instance of LabelingGui.LabelingSlots :param parent_widget: The Qt GUI parent object """ writeSeeds = labelingSlots.labelInput assert isinstance(writeSeeds, lazyflow.graph.Slot), "slot is of type %r" % (type(writeSeeds)) opLabels = writeSeeds.getRealOperator() assert isinstance(opLabels, lazyflow.graph.Operator), "slot's operator is of type %r" % (type(opLabels)) recentlyImported = PreferencesManager().get('labeling', 'recently imported') mostRecentProjectPath = PreferencesManager().get('shell', 'recently opened') mostRecentImageFile = PreferencesManager().get( 'DataSelection', 'recent image' ) if recentlyImported: defaultDirectory = os.path.split(recentlyImported)[0] elif mostRecentProjectPath: defaultDirectory = os.path.split(mostRecentProjectPath)[0] elif mostRecentImageFile: defaultDirectory = os.path.split(mostRecentImageFile)[0] else: defaultDirectory = os.path.expanduser('~') fileNames = DataSelectionGui.getImageFileNamesToOpen(parent_widget, defaultDirectory) fileNames = map(str, fileNames) if not fileNames: return PreferencesManager().set('labeling', 'recently imported', fileNames[0]) try: # Initialize operators opImport = OpInputDataReader( parent=opLabels.parent ) opCache = OpArrayCache( parent=opLabels.parent ) opMetadataInjector = OpMetadataInjector( parent=opLabels.parent ) opReorderAxes = OpReorderAxes( parent=opLabels.parent ) # Set up the pipeline as follows: # # opImport --> opCache --> opMetadataInjector --------> opReorderAxes --(inject via setInSlot)--> labelInput # / / # User-specified axisorder labelInput.meta.axistags opImport.WorkingDirectory.setValue(defaultDirectory) opImport.FilePath.setValue(fileNames[0] if len(fileNames) == 1 else os.path.pathsep.join(fileNames)) assert opImport.Output.ready() opCache.blockShape.setValue( opImport.Output.meta.shape ) opCache.Input.connect( opImport.Output ) assert opCache.Output.ready() opMetadataInjector.Input.connect( opCache.Output ) metadata = opCache.Output.meta.copy() opMetadataInjector.Metadata.setValue( metadata ) opReorderAxes.Input.connect( opMetadataInjector.Output ) # Transpose the axes for assignment to the labeling operator. opReorderAxes.AxisOrder.setValue( writeSeeds.meta.getAxisKeys() ) # We'll show a little window with a busy indicator while the data is loading busy_dlg = QProgressDialog(parent=parent_widget) busy_dlg.setLabelText("Importing Label Data...") busy_dlg.setCancelButton(None) busy_dlg.setMinimum(100) busy_dlg.setMaximum(100) def close_busy_dlg(*args): QApplication.postEvent(busy_dlg, QCloseEvent()) # Load the data from file into our cache # When it's done loading, close the progress dialog. req = opCache.Output[:] req.notify_finished( close_busy_dlg ) req.notify_failed( close_busy_dlg ) req.submit() busy_dlg.exec_() readData = req.result maxLabels = len(labelingSlots.labelNames.value) # Can't use return_counts feature because that requires numpy >= 1.9 #unique_read_labels, readLabelCounts = numpy.unique(readData, return_counts=True) # This does the same as the above, albeit slower, and probably with more ram. unique_read_labels = numpy.unique(readData) readLabelCounts = vigra_bincount(readData)[unique_read_labels] labelInfo = (maxLabels, (unique_read_labels, readLabelCounts)) del readData # Ask the user how to interpret the data. settingsDlg = LabelImportOptionsDlg( parent_widget, fileNames, opMetadataInjector.Output, labelingSlots.labelInput, labelInfo ) def handle_updated_axes(): # The user is specifying a new interpretation of the file's axes updated_axisorder = str(settingsDlg.axesEdit.text()) metadata = opMetadataInjector.Metadata.value.copy() metadata.axistags = vigra.defaultAxistags(updated_axisorder) opMetadataInjector.Metadata.setValue( metadata ) if opReorderAxes._invalid_axes: settingsDlg.buttonBox.button(QDialogButtonBox.Ok).setEnabled(False) # Red background settingsDlg.axesEdit.setStyleSheet("QLineEdit { background: rgb(255, 128, 128);" "selection-background-color: rgb(128, 128, 255); }") settingsDlg.axesEdit.editingFinished.connect( handle_updated_axes ) # Initialize handle_updated_axes() dlg_result = settingsDlg.exec_() if dlg_result != LabelImportOptionsDlg.Accepted: return # Get user's chosen label mapping from dlg labelMapping = settingsDlg.labelMapping # Get user's chosen offsets. # Offsets in dlg only include the file axes, not the 5D axes expected by the label input, # so expand them to full 5D axes_5d = opReorderAxes.Output.meta.getAxisKeys() tagged_offsets = collections.OrderedDict( zip( axes_5d, [0]*len(axes_5d) ) ) tagged_offsets.update( dict( zip( opMetadataInjector.Output.meta.getAxisKeys(), settingsDlg.imageOffsets ) ) ) imageOffsets = tagged_offsets.values() # Optimization if mapping is identity if labelMapping.keys() == labelMapping.values(): labelMapping = None # This will be fast (it's already cached) label_data = opReorderAxes.Output[:].wait() # Map input labels to output labels if labelMapping: # There are other ways to do a relabeling (e.g skimage.segmentation.relabel_sequential) # But this supports potentially huge values of unique_read_labels (in the billions), # without needing GB of RAM. mapping_indexes = numpy.searchsorted(unique_read_labels, label_data) new_labels = numpy.array([labelMapping[x] for x in unique_read_labels]) label_data[:] = new_labels[mapping_indexes] label_roi = numpy.array( roiFromShape(opReorderAxes.Output.meta.shape) ) label_roi += imageOffsets label_slice = roiToSlice(*label_roi) writeSeeds[label_slice] = label_data finally: opReorderAxes.cleanUp() opMetadataInjector.cleanUp() opCache.cleanUp() opImport.cleanUp()
def import_labeling_layer(labelLayer, labelingSlots, parent_widget=None): """ Prompt the user for layer import settings, and perform the layer import. :param labelLayer: The top label layer source :param labelingSlots: An instance of LabelingGui.LabelingSlots :param parent_widget: The Qt GUI parent object """ writeSeeds = labelingSlots.labelInput assert isinstance( writeSeeds, lazyflow.graph.Slot), "slot is of type %r" % (type(writeSeeds)) opLabels = writeSeeds.getRealOperator() assert isinstance(opLabels, lazyflow.graph.Operator ), "slot's operator is of type %r" % (type(opLabels)) recentlyImported = PreferencesManager().get('labeling', 'recently imported') mostRecentProjectPath = PreferencesManager().get('shell', 'recently opened') mostRecentImageFile = PreferencesManager().get('DataSelection', 'recent image') if recentlyImported: defaultDirectory = os.path.split(recentlyImported)[0] elif mostRecentProjectPath: defaultDirectory = os.path.split(mostRecentProjectPath)[0] elif mostRecentImageFile: defaultDirectory = os.path.split(mostRecentImageFile)[0] else: defaultDirectory = os.path.expanduser('~') fileNames = DataSelectionGui.getImageFileNamesToOpen( parent_widget, defaultDirectory) fileNames = map(str, fileNames) if not fileNames: return PreferencesManager().set('labeling', 'recently imported', fileNames[0]) try: # Initialize operators opImport = OpInputDataReader(parent=opLabels.parent) opCache = OpBlockedArrayCache(parent=opLabels.parent) opMetadataInjector = OpMetadataInjector(parent=opLabels.parent) opReorderAxes = OpReorderAxes(parent=opLabels.parent) # Set up the pipeline as follows: # # opImport --> (opCache) --> opMetadataInjector --------> opReorderAxes --(inject via setInSlot)--> labelInput # / / # User-specified axisorder labelInput.meta.axistags opImport.WorkingDirectory.setValue(defaultDirectory) opImport.FilePath.setValue(fileNames[0] if len(fileNames) == 1 else os.path.pathsep.join(fileNames)) assert opImport.Output.ready() maxLabels = len(labelingSlots.labelNames.value) # We don't bother with counting the label pixels # (and caching the data) if it's big (1 GB) if numpy.prod(opImport.Output.meta.shape) > 1e9: reading_slot = opImport.Output # For huge data, we don't go through and search for the pixel values, # because that takes an annoyingly long amount of time. # Instead, we make the reasonable assumption that the input labels are already 1,2,3..N # and we don't tell the user what the label pixel counts are. unique_read_labels = numpy.array(range(maxLabels + 1)) readLabelCounts = numpy.array([-1] * (maxLabels + 1)) labelInfo = (maxLabels, (unique_read_labels, readLabelCounts)) else: opCache.Input.connect(opImport.Output) opCache.CompressionEnabled.setValue(True) assert opCache.Output.ready() reading_slot = opCache.Output # We'll show a little window with a busy indicator while the data is loading busy_dlg = QProgressDialog(parent=parent_widget) busy_dlg.setLabelText("Scanning Label Data...") busy_dlg.setCancelButton(None) busy_dlg.setMinimum(100) busy_dlg.setMaximum(100) def close_busy_dlg(*args): QApplication.postEvent(busy_dlg, QCloseEvent()) # Load the data from file into our cache # When it's done loading, close the progress dialog. req = reading_slot[:] req.notify_finished(close_busy_dlg) req.notify_failed(close_busy_dlg) req.submit() busy_dlg.exec_() readData = req.result # Can't use return_counts feature because that requires numpy >= 1.9 #unique_read_labels, readLabelCounts = numpy.unique(readData, return_counts=True) # This does the same as the above, albeit slower, and probably with more ram. bincounts = chunked_bincount(readData) unique_read_labels = bincounts.nonzero()[0].astype(readData.dtype, copy=False) readLabelCounts = bincounts[unique_read_labels] labelInfo = (maxLabels, (unique_read_labels, readLabelCounts)) del readData opMetadataInjector.Input.connect(reading_slot) metadata = reading_slot.meta.copy() opMetadataInjector.Metadata.setValue(metadata) opReorderAxes.Input.connect(opMetadataInjector.Output) # Transpose the axes for assignment to the labeling operator. opReorderAxes.AxisOrder.setValue(writeSeeds.meta.getAxisKeys()) # Ask the user how to interpret the data. settingsDlg = LabelImportOptionsDlg(parent_widget, fileNames, opMetadataInjector.Output, labelingSlots.labelInput, labelInfo) def handle_updated_axes(): # The user is specifying a new interpretation of the file's axes updated_axisorder = str(settingsDlg.axesEdit.text()) metadata = opMetadataInjector.Metadata.value.copy() metadata.axistags = vigra.defaultAxistags(updated_axisorder) opMetadataInjector.Metadata.setValue(metadata) if opReorderAxes._invalid_axes: settingsDlg.buttonBox.button( QDialogButtonBox.Ok).setEnabled(False) # Red background settingsDlg.axesEdit.setStyleSheet( "QLineEdit { background: rgb(255, 128, 128);" "selection-background-color: rgb(128, 128, 255); }") settingsDlg.axesEdit.editingFinished.connect(handle_updated_axes) # Initialize handle_updated_axes() dlg_result = settingsDlg.exec_() if dlg_result != LabelImportOptionsDlg.Accepted: return # Get user's chosen label mapping from dlg labelMapping = settingsDlg.labelMapping # Get user's chosen offsets, ordered by the 'write seeds' slot axes_5d = opReorderAxes.Output.meta.getAxisKeys() tagged_offsets = collections.OrderedDict( zip(axes_5d, [0] * len(axes_5d))) tagged_offsets.update( dict( zip(opReorderAxes.Output.meta.getAxisKeys(), settingsDlg.imageOffsets))) imageOffsets = tagged_offsets.values() # Optimization if mapping is identity if labelMapping.keys() == labelMapping.values(): labelMapping = None # If the data was already cached, this will be fast. label_data = opReorderAxes.Output[:].wait() # Map input labels to output labels if labelMapping: # There are other ways to do a relabeling (e.g skimage.segmentation.relabel_sequential) # But this supports potentially huge values of unique_read_labels (in the billions), # without needing GB of RAM. mapping_indexes = numpy.searchsorted(unique_read_labels, label_data) new_labels = numpy.array( [labelMapping[x] for x in unique_read_labels]) label_data[:] = new_labels[mapping_indexes] label_roi = numpy.array(roiFromShape(opReorderAxes.Output.meta.shape)) label_roi += imageOffsets label_slice = roiToSlice(*label_roi) writeSeeds[label_slice] = label_data finally: opReorderAxes.cleanUp() opMetadataInjector.cleanUp() opCache.cleanUp() opImport.cleanUp()