def convert_predictions_to_uncertainties( input_path, parsed_export_args ): """ Read exported pixel predictions and calculate/export the uncertainties. input_path: The path to the prediction output file. If hdf5, must include the internal dataset name. parsed_export_args: The already-parsed cmd-line arguments generated from a DataExportApplet-compatible ArgumentParser. """ graph = Graph() opReader = OpInputDataReader(graph=graph) opReader.WorkingDirectory.setValue( os.getcwd() ) opReader.FilePath.setValue(input_path) opUncertainty = OpEnsembleMargin( graph=graph ) opUncertainty.Input.connect( opReader.Output ) opExport = OpFormattedDataExport( graph=graph ) opExport.Input.connect( opUncertainty.Output ) # Apply command-line arguments. DataExportApplet._configure_operator_with_parsed_args(parsed_export_args, opExport) last_progress = [-1] def print_progress(progress_percent): if progress_percent != last_progress[0]: last_progress[0] = progress_percent sys.stdout.write( " {}".format(progress_percent) ) print "Exporting results to : {}".format( opExport.ExportPath.value ) sys.stdout.write("Progress:") opExport.progressSignal.subscribe(print_progress) # Begin export opExport.run_export() sys.stdout.write("\n") print "DONE."
def testBasic(self): graph = Graph() opExport = OpFormattedDataExport(graph=graph) data = numpy.random.random((100, 100)).astype(numpy.float32) * 100 data = vigra.taggedView(data, vigra.defaultAxistags('xy')) opExport.Input.setValue(data) sub_roi = [(10, 0), (None, 80)] opExport.RegionStart.setValue(sub_roi[0]) opExport.RegionStop.setValue(sub_roi[1]) opExport.ExportDtype.setValue(numpy.uint8) opExport.InputMin.setValue(0.0) opExport.InputMax.setValue(100.0) opExport.ExportMin.setValue(100) opExport.ExportMax.setValue(200) opExport.OutputFormat.setValue('hdf5') opExport.OutputFilenameFormat.setValue( self._tmpdir + '/export_x{x_start}-{x_stop}_y{y_start}-{y_stop}') opExport.OutputInternalPath.setValue('volume/data') opExport.TransactionSlot.setValue(True) assert opExport.ImageToExport.ready() assert opExport.ExportPath.ready() assert opExport.ImageToExport.meta.drange == (100, 200) #print "exporting data to: {}".format( opExport.ExportPath.value ) assert opExport.ExportPath.value == self._tmpdir + '/' + 'export_x10-100_y0-80.h5/volume/data' opExport.run_export() opRead = OpInputDataReader(graph=graph) try: opRead.FilePath.setValue(opExport.ExportPath.value) # Compare with the correct subregion and convert dtype. sub_roi[1] = (100, 80) # Replace 'None' with full extent expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)] expected_data = expected_data.astype(numpy.uint8) expected_data += 100 # see renormalization settings assert opRead.Output.meta.shape == expected_data.shape assert opRead.Output.meta.dtype == expected_data.dtype read_data = opRead.Output[:].wait() # Due to rounding errors, the actual result and the expected result may differ by 1 # e.g. if the original pixel value was 32.99999999 # Also, must promote to signed values to avoid unsigned rollover # See issue ( https://github.com/ilastik/lazyflow/issues/165 ). expected_data_signed = expected_data.astype(numpy.int16) read_data_signed = expected_data.astype(numpy.int16) difference_from_expected = expected_data_signed - read_data_signed assert (numpy.abs(difference_from_expected) <= 1).all(), "Read data didn't match exported data!" finally: opRead.cleanUp()
def testBasic(self): graph = Graph() opExport = OpFormattedDataExport(graph=graph) data = numpy.random.random( (100,100) ).astype( numpy.float32 ) * 100 data = vigra.taggedView( data, vigra.defaultAxistags('xy') ) opExport.Input.setValue(data) sub_roi = [(10, 0), (None, 80)] opExport.RegionStart.setValue( sub_roi[0] ) opExport.RegionStop.setValue( sub_roi[1] ) opExport.ExportDtype.setValue( numpy.uint8 ) opExport.InputMin.setValue( 0.0 ) opExport.InputMax.setValue( 100.0 ) opExport.ExportMin.setValue( 100 ) opExport.ExportMax.setValue( 200 ) opExport.OutputFormat.setValue( 'hdf5' ) opExport.OutputFilenameFormat.setValue( self._tmpdir + '/export_x{x_start}-{x_stop}_y{y_start}-{y_stop}' ) opExport.OutputInternalPath.setValue('volume/data') opExport.TransactionSlot.setValue( True ) assert opExport.ImageToExport.ready() assert opExport.ExportPath.ready() assert opExport.ImageToExport.meta.drange == (100,200) #print "exporting data to: {}".format( opExport.ExportPath.value ) assert opExport.ExportPath.value == self._tmpdir + '/' + 'export_x10-100_y0-80.h5/volume/data' opExport.run_export() opRead = OpInputDataReader( graph=graph ) try: opRead.FilePath.setValue( opExport.ExportPath.value ) # Compare with the correct subregion and convert dtype. sub_roi[1] = (100, 80) # Replace 'None' with full extent expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)] expected_data = expected_data.astype(numpy.uint8) expected_data += 100 # see renormalization settings assert opRead.Output.meta.shape == expected_data.shape assert opRead.Output.meta.dtype == expected_data.dtype read_data = opRead.Output[:].wait() # Due to rounding errors, the actual result and the expected result may differ by 1 # e.g. if the original pixel value was 32.99999999 # Also, must promote to signed values to avoid unsigned rollover # See issue ( https://github.com/ilastik/lazyflow/issues/165 ). expected_data_signed = expected_data.astype(numpy.int16) read_data_signed = expected_data.astype(numpy.int16) difference_from_expected = expected_data_signed - read_data_signed assert (numpy.abs(difference_from_expected) <= 1).all(), "Read data didn't match exported data!" finally: opRead.cleanUp()
def __init__(self, *args, **kwargs): super(OpDataExport, self).__init__(*args, **kwargs) self._opFormattedExport = OpFormattedDataExport(parent=self) opFormattedExport = self._opFormattedExport # Forward almost all inputs to the 'real' exporter opFormattedExport.TransactionSlot.connect(self.TransactionSlot) opFormattedExport.Input.connect(self.Input) opFormattedExport.RegionStart.connect(self.RegionStart) opFormattedExport.RegionStop.connect(self.RegionStop) opFormattedExport.InputMin.connect(self.InputMin) opFormattedExport.InputMax.connect(self.InputMax) opFormattedExport.ExportMin.connect(self.ExportMin) opFormattedExport.ExportMax.connect(self.ExportMax) opFormattedExport.ExportDtype.connect(self.ExportDtype) opFormattedExport.OutputAxisOrder.connect(self.OutputAxisOrder) opFormattedExport.OutputFormat.connect(self.OutputFormat) self.ConvertedImage.connect(opFormattedExport.ConvertedImage) self.ImageToExport.connect(opFormattedExport.ImageToExport) self.ExportPath.connect(opFormattedExport.ExportPath) self.FormatSelectionIsValid.connect( opFormattedExport.FormatSelectionIsValid) self.progressSignal = opFormattedExport.progressSignal self.Dirty.setValue(True) # Default to Dirty self._opImageOnDiskProvider = None # We don't export the raw data, but we connect it to it's own op # so it can be displayed alongside the data to export in the same viewer. # This keeps axis order, shape, etc. in sync with the displayed export data. # Note that we must not modify the channels of the raw data, so it gets passed throught a helper. opHelper = OpRawSubRegionHelper(parent=self) opHelper.RawImage.connect(self.RawData) opHelper.ExportStop.connect(self.RegionStart) opHelper.ExportStop.connect(self.RegionStop) opFormatRaw = OpFormattedDataExport(parent=self) opFormatRaw.TransactionSlot.connect(self.TransactionSlot) opFormatRaw.Input.connect(self.RawData) opFormatRaw.RegionStart.connect(opHelper.RawStart) opFormatRaw.RegionStop.connect(opHelper.RawStop) # Don't normalize the raw data. #opFormatRaw.InputMin.connect( self.InputMin ) #opFormatRaw.InputMax.connect( self.InputMax ) #opFormatRaw.ExportMin.connect( self.ExportMin ) #opFormatRaw.ExportMax.connect( self.ExportMax ) #opFormatRaw.ExportDtype.connect( self.ExportDtype ) opFormatRaw.OutputAxisOrder.connect(self.OutputAxisOrder) opFormatRaw.OutputFormat.connect(self.OutputFormat) self._opFormatRaw = opFormatRaw self.FormattedRawData.connect(opFormatRaw.ImageToExport)
def get_settings_and_export_layer(layer, parent_widget=None): """ Prompt the user for layer export settings, and perform the layer export. """ sourceTags = [True for l in layer.datasources] for i, source in enumerate(layer.datasources): if not hasattr(source, "dataSlot"): sourceTags[i] = False if not any(sourceTags): raise RuntimeError( "can not export from a non-lazyflow data source (layer=%r, datasource=%r)" % (type(layer), type(layer.datasources[0]))) if not _has_lazyflow: raise RuntimeError("lazyflow not installed") import lazyflow dataSlots = [ slot.dataSlot for (slot, isSlot) in zip(layer.datasources, sourceTags) if isSlot is True ] opStackChannels = lazyflow.operators.OpMultiArrayStacker( dataSlots[0].getRealOperator().parent) for slot in dataSlots: assert isinstance( slot, lazyflow.graph.Slot), "slot is of type %r" % (type(slot)) assert isinstance( slot.getRealOperator(), lazyflow.graph.Operator), "slot's operator is of type %r" % (type( slot.getRealOperator())) opStackChannels.AxisFlag.setValue("c") opStackChannels.Images.resize(len(dataSlots)) for i, islot in enumerate(opStackChannels.Images): islot.connect(dataSlots[i]) # Create an operator to do the work from lazyflow.operators.ioOperators import OpFormattedDataExport opExport = OpFormattedDataExport(parent=opStackChannels.parent) opExport.OutputFilenameFormat.setValue(str(layer.name)) opExport.Input.connect(opStackChannels.Output) opExport.TransactionSlot.setValue(True) # Use this dialog to populate the operator's slot settings settingsDlg = DataExportOptionsDlg(parent_widget, opExport) # If user didn't cancel, run the export now. if (settingsDlg.exec_() == DataExportOptionsDlg.Accepted): helper = ExportHelper(parent_widget) helper.run(opExport) # Clean up our temporary operators opExport.cleanUp() opStackChannels.cleanUp()
def convert_predictions_to_segmentation(input_paths, parsed_export_args): """ Read exported pixel predictions and calculate/export the segmentation. input_path: The path to the prediction output file. If hdf5, must include the internal dataset name. parsed_export_args: The already-parsed cmd-line arguments generated from a DataExportApplet-compatible ArgumentParser. """ graph = Graph() opReader = OpInputDataReader(graph=graph) opReader.WorkingDirectory.setValue(os.getcwd()) opArgmaxChannel = OpArgmaxChannel(graph=graph) opArgmaxChannel.Input.connect(opReader.Output) opExport = OpFormattedDataExport(graph=graph) opExport.Input.connect(opArgmaxChannel.Output) # Apply command-line arguments. DataExportApplet._configure_operator_with_parsed_args( parsed_export_args, opExport) last_progress = [-1] def print_progress(progress_percent): if progress_percent != last_progress[0]: last_progress[0] = progress_percent sys.stdout.write(" {}".format(progress_percent)) opExport.progressSignal.subscribe(print_progress) for input_path in input_paths: opReader.FilePath.setValue(input_path) input_pathcomp = PathComponents(input_path) opExport.OutputFilenameFormat.setValue(str( input_pathcomp.externalPath)) output_path = opExport.ExportPath.value output_pathcomp = PathComponents(output_path) output_pathcomp.filenameBase += "_Segmentation" opExport.OutputFilenameFormat.setValue( str(output_pathcomp.externalPath)) print("Exporting results to : {}".format(opExport.ExportPath.value)) sys.stdout.write("Progress:") # Begin export opExport.run_export() sys.stdout.write("\n") print("DONE.")
def get_model_op(wrappedOp): """ Create a "model operator" that the gui can use. The model op is a single (non-wrapped) export operator that the gui will manipulate while the user plays around with the export settings. When the user is finished, the model op slot settings can be copied over to the 'real' (wrapped) operator slots. """ if len(wrappedOp) == 0: return None, None # These are the slots the export settings gui will manipulate. setting_slots = [ wrappedOp.RegionStart, wrappedOp.RegionStop, wrappedOp.InputMin, wrappedOp.InputMax, wrappedOp.ExportMin, wrappedOp.ExportMax, wrappedOp.ExportDtype, wrappedOp.OutputAxisOrder, wrappedOp.OutputFilenameFormat, wrappedOp.OutputInternalPath, wrappedOp.OutputFormat ] # Use an instance of OpFormattedDataExport, since has the important slots and no others. model_op = OpFormattedDataExport(parent=wrappedOp.parent) for slot in setting_slots: model_inslot = getattr(model_op, slot.name) if slot.ready(): model_inslot.setValue(slot.value) # Choose a roi that can apply to all images in the original operator shape = None axes = None for multislot in wrappedOp.Inputs: slot = multislot[wrappedOp.InputSelection.value] if slot.ready(): if shape is None: shape = slot.meta.shape axes = slot.meta.getAxisKeys() dtype = slot.meta.dtype else: assert slot.meta.getAxisKeys( ) == axes, "Can't export multiple slots with different axes." assert slot.meta.dtype == dtype shape = numpy.minimum(slot.meta.shape, shape) # If NO slots were ready, then we can't do anything here. if shape is None: return None, None # Must provide a 'ready' slot for the gui # Use a subregion operator to provide a slot with the meta data we chose. opSubRegion = OpSubRegion(parent=wrappedOp.parent) opSubRegion.Roi.setValue([(0, ) * len(shape), tuple(shape)]) opSubRegion.Input.connect(slot) # (The actual contents of this slot are not important to the settings gui. # It only cares about the metadata.) model_op.Input.connect(opSubRegion.Output) return model_op, opSubRegion # We return the subregion op, too, so the caller can clean it up.
def get_settings_and_export_layer(layer, parent_widget=None): """ Prompt the user for layer export settings, and perform the layer export. """ sourceTags = [True for l in layer.datasources] for i, source in enumerate(layer.datasources): if not hasattr(source, "dataSlot"): sourceTags[i] = False if not any(sourceTags): raise RuntimeError("can not export from a non-lazyflow data source (layer=%r, datasource=%r)" % (type(layer), type(layer.datasources[0])) ) if not _has_lazyflow: raise RuntimeError("lazyflow not installed") import lazyflow dataSlots = [slot.dataSlot for (slot, isSlot) in zip(layer.datasources, sourceTags) if isSlot is True] opStackChannels = lazyflow.operators.OpMultiArrayStacker(dataSlots[0].getRealOperator().parent) for slot in dataSlots: assert isinstance(slot, lazyflow.graph.Slot), "slot is of type %r" % (type(slot)) assert isinstance(slot.getRealOperator(), lazyflow.graph.Operator), "slot's operator is of type %r" % (type(slot.getRealOperator())) opStackChannels.AxisFlag.setValue("c") opStackChannels.Images.resize(len(dataSlots)) for i,islot in enumerate(opStackChannels.Images): islot.connect(dataSlots[i]) # Create an operator to do the work from lazyflow.operators.ioOperators import OpFormattedDataExport opExport = OpFormattedDataExport( parent=opStackChannels.parent ) opExport.OutputFilenameFormat.setValue( encode_from_qstring(layer.name) ) opExport.Input.connect( opStackChannels.Output ) opExport.TransactionSlot.setValue(True) # Use this dialog to populate the operator's slot settings settingsDlg = DataExportOptionsDlg( parent_widget, opExport ) # If user didn't cancel, run the export now. if ( settingsDlg.exec_() == DataExportOptionsDlg.Accepted ): helper = ExportHelper( parent_widget ) helper.run(opExport) # Clean up our temporary operators opExport.cleanUp() opStackChannels.cleanUp()
def convert_predictions_to_segmentation( input_paths, parsed_export_args ): """ Read exported pixel predictions and calculate/export the segmentation. input_path: The path to the prediction output file. If hdf5, must include the internal dataset name. parsed_export_args: The already-parsed cmd-line arguments generated from a DataExportApplet-compatible ArgumentParser. """ graph = Graph() opReader = OpInputDataReader(graph=graph) opReader.WorkingDirectory.setValue( os.getcwd() ) opArgmaxChannel = OpArgmaxChannel( graph=graph ) opArgmaxChannel.Input.connect( opReader.Output ) opExport = OpFormattedDataExport( graph=graph ) opExport.Input.connect( opArgmaxChannel.Output ) # Apply command-line arguments. DataExportApplet._configure_operator_with_parsed_args(parsed_export_args, opExport) last_progress = [-1] def print_progress(progress_percent): if progress_percent != last_progress[0]: last_progress[0] = progress_percent sys.stdout.write( " {}".format(progress_percent) ) opExport.progressSignal.subscribe(print_progress) for input_path in input_paths: opReader.FilePath.setValue(input_path) input_pathcomp = PathComponents(input_path) opExport.OutputFilenameFormat.setValue(str(input_pathcomp.externalPath)) output_path = opExport.ExportPath.value output_pathcomp = PathComponents( output_path ) output_pathcomp.filenameBase += "_Segmentation" opExport.OutputFilenameFormat.setValue(str(output_pathcomp.externalPath)) print "Exporting results to : {}".format( opExport.ExportPath.value ) sys.stdout.write("Progress:") # Begin export opExport.run_export() sys.stdout.write("\n") print "DONE."
def testBasic(self): graph = Graph() opExport = OpFormattedDataExport(graph=graph) data = numpy.random.random( (100,100) ).astype( numpy.float32 ) * 100 data = vigra.taggedView( data, vigra.defaultAxistags('xy') ) opExport.Input.setValue(data) sub_roi = [(10, 0), (None, 80)] opExport.RegionStart.setValue( sub_roi[0] ) opExport.RegionStop.setValue( sub_roi[1] ) opExport.ExportDtype.setValue( numpy.uint8 ) opExport.InputMin.setValue( 0.0 ) opExport.InputMax.setValue( 100.0 ) opExport.ExportMin.setValue( 100 ) opExport.ExportMax.setValue( 200 ) opExport.OutputFormat.setValue( 'hdf5' ) opExport.OutputFilenameFormat.setValue( self._tmpdir + '/export_x{x_start}-{x_stop}_y{y_start}-{y_stop}' ) opExport.OutputInternalPath.setValue('volume/data') opExport.TransactionSlot.setValue( True ) assert opExport.ImageToExport.ready() assert opExport.ExportPath.ready() assert opExport.ImageToExport.meta.drange == (100,200) #print "exporting data to: {}".format( opExport.ExportPath.value ) assert opExport.ExportPath.value == self._tmpdir + '/' + 'export_x10-100_y0-80.h5/volume/data' opExport.run_export() opRead = OpInputDataReader( graph=graph ) opRead.FilePath.setValue( opExport.ExportPath.value ) # Compare with the correct subregion and convert dtype. sub_roi[1] = (100, 80) # Replace 'None' with full extent expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)] expected_data = expected_data.astype(numpy.uint8) expected_data += 100 # see renormalization settings read_data = opRead.Output[:].wait() assert (read_data == expected_data).all(), "Read data didn't match exported data!"
def __init__(self, *args, **kwargs): super( OpDataExport, self ).__init__(*args, **kwargs) self._opFormattedExport = OpFormattedDataExport( parent=self ) opFormattedExport = self._opFormattedExport # Forward almost all inputs to the 'real' exporter opFormattedExport.TransactionSlot.connect( self.TransactionSlot ) opFormattedExport.Input.connect( self.Input ) opFormattedExport.RegionStart.connect( self.RegionStart ) opFormattedExport.RegionStop.connect( self.RegionStop ) opFormattedExport.InputMin.connect( self.InputMin ) opFormattedExport.InputMax.connect( self.InputMax ) opFormattedExport.ExportMin.connect( self.ExportMin ) opFormattedExport.ExportMax.connect( self.ExportMax ) opFormattedExport.ExportDtype.connect( self.ExportDtype ) opFormattedExport.OutputAxisOrder.connect( self.OutputAxisOrder ) opFormattedExport.OutputFormat.connect( self.OutputFormat ) self.ConvertedImage.connect( opFormattedExport.ConvertedImage ) self.ImageToExport.connect( opFormattedExport.ImageToExport ) self.ExportPath.connect( opFormattedExport.ExportPath ) self.FormatSelectionIsValid.connect( opFormattedExport.FormatSelectionIsValid ) self.progressSignal = opFormattedExport.progressSignal self.Dirty.setValue(True) # Default to Dirty self._opImageOnDiskProvider = None # We don't export the raw data, but we connect it to it's own op # so it can be displayed alongside the data to export in the same viewer. # This keeps axis order, shape, etc. in sync with the displayed export data. # Note that we must not modify the channels of the raw data, so it gets passed through a helper. opHelper = OpRawSubRegionHelper( parent=self ) opHelper.RawImage.connect( self.RawData ) opHelper.ExportStart.connect( self.RegionStart ) opHelper.ExportStop.connect( self.RegionStop ) opFormatRaw = OpFormattedDataExport( parent=self ) opFormatRaw.TransactionSlot.connect( self.TransactionSlot ) opFormatRaw.Input.connect( self.RawData ) opFormatRaw.RegionStart.connect( opHelper.RawStart ) opFormatRaw.RegionStop.connect( opHelper.RawStop ) # Don't normalize the raw data. #opFormatRaw.InputMin.connect( self.InputMin ) #opFormatRaw.InputMax.connect( self.InputMax ) #opFormatRaw.ExportMin.connect( self.ExportMin ) #opFormatRaw.ExportMax.connect( self.ExportMax ) #opFormatRaw.ExportDtype.connect( self.ExportDtype ) opFormatRaw.OutputAxisOrder.connect( self.OutputAxisOrder ) opFormatRaw.OutputFormat.connect( self.OutputFormat ) self._opFormatRaw = opFormatRaw self.FormattedRawData.connect( opFormatRaw.ImageToExport )
def get_export_operator(layer: Layer) -> OpFormattedDataExport: """Get export operator configured with stacked output from layer Args: layer: layer containing datasources that can be exported (datasource.dataSlot -> Slot) Returns: OpFormattedDataExport configured with all stackable input slots from layer. """ opStackChannels = _get_stacked_data_sources(layer) # Create an operator to do the work opExport = OpFormattedDataExport(parent=opStackChannels.parent) opExport.Input.connect(opStackChannels.Output) opExport.TransactionSlot.setValue(True) return opExport
class OpDataExport(Operator): """ Top-level operator for the export applet. Mostly a simple wrapper for OpProcessedDataExport, but with extra formatting of the export path based on lane attributes. """ TransactionSlot = InputSlot() # To apply all settings in one 'transaction', # disconnect this slot and reconnect it when all slots are ready RawData = InputSlot(optional=True) # Display only FormattedRawData = OutputSlot() # OUTPUT - for overlaying the transformed raw data with the export data. # The dataset info for the original dataset (raw data) RawDatasetInfo = InputSlot() WorkingDirectory = InputSlot() # Non-absolute paths are relative to this directory. If not provided, paths must be absolute. Input = InputSlot() # The results slot we want to export # Subregion params RegionStart = InputSlot(optional=True) RegionStop = InputSlot(optional=True) # Normalization params InputMin = InputSlot(optional=True) InputMax = InputSlot(optional=True) ExportMin = InputSlot(optional=True) ExportMax = InputSlot(optional=True) ExportDtype = InputSlot(optional=True) OutputAxisOrder = InputSlot(optional=True) # File settings OutputFilenameFormat = InputSlot(value='{dataset_dir}/{nickname}_export') # A format string allowing {dataset_dir} {nickname}, {roi}, {x_start}, {x_stop}, etc. OutputInternalPath = InputSlot(value='exported_data') OutputFormat = InputSlot(value='hdf5') ExportPath = OutputSlot() # Location of the saved file after export is complete. ConvertedImage = OutputSlot() # Cropped image, not yet re-ordered (useful for guis) ImageToExport = OutputSlot() # The image that will be exported ImageOnDisk = OutputSlot() # This slot reads the exported image from disk (after the export is complete) Dirty = OutputSlot() # Whether or not the result currently matches what's on disk FormatSelectionIsValid = OutputSlot() ALL_FORMATS = OpFormattedDataExport.ALL_FORMATS #### # Simplified block diagram for actual export data and 'live preview' display: # # --> ExportPath # / # Input -> opFormattedExport --> ImageToExport (live preview) # |\ # \ --> ConvertedImage # \ # --> FormatSeletionIsValid #### # Simplified block diagram for Raw data display: # # RegionStart -- # \ # RegionStop --> OpRawSubRegionHelper.RawStart - # / .RawStop --\ # / \ # RawData ---->---------------------------------> opFormatRaw --> FormattedRawData #### # Simplified block diagram for "on-disk view" of the exported results file: # # opFormattedExport.ImageToExport (for metadata only) --> # \ # opFormattedExport.ExportPath -------------------------> opImageOnDiskProvider --> ImageOnDisk def __init__(self, *args, **kwargs): super( OpDataExport, self ).__init__(*args, **kwargs) self._opFormattedExport = OpFormattedDataExport( parent=self ) opFormattedExport = self._opFormattedExport # Forward almost all inputs to the 'real' exporter opFormattedExport.TransactionSlot.connect( self.TransactionSlot ) opFormattedExport.Input.connect( self.Input ) opFormattedExport.RegionStart.connect( self.RegionStart ) opFormattedExport.RegionStop.connect( self.RegionStop ) opFormattedExport.InputMin.connect( self.InputMin ) opFormattedExport.InputMax.connect( self.InputMax ) opFormattedExport.ExportMin.connect( self.ExportMin ) opFormattedExport.ExportMax.connect( self.ExportMax ) opFormattedExport.ExportDtype.connect( self.ExportDtype ) opFormattedExport.OutputAxisOrder.connect( self.OutputAxisOrder ) opFormattedExport.OutputFormat.connect( self.OutputFormat ) self.ConvertedImage.connect( opFormattedExport.ConvertedImage ) self.ImageToExport.connect( opFormattedExport.ImageToExport ) self.ExportPath.connect( opFormattedExport.ExportPath ) self.FormatSelectionIsValid.connect( opFormattedExport.FormatSelectionIsValid ) self.progressSignal = opFormattedExport.progressSignal self.Dirty.setValue(True) # Default to Dirty self._opImageOnDiskProvider = None # We don't export the raw data, but we connect it to it's own op # so it can be displayed alongside the data to export in the same viewer. # This keeps axis order, shape, etc. in sync with the displayed export data. # Note that we must not modify the channels of the raw data, so it gets passed through a helper. opHelper = OpRawSubRegionHelper( parent=self ) opHelper.RawImage.connect( self.RawData ) opHelper.ExportStart.connect( self.RegionStart ) opHelper.ExportStop.connect( self.RegionStop ) opFormatRaw = OpFormattedDataExport( parent=self ) opFormatRaw.TransactionSlot.connect( self.TransactionSlot ) opFormatRaw.Input.connect( self.RawData ) opFormatRaw.RegionStart.connect( opHelper.RawStart ) opFormatRaw.RegionStop.connect( opHelper.RawStop ) # Don't normalize the raw data. #opFormatRaw.InputMin.connect( self.InputMin ) #opFormatRaw.InputMax.connect( self.InputMax ) #opFormatRaw.ExportMin.connect( self.ExportMin ) #opFormatRaw.ExportMax.connect( self.ExportMax ) #opFormatRaw.ExportDtype.connect( self.ExportDtype ) opFormatRaw.OutputAxisOrder.connect( self.OutputAxisOrder ) opFormatRaw.OutputFormat.connect( self.OutputFormat ) self._opFormatRaw = opFormatRaw self.FormattedRawData.connect( opFormatRaw.ImageToExport ) def cleanupOnDiskView(self): if self._opImageOnDiskProvider is not None: self.ImageOnDisk.disconnect() self._opImageOnDiskProvider.cleanUp() self._opImageOnDiskProvider = None def setupOnDiskView(self): # Set up the output that let's us view the exported file self._opImageOnDiskProvider = OpImageOnDiskProvider( parent=self ) self._opImageOnDiskProvider.TransactionSlot.connect( self.TransactionSlot ) self._opImageOnDiskProvider.Input.connect( self._opFormattedExport.ImageToExport ) self._opImageOnDiskProvider.WorkingDirectory.connect( self.WorkingDirectory ) self._opImageOnDiskProvider.DatasetPath.connect( self._opFormattedExport.ExportPath ) # Not permitted to make this connection because we can't connect our own output to a child operator. # Instead, dirty state is copied manually into the child op whenever we change it. #self._opImageOnDiskProvider.Dirty.connect( self.Dirty ) self.ImageOnDisk.connect( self._opImageOnDiskProvider.Output ) def setupOutputs(self): self.cleanupOnDiskView() # FIXME: If RawData becomes unready() at the same time as RawDatasetInfo(), then # we have no guarantees about which one will trigger setupOutputs() first. # It is therefore possible for 'RawDatasetInfo' to appear ready() to us, # even though it's upstream partner is UNready. We are about to get the # unready() notification, but it will come too late to prevent our # setupOutputs method from being called. # Without proper graph setup transaction semantics, we have to use this # hack as a workaround. try: rawInfo = self.RawDatasetInfo.value except: for oslot in self.outputs.values(): if oslot.partner is None: oslot.meta.NOTREADY = True return dataset_dir = PathComponents(rawInfo.filePath).externalDirectory abs_dataset_dir, _ = getPathVariants(dataset_dir, self.WorkingDirectory.value) known_keys = {} known_keys['dataset_dir'] = abs_dataset_dir known_keys['nickname'] = rawInfo.nickname # Disconnect to open the 'transaction' if self._opImageOnDiskProvider is not None: self._opImageOnDiskProvider.TransactionSlot.disconnect() self._opFormattedExport.TransactionSlot.disconnect() # Blank the internal path while we manipulate the external path # to avoid invalid intermediate states of ExportPath self._opFormattedExport.OutputInternalPath.setValue( "" ) # use partial formatting to fill in non-coordinate name fields name_format = self.OutputFilenameFormat.value partially_formatted_name = format_known_keys( name_format, known_keys ) # Convert to absolute path before configuring the internal op abs_path, _ = getPathVariants( partially_formatted_name, self.WorkingDirectory.value ) self._opFormattedExport.OutputFilenameFormat.setValue( abs_path ) # use partial formatting on the internal dataset name, too internal_dataset_format = self.OutputInternalPath.value partially_formatted_dataset_name = format_known_keys( internal_dataset_format, known_keys ) self._opFormattedExport.OutputInternalPath.setValue( partially_formatted_dataset_name ) # Re-connect to finish the 'transaction' self._opFormattedExport.TransactionSlot.connect( self.TransactionSlot ) if self._opImageOnDiskProvider is not None: self._opImageOnDiskProvider.TransactionSlot.connect( self.TransactionSlot ) self.setupOnDiskView() def execute(self, slot, subindex, roi, result): assert False, "Shouldn't get here" def propagateDirty(self, slot, subindex, roi): # Out input data changed, so we have work to do when we get executed. self.Dirty.setValue(True) if self._opImageOnDiskProvider: self._opImageOnDiskProvider.Dirty.setValue( False ) def run_export(self): # If we're not dirty, we don't have to do anything. if self.Dirty.value: self.cleanupOnDiskView() self._opFormattedExport.run_export() self.Dirty.setValue( False ) self.setupOnDiskView() self._opImageOnDiskProvider.Dirty.setValue( False )
from PyQt4.QtGui import QApplication from lazyflow.graph import Graph, Operator, InputSlot from lazyflow.operators.ioOperators import OpFormattedDataExport class OpMock(Operator): OutputFilenameFormat = InputSlot(value='~/something.h5') OutputInternalPath = InputSlot(value='volume/data') OutputFormat = InputSlot(value='hdf5') FormatSelectionErrorMsg = InputSlot( value=True) # Normally an output slot def setupOutputs(self): pass def execute(self, *args): pass def propagateDirty(self, *args): pass op = OpFormattedDataExport(graph=Graph()) app = QApplication([]) w = MultiformatSlotExportFileOptionsWidget(None) w.initExportOp(op) w.show() app.exec_() print "Selected Filepath: {}".format(op.OutputFilenameFormat.value) print "Selected Dataset: {}".format(op.OutputInternalPath.value)
class OpDataExport(Operator): """ Top-level operator for the export applet. Mostly a simple wrapper for OpProcessedDataExport, but with extra formatting of the export path based on lane attributes. """ TransactionSlot = InputSlot() # To apply all settings in one 'transaction', # disconnect this slot and reconnect it when all slots are ready RawData = InputSlot(optional=True) # Display only FormattedRawData = OutputSlot() # OUTPUT - for overlaying the transformed raw data with the export data. # The dataset info for the original dataset (raw data) RawDatasetInfo = InputSlot() WorkingDirectory = ( InputSlot() ) # Non-absolute paths are relative to this directory. If not provided, paths must be absolute. Inputs = InputSlot(level=1) # The exportable slots (should all be of the same shape, except for channel) InputSelection = InputSlot(value=0) SelectionNames = InputSlot() # A list of names corresponding to the exportable inputs # Subregion params RegionStart = InputSlot(optional=True) RegionStop = InputSlot(optional=True) # Normalization params InputMin = InputSlot(optional=True) InputMax = InputSlot(optional=True) ExportMin = InputSlot(optional=True) ExportMax = InputSlot(optional=True) ExportDtype = InputSlot(optional=True) OutputAxisOrder = InputSlot(optional=True) # File settings OutputFilenameFormat = InputSlot( value="{dataset_dir}/{nickname}_{result_type}" ) # A format string allowing {dataset_dir} {nickname}, {roi}, {x_start}, {x_stop}, etc. OutputInternalPath = InputSlot(value="exported_data") OutputFormat = InputSlot(value="hdf5") # Only export csv/HDF5 table (don't export volume) TableOnlyName = InputSlot(value="Table-Only") TableOnly = InputSlot(value=False) ExportPath = OutputSlot() # Location of the saved file after export is complete. ConvertedImage = OutputSlot() # Cropped image, not yet re-ordered (useful for guis) ImageToExport = OutputSlot() # The image that will be exported Dirty = OutputSlot() # Whether or not the result currently matches what's on disk FormatSelectionErrorMsg = OutputSlot() ALL_FORMATS = OpFormattedDataExport.ALL_FORMATS #### # Simplified block diagram for actual export data and 'live preview' display: # # --> ExportPath # / # Input -> opFormattedExport --> ImageToExport (live preview) # |\ # \ --> ConvertedImage # \ # --> FormatSeletionIsValid #### # Simplified block diagram for Raw data display: # # RegionStart -- # \ # RegionStop --> OpRawSubRegionHelper.RawStart - # / .RawStop --\ # / \ # RawData ---->---------------------------------> opFormatRaw --> FormattedRawData #### def __init__(self, *args, **kwargs): super(OpDataExport, self).__init__(*args, **kwargs) self._opFormattedExport = OpFormattedDataExport(parent=self) opFormattedExport = self._opFormattedExport # Forward almost all inputs to the 'real' exporter opFormattedExport.TransactionSlot.connect(self.TransactionSlot) opFormattedExport.RegionStart.connect(self.RegionStart) opFormattedExport.RegionStop.connect(self.RegionStop) opFormattedExport.InputMin.connect(self.InputMin) opFormattedExport.InputMax.connect(self.InputMax) opFormattedExport.ExportMin.connect(self.ExportMin) opFormattedExport.ExportMax.connect(self.ExportMax) opFormattedExport.ExportDtype.connect(self.ExportDtype) opFormattedExport.OutputAxisOrder.connect(self.OutputAxisOrder) opFormattedExport.OutputFormat.connect(self.OutputFormat) self.ConvertedImage.connect(opFormattedExport.ConvertedImage) self.ImageToExport.connect(opFormattedExport.ImageToExport) self.ExportPath.connect(opFormattedExport.ExportPath) self.FormatSelectionErrorMsg.connect(opFormattedExport.FormatSelectionErrorMsg) self.progressSignal = opFormattedExport.progressSignal self.Dirty.setValue(True) # Default to Dirty # We don't export the raw data, but we connect it to it's own op # so it can be displayed alongside the data to export in the same viewer. # This keeps axis order, shape, etc. in sync with the displayed export data. # Note that we must not modify the channels of the raw data, so it gets passed through a helper. opHelper = OpRawSubRegionHelper(parent=self) opHelper.RawImage.connect(self.RawData) opHelper.ExportStart.connect(self.RegionStart) opHelper.ExportStop.connect(self.RegionStop) opFormatRaw = OpFormattedDataExport(parent=self) opFormatRaw.TransactionSlot.connect(self.TransactionSlot) opFormatRaw.Input.connect(self.RawData) opFormatRaw.RegionStart.connect(opHelper.RawStart) opFormatRaw.RegionStop.connect(opHelper.RawStop) # Don't normalize the raw data. # opFormatRaw.InputMin.connect( self.InputMin ) # opFormatRaw.InputMax.connect( self.InputMax ) # opFormatRaw.ExportMin.connect( self.ExportMin ) # opFormatRaw.ExportMax.connect( self.ExportMax ) # opFormatRaw.ExportDtype.connect( self.ExportDtype ) opFormatRaw.OutputAxisOrder.connect(self.OutputAxisOrder) opFormatRaw.OutputFormat.connect(self.OutputFormat) self._opFormatRaw = opFormatRaw self.FormattedRawData.connect(opFormatRaw.ImageToExport) def setupOutputs(self): # FIXME: If RawData becomes unready() at the same time as RawDatasetInfo(), then # we have no guarantees about which one will trigger setupOutputs() first. # It is therefore possible for 'RawDatasetInfo' to appear ready() to us, # even though it's upstream partner is UNready. We are about to get the # unready() notification, but it will come too late to prevent our # setupOutputs method from being called. # Without proper graph setup transaction semantics, we have to use this # hack as a workaround. try: rawInfo = self.RawDatasetInfo.value except: for oslot in list(self.outputs.values()): if oslot.upstream_slot is None: oslot.meta.NOTREADY = True return selection_index = self.InputSelection.value if not self.Inputs[selection_index].ready(): for oslot in list(self.outputs.values()): if oslot.upstream_slot is None: oslot.meta.NOTREADY = True return self._opFormattedExport.Input.connect(self.Inputs[selection_index]) dataset_dir = str(rawInfo.default_output_dir) abs_dataset_dir, _ = getPathVariants(dataset_dir, self.WorkingDirectory.value) known_keys = {} known_keys["dataset_dir"] = abs_dataset_dir nickname = rawInfo.nickname.replace("*", "") if os.path.pathsep in nickname: nickname = PathComponents(nickname.split(os.path.pathsep)[0]).fileNameBase known_keys["nickname"] = nickname result_types = self.SelectionNames.value known_keys["result_type"] = result_types[selection_index] self._opFormattedExport.TransactionSlot.disconnect() # Blank the internal path while we manipulate the external path # to avoid invalid intermediate states of ExportPath self._opFormattedExport.OutputInternalPath.setValue("") # use partial formatting to fill in non-coordinate name fields name_format = self.OutputFilenameFormat.value partially_formatted_name = format_known_keys(name_format, known_keys) # Convert to absolute path before configuring the internal op abs_path, _ = getPathVariants(partially_formatted_name, self.WorkingDirectory.value) self._opFormattedExport.OutputFilenameFormat.setValue(abs_path) # use partial formatting on the internal dataset name, too internal_dataset_format = self.OutputInternalPath.value partially_formatted_dataset_name = format_known_keys(internal_dataset_format, known_keys) self._opFormattedExport.OutputInternalPath.setValue(partially_formatted_dataset_name) # Re-connect to finish the 'transaction' self._opFormattedExport.TransactionSlot.connect(self.TransactionSlot) def execute(self, slot, subindex, roi, result): assert False, "Shouldn't get here" def propagateDirty(self, slot, subindex, roi): # Out input data changed, so we have work to do when we get executed. self.Dirty.setValue(True) def run_export(self): # If Table-Only is disabled or we're not dirty, we don't have to do anything. if not self.TableOnly.value and self.Dirty.value: self._opFormattedExport.run_export() self.Dirty.setValue(False) def run_export_to_array(self): # This function can be used to export the results to an in-memory array, instead of to disk # (Typically used from pure-python clients in batch mode.) return self._opFormattedExport.run_export_to_array()
class OpDataExport(Operator): """ Top-level operator for the export applet. Mostly a simple wrapper for OpProcessedDataExport, but with extra formatting of the export path based on lane attributes. """ TransactionSlot = InputSlot( ) # To apply all settings in one 'transaction', # disconnect this slot and reconnect it when all slots are ready RawData = InputSlot(optional=True) # Display only FormattedRawData = OutputSlot( ) # OUTPUT - for overlaying the transformed raw data with the export data. # The dataset info for the original dataset (raw data) RawDatasetInfo = InputSlot() WorkingDirectory = InputSlot( ) # Non-absolute paths are relative to this directory. If not provided, paths must be absolute. Input = InputSlot() # The results slot we want to export # Subregion params RegionStart = InputSlot(optional=True) RegionStop = InputSlot(optional=True) # Normalization params InputMin = InputSlot(optional=True) InputMax = InputSlot(optional=True) ExportMin = InputSlot(optional=True) ExportMax = InputSlot(optional=True) ExportDtype = InputSlot(optional=True) OutputAxisOrder = InputSlot(optional=True) # File settings OutputFilenameFormat = InputSlot( value='{dataset_dir}/{nickname}_export' ) # A format string allowing {dataset_dir} {nickname}, {roi}, {x_start}, {x_stop}, etc. OutputInternalPath = InputSlot(value='exported_data') OutputFormat = InputSlot(value='hdf5') ExportPath = OutputSlot( ) # Location of the saved file after export is complete. ConvertedImage = OutputSlot( ) # Cropped image, not yet re-ordered (useful for guis) ImageToExport = OutputSlot() # The image that will be exported ImageOnDisk = OutputSlot( ) # This slot reads the exported image from disk (after the export is complete) Dirty = OutputSlot( ) # Whether or not the result currently matches what's on disk FormatSelectionIsValid = OutputSlot() #### # Simplified block diagram for actual export data and 'live preview' display: # # --> ExportPath # / # Input -> opFormattedExport --> ImageToExport (live preview) # |\ # \ --> ConvertedImage # \ # --> FormatSeletionIsValid #### # Simplified block diagram for Raw data display: # # RegionStart -- # \ # RegionStop --> OpRawSubRegionHelper.RawStart - # / .RawStop --\ # / \ # RawData ---->---------------------------------> opFormatRaw --> FormattedRawData #### # Simplified block diagram for "on-disk view" of the exported results file: # # opFormattedExport.ImageToExport (for metadata only) --> # \ # opFormattedExport.ExportPath -------------------------> opImageOnDiskProvider --> ImageOnDisk def __init__(self, *args, **kwargs): super(OpDataExport, self).__init__(*args, **kwargs) self._opFormattedExport = OpFormattedDataExport(parent=self) opFormattedExport = self._opFormattedExport # Forward almost all inputs to the 'real' exporter opFormattedExport.TransactionSlot.connect(self.TransactionSlot) opFormattedExport.Input.connect(self.Input) opFormattedExport.RegionStart.connect(self.RegionStart) opFormattedExport.RegionStop.connect(self.RegionStop) opFormattedExport.InputMin.connect(self.InputMin) opFormattedExport.InputMax.connect(self.InputMax) opFormattedExport.ExportMin.connect(self.ExportMin) opFormattedExport.ExportMax.connect(self.ExportMax) opFormattedExport.ExportDtype.connect(self.ExportDtype) opFormattedExport.OutputAxisOrder.connect(self.OutputAxisOrder) opFormattedExport.OutputFormat.connect(self.OutputFormat) self.ConvertedImage.connect(opFormattedExport.ConvertedImage) self.ImageToExport.connect(opFormattedExport.ImageToExport) self.ExportPath.connect(opFormattedExport.ExportPath) self.FormatSelectionIsValid.connect( opFormattedExport.FormatSelectionIsValid) self.progressSignal = opFormattedExport.progressSignal self.Dirty.setValue(True) # Default to Dirty self._opImageOnDiskProvider = None # We don't export the raw data, but we connect it to it's own op # so it can be displayed alongside the data to export in the same viewer. # This keeps axis order, shape, etc. in sync with the displayed export data. # Note that we must not modify the channels of the raw data, so it gets passed throught a helper. opHelper = OpRawSubRegionHelper(parent=self) opHelper.RawImage.connect(self.RawData) opHelper.ExportStop.connect(self.RegionStart) opHelper.ExportStop.connect(self.RegionStop) opFormatRaw = OpFormattedDataExport(parent=self) opFormatRaw.TransactionSlot.connect(self.TransactionSlot) opFormatRaw.Input.connect(self.RawData) opFormatRaw.RegionStart.connect(opHelper.RawStart) opFormatRaw.RegionStop.connect(opHelper.RawStop) # Don't normalize the raw data. #opFormatRaw.InputMin.connect( self.InputMin ) #opFormatRaw.InputMax.connect( self.InputMax ) #opFormatRaw.ExportMin.connect( self.ExportMin ) #opFormatRaw.ExportMax.connect( self.ExportMax ) #opFormatRaw.ExportDtype.connect( self.ExportDtype ) opFormatRaw.OutputAxisOrder.connect(self.OutputAxisOrder) opFormatRaw.OutputFormat.connect(self.OutputFormat) self._opFormatRaw = opFormatRaw self.FormattedRawData.connect(opFormatRaw.ImageToExport) def cleanupOnDiskView(self): if self._opImageOnDiskProvider is not None: self.ImageOnDisk.disconnect() self._opImageOnDiskProvider.cleanUp() self._opImageOnDiskProvider = None def setupOnDiskView(self): # Set up the output that let's us view the exported file self._opImageOnDiskProvider = OpImageOnDiskProvider(parent=self) self._opImageOnDiskProvider.TransactionSlot.connect( self.TransactionSlot) self._opImageOnDiskProvider.Input.connect( self._opFormattedExport.ImageToExport) self._opImageOnDiskProvider.WorkingDirectory.connect( self.WorkingDirectory) self._opImageOnDiskProvider.DatasetPath.connect( self._opFormattedExport.ExportPath) self._opImageOnDiskProvider.Dirty.connect(self.Dirty) self.ImageOnDisk.connect(self._opImageOnDiskProvider.Output) def setupOutputs(self): self.cleanupOnDiskView() # FIXME: If RawData becomes unready() at the same time as RawDatasetInfo(), then # we have no guarantees about which one will trigger setupOutputs() first. # It is therefore possible for 'RawDatasetInfo' to appear ready() to us, # even though it's upstream partner is UNready. We are about to get the # unready() notification, but it will come too late to prevent our # setupOutputs method from being called. # Without proper graph setup transaction semantics, we have to use this # hack as a workaround. try: rawInfo = self.RawDatasetInfo.value except: for oslot in self.outputs.values(): if oslot.partner is None: oslot.meta.NOTREADY = True return dataset_dir = PathComponents(rawInfo.filePath).externalDirectory abs_dataset_dir, _ = getPathVariants(dataset_dir, self.WorkingDirectory.value) known_keys = {} known_keys['dataset_dir'] = abs_dataset_dir known_keys['nickname'] = rawInfo.nickname # Disconnect to open the 'transaction' if self._opImageOnDiskProvider is not None: self._opImageOnDiskProvider.TransactionSlot.disconnect() self._opFormattedExport.TransactionSlot.disconnect() # Blank the internal path while we manipulate the external path # to avoid invalid intermediate states of ExportPath self._opFormattedExport.OutputInternalPath.setValue("") # use partial formatting to fill in non-coordinate name fields name_format = self.OutputFilenameFormat.value partially_formatted_name = format_known_keys(name_format, known_keys) # Convert to absolute path before configuring the internal op abs_path, _ = getPathVariants(partially_formatted_name, self.WorkingDirectory.value) self._opFormattedExport.OutputFilenameFormat.setValue(abs_path) # use partial formatting on the internal dataset name, too internal_dataset_format = self.OutputInternalPath.value partially_formatted_dataset_name = format_known_keys( internal_dataset_format, known_keys) self._opFormattedExport.OutputInternalPath.setValue( partially_formatted_dataset_name) # Re-connect to finish the 'transaction' self._opFormattedExport.TransactionSlot.connect(self.TransactionSlot) if self._opImageOnDiskProvider is not None: self._opImageOnDiskProvider.TransactionSlot.connect( self.TransactionSlot) self.setupOnDiskView() def execute(self, slot, subindex, roi, result): assert False, "Shouldn't get here" def propagateDirty(self, slot, subindex, roi): # Out input data changed, so we have work to do when we get executed. self.Dirty.setValue(True) def run_export(self): # If we're not dirty, we don't have to do anything. if self.Dirty.value: self.cleanupOnDiskView() self._opFormattedExport.run_export() self.Dirty.setValue(False) self.setupOnDiskView()