def convert_predictions_to_uncertainties( input_path, parsed_export_args ): """ Read exported pixel predictions and calculate/export the uncertainties. input_path: The path to the prediction output file. If hdf5, must include the internal dataset name. parsed_export_args: The already-parsed cmd-line arguments generated from a DataExportApplet-compatible ArgumentParser. """ graph = Graph() opReader = OpInputDataReader(graph=graph) opReader.WorkingDirectory.setValue( os.getcwd() ) opReader.FilePath.setValue(input_path) opUncertainty = OpEnsembleMargin( graph=graph ) opUncertainty.Input.connect( opReader.Output ) opExport = OpFormattedDataExport( graph=graph ) opExport.Input.connect( opUncertainty.Output ) # Apply command-line arguments. DataExportApplet._configure_operator_with_parsed_args(parsed_export_args, opExport) last_progress = [-1] def print_progress(progress_percent): if progress_percent != last_progress[0]: last_progress[0] = progress_percent sys.stdout.write( " {}".format(progress_percent) ) print "Exporting results to : {}".format( opExport.ExportPath.value ) sys.stdout.write("Progress:") opExport.progressSignal.subscribe(print_progress) # Begin export opExport.run_export() sys.stdout.write("\n") print "DONE."
def testBasic(self): graph = Graph() opExport = OpFormattedDataExport(graph=graph) data = numpy.random.random((100, 100)).astype(numpy.float32) * 100 data = vigra.taggedView(data, vigra.defaultAxistags('xy')) opExport.Input.setValue(data) sub_roi = [(10, 0), (None, 80)] opExport.RegionStart.setValue(sub_roi[0]) opExport.RegionStop.setValue(sub_roi[1]) opExport.ExportDtype.setValue(numpy.uint8) opExport.InputMin.setValue(0.0) opExport.InputMax.setValue(100.0) opExport.ExportMin.setValue(100) opExport.ExportMax.setValue(200) opExport.OutputFormat.setValue('hdf5') opExport.OutputFilenameFormat.setValue( self._tmpdir + '/export_x{x_start}-{x_stop}_y{y_start}-{y_stop}') opExport.OutputInternalPath.setValue('volume/data') opExport.TransactionSlot.setValue(True) assert opExport.ImageToExport.ready() assert opExport.ExportPath.ready() assert opExport.ImageToExport.meta.drange == (100, 200) #print "exporting data to: {}".format( opExport.ExportPath.value ) assert opExport.ExportPath.value == self._tmpdir + '/' + 'export_x10-100_y0-80.h5/volume/data' opExport.run_export() opRead = OpInputDataReader(graph=graph) try: opRead.FilePath.setValue(opExport.ExportPath.value) # Compare with the correct subregion and convert dtype. sub_roi[1] = (100, 80) # Replace 'None' with full extent expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)] expected_data = expected_data.astype(numpy.uint8) expected_data += 100 # see renormalization settings assert opRead.Output.meta.shape == expected_data.shape assert opRead.Output.meta.dtype == expected_data.dtype read_data = opRead.Output[:].wait() # Due to rounding errors, the actual result and the expected result may differ by 1 # e.g. if the original pixel value was 32.99999999 # Also, must promote to signed values to avoid unsigned rollover # See issue ( https://github.com/ilastik/lazyflow/issues/165 ). expected_data_signed = expected_data.astype(numpy.int16) read_data_signed = expected_data.astype(numpy.int16) difference_from_expected = expected_data_signed - read_data_signed assert (numpy.abs(difference_from_expected) <= 1).all(), "Read data didn't match exported data!" finally: opRead.cleanUp()
def testBasic(self): graph = Graph() opExport = OpFormattedDataExport(graph=graph) data = numpy.random.random( (100,100) ).astype( numpy.float32 ) * 100 data = vigra.taggedView( data, vigra.defaultAxistags('xy') ) opExport.Input.setValue(data) sub_roi = [(10, 0), (None, 80)] opExport.RegionStart.setValue( sub_roi[0] ) opExport.RegionStop.setValue( sub_roi[1] ) opExport.ExportDtype.setValue( numpy.uint8 ) opExport.InputMin.setValue( 0.0 ) opExport.InputMax.setValue( 100.0 ) opExport.ExportMin.setValue( 100 ) opExport.ExportMax.setValue( 200 ) opExport.OutputFormat.setValue( 'hdf5' ) opExport.OutputFilenameFormat.setValue( self._tmpdir + '/export_x{x_start}-{x_stop}_y{y_start}-{y_stop}' ) opExport.OutputInternalPath.setValue('volume/data') opExport.TransactionSlot.setValue( True ) assert opExport.ImageToExport.ready() assert opExport.ExportPath.ready() assert opExport.ImageToExport.meta.drange == (100,200) #print "exporting data to: {}".format( opExport.ExportPath.value ) assert opExport.ExportPath.value == self._tmpdir + '/' + 'export_x10-100_y0-80.h5/volume/data' opExport.run_export() opRead = OpInputDataReader( graph=graph ) try: opRead.FilePath.setValue( opExport.ExportPath.value ) # Compare with the correct subregion and convert dtype. sub_roi[1] = (100, 80) # Replace 'None' with full extent expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)] expected_data = expected_data.astype(numpy.uint8) expected_data += 100 # see renormalization settings assert opRead.Output.meta.shape == expected_data.shape assert opRead.Output.meta.dtype == expected_data.dtype read_data = opRead.Output[:].wait() # Due to rounding errors, the actual result and the expected result may differ by 1 # e.g. if the original pixel value was 32.99999999 # Also, must promote to signed values to avoid unsigned rollover # See issue ( https://github.com/ilastik/lazyflow/issues/165 ). expected_data_signed = expected_data.astype(numpy.int16) read_data_signed = expected_data.astype(numpy.int16) difference_from_expected = expected_data_signed - read_data_signed assert (numpy.abs(difference_from_expected) <= 1).all(), "Read data didn't match exported data!" finally: opRead.cleanUp()
def convert_predictions_to_segmentation(input_paths, parsed_export_args): """ Read exported pixel predictions and calculate/export the segmentation. input_path: The path to the prediction output file. If hdf5, must include the internal dataset name. parsed_export_args: The already-parsed cmd-line arguments generated from a DataExportApplet-compatible ArgumentParser. """ graph = Graph() opReader = OpInputDataReader(graph=graph) opReader.WorkingDirectory.setValue(os.getcwd()) opArgmaxChannel = OpArgmaxChannel(graph=graph) opArgmaxChannel.Input.connect(opReader.Output) opExport = OpFormattedDataExport(graph=graph) opExport.Input.connect(opArgmaxChannel.Output) # Apply command-line arguments. DataExportApplet._configure_operator_with_parsed_args( parsed_export_args, opExport) last_progress = [-1] def print_progress(progress_percent): if progress_percent != last_progress[0]: last_progress[0] = progress_percent sys.stdout.write(" {}".format(progress_percent)) opExport.progressSignal.subscribe(print_progress) for input_path in input_paths: opReader.FilePath.setValue(input_path) input_pathcomp = PathComponents(input_path) opExport.OutputFilenameFormat.setValue(str( input_pathcomp.externalPath)) output_path = opExport.ExportPath.value output_pathcomp = PathComponents(output_path) output_pathcomp.filenameBase += "_Segmentation" opExport.OutputFilenameFormat.setValue( str(output_pathcomp.externalPath)) print("Exporting results to : {}".format(opExport.ExportPath.value)) sys.stdout.write("Progress:") # Begin export opExport.run_export() sys.stdout.write("\n") print("DONE.")
def convert_predictions_to_segmentation( input_paths, parsed_export_args ): """ Read exported pixel predictions and calculate/export the segmentation. input_path: The path to the prediction output file. If hdf5, must include the internal dataset name. parsed_export_args: The already-parsed cmd-line arguments generated from a DataExportApplet-compatible ArgumentParser. """ graph = Graph() opReader = OpInputDataReader(graph=graph) opReader.WorkingDirectory.setValue( os.getcwd() ) opArgmaxChannel = OpArgmaxChannel( graph=graph ) opArgmaxChannel.Input.connect( opReader.Output ) opExport = OpFormattedDataExport( graph=graph ) opExport.Input.connect( opArgmaxChannel.Output ) # Apply command-line arguments. DataExportApplet._configure_operator_with_parsed_args(parsed_export_args, opExport) last_progress = [-1] def print_progress(progress_percent): if progress_percent != last_progress[0]: last_progress[0] = progress_percent sys.stdout.write( " {}".format(progress_percent) ) opExport.progressSignal.subscribe(print_progress) for input_path in input_paths: opReader.FilePath.setValue(input_path) input_pathcomp = PathComponents(input_path) opExport.OutputFilenameFormat.setValue(str(input_pathcomp.externalPath)) output_path = opExport.ExportPath.value output_pathcomp = PathComponents( output_path ) output_pathcomp.filenameBase += "_Segmentation" opExport.OutputFilenameFormat.setValue(str(output_pathcomp.externalPath)) print "Exporting results to : {}".format( opExport.ExportPath.value ) sys.stdout.write("Progress:") # Begin export opExport.run_export() sys.stdout.write("\n") print "DONE."
def testBasic(self): graph = Graph() opExport = OpFormattedDataExport(graph=graph) data = numpy.random.random( (100,100) ).astype( numpy.float32 ) * 100 data = vigra.taggedView( data, vigra.defaultAxistags('xy') ) opExport.Input.setValue(data) sub_roi = [(10, 0), (None, 80)] opExport.RegionStart.setValue( sub_roi[0] ) opExport.RegionStop.setValue( sub_roi[1] ) opExport.ExportDtype.setValue( numpy.uint8 ) opExport.InputMin.setValue( 0.0 ) opExport.InputMax.setValue( 100.0 ) opExport.ExportMin.setValue( 100 ) opExport.ExportMax.setValue( 200 ) opExport.OutputFormat.setValue( 'hdf5' ) opExport.OutputFilenameFormat.setValue( self._tmpdir + '/export_x{x_start}-{x_stop}_y{y_start}-{y_stop}' ) opExport.OutputInternalPath.setValue('volume/data') opExport.TransactionSlot.setValue( True ) assert opExport.ImageToExport.ready() assert opExport.ExportPath.ready() assert opExport.ImageToExport.meta.drange == (100,200) #print "exporting data to: {}".format( opExport.ExportPath.value ) assert opExport.ExportPath.value == self._tmpdir + '/' + 'export_x10-100_y0-80.h5/volume/data' opExport.run_export() opRead = OpInputDataReader( graph=graph ) opRead.FilePath.setValue( opExport.ExportPath.value ) # Compare with the correct subregion and convert dtype. sub_roi[1] = (100, 80) # Replace 'None' with full extent expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)] expected_data = expected_data.astype(numpy.uint8) expected_data += 100 # see renormalization settings read_data = opRead.Output[:].wait() assert (read_data == expected_data).all(), "Read data didn't match exported data!"
class OpDataExport(Operator): """ Top-level operator for the export applet. Mostly a simple wrapper for OpProcessedDataExport, but with extra formatting of the export path based on lane attributes. """ TransactionSlot = InputSlot() # To apply all settings in one 'transaction', # disconnect this slot and reconnect it when all slots are ready RawData = InputSlot(optional=True) # Display only FormattedRawData = OutputSlot() # OUTPUT - for overlaying the transformed raw data with the export data. # The dataset info for the original dataset (raw data) RawDatasetInfo = InputSlot() WorkingDirectory = InputSlot() # Non-absolute paths are relative to this directory. If not provided, paths must be absolute. Input = InputSlot() # The results slot we want to export # Subregion params RegionStart = InputSlot(optional=True) RegionStop = InputSlot(optional=True) # Normalization params InputMin = InputSlot(optional=True) InputMax = InputSlot(optional=True) ExportMin = InputSlot(optional=True) ExportMax = InputSlot(optional=True) ExportDtype = InputSlot(optional=True) OutputAxisOrder = InputSlot(optional=True) # File settings OutputFilenameFormat = InputSlot(value='{dataset_dir}/{nickname}_export') # A format string allowing {dataset_dir} {nickname}, {roi}, {x_start}, {x_stop}, etc. OutputInternalPath = InputSlot(value='exported_data') OutputFormat = InputSlot(value='hdf5') ExportPath = OutputSlot() # Location of the saved file after export is complete. ConvertedImage = OutputSlot() # Cropped image, not yet re-ordered (useful for guis) ImageToExport = OutputSlot() # The image that will be exported ImageOnDisk = OutputSlot() # This slot reads the exported image from disk (after the export is complete) Dirty = OutputSlot() # Whether or not the result currently matches what's on disk FormatSelectionIsValid = OutputSlot() ALL_FORMATS = OpFormattedDataExport.ALL_FORMATS #### # Simplified block diagram for actual export data and 'live preview' display: # # --> ExportPath # / # Input -> opFormattedExport --> ImageToExport (live preview) # |\ # \ --> ConvertedImage # \ # --> FormatSeletionIsValid #### # Simplified block diagram for Raw data display: # # RegionStart -- # \ # RegionStop --> OpRawSubRegionHelper.RawStart - # / .RawStop --\ # / \ # RawData ---->---------------------------------> opFormatRaw --> FormattedRawData #### # Simplified block diagram for "on-disk view" of the exported results file: # # opFormattedExport.ImageToExport (for metadata only) --> # \ # opFormattedExport.ExportPath -------------------------> opImageOnDiskProvider --> ImageOnDisk def __init__(self, *args, **kwargs): super( OpDataExport, self ).__init__(*args, **kwargs) self._opFormattedExport = OpFormattedDataExport( parent=self ) opFormattedExport = self._opFormattedExport # Forward almost all inputs to the 'real' exporter opFormattedExport.TransactionSlot.connect( self.TransactionSlot ) opFormattedExport.Input.connect( self.Input ) opFormattedExport.RegionStart.connect( self.RegionStart ) opFormattedExport.RegionStop.connect( self.RegionStop ) opFormattedExport.InputMin.connect( self.InputMin ) opFormattedExport.InputMax.connect( self.InputMax ) opFormattedExport.ExportMin.connect( self.ExportMin ) opFormattedExport.ExportMax.connect( self.ExportMax ) opFormattedExport.ExportDtype.connect( self.ExportDtype ) opFormattedExport.OutputAxisOrder.connect( self.OutputAxisOrder ) opFormattedExport.OutputFormat.connect( self.OutputFormat ) self.ConvertedImage.connect( opFormattedExport.ConvertedImage ) self.ImageToExport.connect( opFormattedExport.ImageToExport ) self.ExportPath.connect( opFormattedExport.ExportPath ) self.FormatSelectionIsValid.connect( opFormattedExport.FormatSelectionIsValid ) self.progressSignal = opFormattedExport.progressSignal self.Dirty.setValue(True) # Default to Dirty self._opImageOnDiskProvider = None # We don't export the raw data, but we connect it to it's own op # so it can be displayed alongside the data to export in the same viewer. # This keeps axis order, shape, etc. in sync with the displayed export data. # Note that we must not modify the channels of the raw data, so it gets passed through a helper. opHelper = OpRawSubRegionHelper( parent=self ) opHelper.RawImage.connect( self.RawData ) opHelper.ExportStart.connect( self.RegionStart ) opHelper.ExportStop.connect( self.RegionStop ) opFormatRaw = OpFormattedDataExport( parent=self ) opFormatRaw.TransactionSlot.connect( self.TransactionSlot ) opFormatRaw.Input.connect( self.RawData ) opFormatRaw.RegionStart.connect( opHelper.RawStart ) opFormatRaw.RegionStop.connect( opHelper.RawStop ) # Don't normalize the raw data. #opFormatRaw.InputMin.connect( self.InputMin ) #opFormatRaw.InputMax.connect( self.InputMax ) #opFormatRaw.ExportMin.connect( self.ExportMin ) #opFormatRaw.ExportMax.connect( self.ExportMax ) #opFormatRaw.ExportDtype.connect( self.ExportDtype ) opFormatRaw.OutputAxisOrder.connect( self.OutputAxisOrder ) opFormatRaw.OutputFormat.connect( self.OutputFormat ) self._opFormatRaw = opFormatRaw self.FormattedRawData.connect( opFormatRaw.ImageToExport ) def cleanupOnDiskView(self): if self._opImageOnDiskProvider is not None: self.ImageOnDisk.disconnect() self._opImageOnDiskProvider.cleanUp() self._opImageOnDiskProvider = None def setupOnDiskView(self): # Set up the output that let's us view the exported file self._opImageOnDiskProvider = OpImageOnDiskProvider( parent=self ) self._opImageOnDiskProvider.TransactionSlot.connect( self.TransactionSlot ) self._opImageOnDiskProvider.Input.connect( self._opFormattedExport.ImageToExport ) self._opImageOnDiskProvider.WorkingDirectory.connect( self.WorkingDirectory ) self._opImageOnDiskProvider.DatasetPath.connect( self._opFormattedExport.ExportPath ) # Not permitted to make this connection because we can't connect our own output to a child operator. # Instead, dirty state is copied manually into the child op whenever we change it. #self._opImageOnDiskProvider.Dirty.connect( self.Dirty ) self.ImageOnDisk.connect( self._opImageOnDiskProvider.Output ) def setupOutputs(self): self.cleanupOnDiskView() # FIXME: If RawData becomes unready() at the same time as RawDatasetInfo(), then # we have no guarantees about which one will trigger setupOutputs() first. # It is therefore possible for 'RawDatasetInfo' to appear ready() to us, # even though it's upstream partner is UNready. We are about to get the # unready() notification, but it will come too late to prevent our # setupOutputs method from being called. # Without proper graph setup transaction semantics, we have to use this # hack as a workaround. try: rawInfo = self.RawDatasetInfo.value except: for oslot in self.outputs.values(): if oslot.partner is None: oslot.meta.NOTREADY = True return dataset_dir = PathComponents(rawInfo.filePath).externalDirectory abs_dataset_dir, _ = getPathVariants(dataset_dir, self.WorkingDirectory.value) known_keys = {} known_keys['dataset_dir'] = abs_dataset_dir known_keys['nickname'] = rawInfo.nickname # Disconnect to open the 'transaction' if self._opImageOnDiskProvider is not None: self._opImageOnDiskProvider.TransactionSlot.disconnect() self._opFormattedExport.TransactionSlot.disconnect() # Blank the internal path while we manipulate the external path # to avoid invalid intermediate states of ExportPath self._opFormattedExport.OutputInternalPath.setValue( "" ) # use partial formatting to fill in non-coordinate name fields name_format = self.OutputFilenameFormat.value partially_formatted_name = format_known_keys( name_format, known_keys ) # Convert to absolute path before configuring the internal op abs_path, _ = getPathVariants( partially_formatted_name, self.WorkingDirectory.value ) self._opFormattedExport.OutputFilenameFormat.setValue( abs_path ) # use partial formatting on the internal dataset name, too internal_dataset_format = self.OutputInternalPath.value partially_formatted_dataset_name = format_known_keys( internal_dataset_format, known_keys ) self._opFormattedExport.OutputInternalPath.setValue( partially_formatted_dataset_name ) # Re-connect to finish the 'transaction' self._opFormattedExport.TransactionSlot.connect( self.TransactionSlot ) if self._opImageOnDiskProvider is not None: self._opImageOnDiskProvider.TransactionSlot.connect( self.TransactionSlot ) self.setupOnDiskView() def execute(self, slot, subindex, roi, result): assert False, "Shouldn't get here" def propagateDirty(self, slot, subindex, roi): # Out input data changed, so we have work to do when we get executed. self.Dirty.setValue(True) if self._opImageOnDiskProvider: self._opImageOnDiskProvider.Dirty.setValue( False ) def run_export(self): # If we're not dirty, we don't have to do anything. if self.Dirty.value: self.cleanupOnDiskView() self._opFormattedExport.run_export() self.Dirty.setValue( False ) self.setupOnDiskView() self._opImageOnDiskProvider.Dirty.setValue( False )
class OpDataExport(Operator): """ Top-level operator for the export applet. Mostly a simple wrapper for OpProcessedDataExport, but with extra formatting of the export path based on lane attributes. """ TransactionSlot = InputSlot() # To apply all settings in one 'transaction', # disconnect this slot and reconnect it when all slots are ready RawData = InputSlot(optional=True) # Display only FormattedRawData = OutputSlot() # OUTPUT - for overlaying the transformed raw data with the export data. # The dataset info for the original dataset (raw data) RawDatasetInfo = InputSlot() WorkingDirectory = ( InputSlot() ) # Non-absolute paths are relative to this directory. If not provided, paths must be absolute. Inputs = InputSlot(level=1) # The exportable slots (should all be of the same shape, except for channel) InputSelection = InputSlot(value=0) SelectionNames = InputSlot() # A list of names corresponding to the exportable inputs # Subregion params RegionStart = InputSlot(optional=True) RegionStop = InputSlot(optional=True) # Normalization params InputMin = InputSlot(optional=True) InputMax = InputSlot(optional=True) ExportMin = InputSlot(optional=True) ExportMax = InputSlot(optional=True) ExportDtype = InputSlot(optional=True) OutputAxisOrder = InputSlot(optional=True) # File settings OutputFilenameFormat = InputSlot( value="{dataset_dir}/{nickname}_{result_type}" ) # A format string allowing {dataset_dir} {nickname}, {roi}, {x_start}, {x_stop}, etc. OutputInternalPath = InputSlot(value="exported_data") OutputFormat = InputSlot(value="hdf5") # Only export csv/HDF5 table (don't export volume) TableOnlyName = InputSlot(value="Table-Only") TableOnly = InputSlot(value=False) ExportPath = OutputSlot() # Location of the saved file after export is complete. ConvertedImage = OutputSlot() # Cropped image, not yet re-ordered (useful for guis) ImageToExport = OutputSlot() # The image that will be exported Dirty = OutputSlot() # Whether or not the result currently matches what's on disk FormatSelectionErrorMsg = OutputSlot() ALL_FORMATS = OpFormattedDataExport.ALL_FORMATS #### # Simplified block diagram for actual export data and 'live preview' display: # # --> ExportPath # / # Input -> opFormattedExport --> ImageToExport (live preview) # |\ # \ --> ConvertedImage # \ # --> FormatSeletionIsValid #### # Simplified block diagram for Raw data display: # # RegionStart -- # \ # RegionStop --> OpRawSubRegionHelper.RawStart - # / .RawStop --\ # / \ # RawData ---->---------------------------------> opFormatRaw --> FormattedRawData #### def __init__(self, *args, **kwargs): super(OpDataExport, self).__init__(*args, **kwargs) self._opFormattedExport = OpFormattedDataExport(parent=self) opFormattedExport = self._opFormattedExport # Forward almost all inputs to the 'real' exporter opFormattedExport.TransactionSlot.connect(self.TransactionSlot) opFormattedExport.RegionStart.connect(self.RegionStart) opFormattedExport.RegionStop.connect(self.RegionStop) opFormattedExport.InputMin.connect(self.InputMin) opFormattedExport.InputMax.connect(self.InputMax) opFormattedExport.ExportMin.connect(self.ExportMin) opFormattedExport.ExportMax.connect(self.ExportMax) opFormattedExport.ExportDtype.connect(self.ExportDtype) opFormattedExport.OutputAxisOrder.connect(self.OutputAxisOrder) opFormattedExport.OutputFormat.connect(self.OutputFormat) self.ConvertedImage.connect(opFormattedExport.ConvertedImage) self.ImageToExport.connect(opFormattedExport.ImageToExport) self.ExportPath.connect(opFormattedExport.ExportPath) self.FormatSelectionErrorMsg.connect(opFormattedExport.FormatSelectionErrorMsg) self.progressSignal = opFormattedExport.progressSignal self.Dirty.setValue(True) # Default to Dirty # We don't export the raw data, but we connect it to it's own op # so it can be displayed alongside the data to export in the same viewer. # This keeps axis order, shape, etc. in sync with the displayed export data. # Note that we must not modify the channels of the raw data, so it gets passed through a helper. opHelper = OpRawSubRegionHelper(parent=self) opHelper.RawImage.connect(self.RawData) opHelper.ExportStart.connect(self.RegionStart) opHelper.ExportStop.connect(self.RegionStop) opFormatRaw = OpFormattedDataExport(parent=self) opFormatRaw.TransactionSlot.connect(self.TransactionSlot) opFormatRaw.Input.connect(self.RawData) opFormatRaw.RegionStart.connect(opHelper.RawStart) opFormatRaw.RegionStop.connect(opHelper.RawStop) # Don't normalize the raw data. # opFormatRaw.InputMin.connect( self.InputMin ) # opFormatRaw.InputMax.connect( self.InputMax ) # opFormatRaw.ExportMin.connect( self.ExportMin ) # opFormatRaw.ExportMax.connect( self.ExportMax ) # opFormatRaw.ExportDtype.connect( self.ExportDtype ) opFormatRaw.OutputAxisOrder.connect(self.OutputAxisOrder) opFormatRaw.OutputFormat.connect(self.OutputFormat) self._opFormatRaw = opFormatRaw self.FormattedRawData.connect(opFormatRaw.ImageToExport) def setupOutputs(self): # FIXME: If RawData becomes unready() at the same time as RawDatasetInfo(), then # we have no guarantees about which one will trigger setupOutputs() first. # It is therefore possible for 'RawDatasetInfo' to appear ready() to us, # even though it's upstream partner is UNready. We are about to get the # unready() notification, but it will come too late to prevent our # setupOutputs method from being called. # Without proper graph setup transaction semantics, we have to use this # hack as a workaround. try: rawInfo = self.RawDatasetInfo.value except: for oslot in list(self.outputs.values()): if oslot.upstream_slot is None: oslot.meta.NOTREADY = True return selection_index = self.InputSelection.value if not self.Inputs[selection_index].ready(): for oslot in list(self.outputs.values()): if oslot.upstream_slot is None: oslot.meta.NOTREADY = True return self._opFormattedExport.Input.connect(self.Inputs[selection_index]) dataset_dir = str(rawInfo.default_output_dir) abs_dataset_dir, _ = getPathVariants(dataset_dir, self.WorkingDirectory.value) known_keys = {} known_keys["dataset_dir"] = abs_dataset_dir nickname = rawInfo.nickname.replace("*", "") if os.path.pathsep in nickname: nickname = PathComponents(nickname.split(os.path.pathsep)[0]).fileNameBase known_keys["nickname"] = nickname result_types = self.SelectionNames.value known_keys["result_type"] = result_types[selection_index] self._opFormattedExport.TransactionSlot.disconnect() # Blank the internal path while we manipulate the external path # to avoid invalid intermediate states of ExportPath self._opFormattedExport.OutputInternalPath.setValue("") # use partial formatting to fill in non-coordinate name fields name_format = self.OutputFilenameFormat.value partially_formatted_name = format_known_keys(name_format, known_keys) # Convert to absolute path before configuring the internal op abs_path, _ = getPathVariants(partially_formatted_name, self.WorkingDirectory.value) self._opFormattedExport.OutputFilenameFormat.setValue(abs_path) # use partial formatting on the internal dataset name, too internal_dataset_format = self.OutputInternalPath.value partially_formatted_dataset_name = format_known_keys(internal_dataset_format, known_keys) self._opFormattedExport.OutputInternalPath.setValue(partially_formatted_dataset_name) # Re-connect to finish the 'transaction' self._opFormattedExport.TransactionSlot.connect(self.TransactionSlot) def execute(self, slot, subindex, roi, result): assert False, "Shouldn't get here" def propagateDirty(self, slot, subindex, roi): # Out input data changed, so we have work to do when we get executed. self.Dirty.setValue(True) def run_export(self): # If Table-Only is disabled or we're not dirty, we don't have to do anything. if not self.TableOnly.value and self.Dirty.value: self._opFormattedExport.run_export() self.Dirty.setValue(False) def run_export_to_array(self): # This function can be used to export the results to an in-memory array, instead of to disk # (Typically used from pure-python clients in batch mode.) return self._opFormattedExport.run_export_to_array()
class OpDataExport(Operator): """ Top-level operator for the export applet. Mostly a simple wrapper for OpProcessedDataExport, but with extra formatting of the export path based on lane attributes. """ TransactionSlot = InputSlot( ) # To apply all settings in one 'transaction', # disconnect this slot and reconnect it when all slots are ready RawData = InputSlot(optional=True) # Display only FormattedRawData = OutputSlot( ) # OUTPUT - for overlaying the transformed raw data with the export data. # The dataset info for the original dataset (raw data) RawDatasetInfo = InputSlot() WorkingDirectory = InputSlot( ) # Non-absolute paths are relative to this directory. If not provided, paths must be absolute. Input = InputSlot() # The results slot we want to export # Subregion params RegionStart = InputSlot(optional=True) RegionStop = InputSlot(optional=True) # Normalization params InputMin = InputSlot(optional=True) InputMax = InputSlot(optional=True) ExportMin = InputSlot(optional=True) ExportMax = InputSlot(optional=True) ExportDtype = InputSlot(optional=True) OutputAxisOrder = InputSlot(optional=True) # File settings OutputFilenameFormat = InputSlot( value='{dataset_dir}/{nickname}_export' ) # A format string allowing {dataset_dir} {nickname}, {roi}, {x_start}, {x_stop}, etc. OutputInternalPath = InputSlot(value='exported_data') OutputFormat = InputSlot(value='hdf5') ExportPath = OutputSlot( ) # Location of the saved file after export is complete. ConvertedImage = OutputSlot( ) # Cropped image, not yet re-ordered (useful for guis) ImageToExport = OutputSlot() # The image that will be exported ImageOnDisk = OutputSlot( ) # This slot reads the exported image from disk (after the export is complete) Dirty = OutputSlot( ) # Whether or not the result currently matches what's on disk FormatSelectionIsValid = OutputSlot() #### # Simplified block diagram for actual export data and 'live preview' display: # # --> ExportPath # / # Input -> opFormattedExport --> ImageToExport (live preview) # |\ # \ --> ConvertedImage # \ # --> FormatSeletionIsValid #### # Simplified block diagram for Raw data display: # # RegionStart -- # \ # RegionStop --> OpRawSubRegionHelper.RawStart - # / .RawStop --\ # / \ # RawData ---->---------------------------------> opFormatRaw --> FormattedRawData #### # Simplified block diagram for "on-disk view" of the exported results file: # # opFormattedExport.ImageToExport (for metadata only) --> # \ # opFormattedExport.ExportPath -------------------------> opImageOnDiskProvider --> ImageOnDisk def __init__(self, *args, **kwargs): super(OpDataExport, self).__init__(*args, **kwargs) self._opFormattedExport = OpFormattedDataExport(parent=self) opFormattedExport = self._opFormattedExport # Forward almost all inputs to the 'real' exporter opFormattedExport.TransactionSlot.connect(self.TransactionSlot) opFormattedExport.Input.connect(self.Input) opFormattedExport.RegionStart.connect(self.RegionStart) opFormattedExport.RegionStop.connect(self.RegionStop) opFormattedExport.InputMin.connect(self.InputMin) opFormattedExport.InputMax.connect(self.InputMax) opFormattedExport.ExportMin.connect(self.ExportMin) opFormattedExport.ExportMax.connect(self.ExportMax) opFormattedExport.ExportDtype.connect(self.ExportDtype) opFormattedExport.OutputAxisOrder.connect(self.OutputAxisOrder) opFormattedExport.OutputFormat.connect(self.OutputFormat) self.ConvertedImage.connect(opFormattedExport.ConvertedImage) self.ImageToExport.connect(opFormattedExport.ImageToExport) self.ExportPath.connect(opFormattedExport.ExportPath) self.FormatSelectionIsValid.connect( opFormattedExport.FormatSelectionIsValid) self.progressSignal = opFormattedExport.progressSignal self.Dirty.setValue(True) # Default to Dirty self._opImageOnDiskProvider = None # We don't export the raw data, but we connect it to it's own op # so it can be displayed alongside the data to export in the same viewer. # This keeps axis order, shape, etc. in sync with the displayed export data. # Note that we must not modify the channels of the raw data, so it gets passed throught a helper. opHelper = OpRawSubRegionHelper(parent=self) opHelper.RawImage.connect(self.RawData) opHelper.ExportStop.connect(self.RegionStart) opHelper.ExportStop.connect(self.RegionStop) opFormatRaw = OpFormattedDataExport(parent=self) opFormatRaw.TransactionSlot.connect(self.TransactionSlot) opFormatRaw.Input.connect(self.RawData) opFormatRaw.RegionStart.connect(opHelper.RawStart) opFormatRaw.RegionStop.connect(opHelper.RawStop) # Don't normalize the raw data. #opFormatRaw.InputMin.connect( self.InputMin ) #opFormatRaw.InputMax.connect( self.InputMax ) #opFormatRaw.ExportMin.connect( self.ExportMin ) #opFormatRaw.ExportMax.connect( self.ExportMax ) #opFormatRaw.ExportDtype.connect( self.ExportDtype ) opFormatRaw.OutputAxisOrder.connect(self.OutputAxisOrder) opFormatRaw.OutputFormat.connect(self.OutputFormat) self._opFormatRaw = opFormatRaw self.FormattedRawData.connect(opFormatRaw.ImageToExport) def cleanupOnDiskView(self): if self._opImageOnDiskProvider is not None: self.ImageOnDisk.disconnect() self._opImageOnDiskProvider.cleanUp() self._opImageOnDiskProvider = None def setupOnDiskView(self): # Set up the output that let's us view the exported file self._opImageOnDiskProvider = OpImageOnDiskProvider(parent=self) self._opImageOnDiskProvider.TransactionSlot.connect( self.TransactionSlot) self._opImageOnDiskProvider.Input.connect( self._opFormattedExport.ImageToExport) self._opImageOnDiskProvider.WorkingDirectory.connect( self.WorkingDirectory) self._opImageOnDiskProvider.DatasetPath.connect( self._opFormattedExport.ExportPath) self._opImageOnDiskProvider.Dirty.connect(self.Dirty) self.ImageOnDisk.connect(self._opImageOnDiskProvider.Output) def setupOutputs(self): self.cleanupOnDiskView() # FIXME: If RawData becomes unready() at the same time as RawDatasetInfo(), then # we have no guarantees about which one will trigger setupOutputs() first. # It is therefore possible for 'RawDatasetInfo' to appear ready() to us, # even though it's upstream partner is UNready. We are about to get the # unready() notification, but it will come too late to prevent our # setupOutputs method from being called. # Without proper graph setup transaction semantics, we have to use this # hack as a workaround. try: rawInfo = self.RawDatasetInfo.value except: for oslot in self.outputs.values(): if oslot.partner is None: oslot.meta.NOTREADY = True return dataset_dir = PathComponents(rawInfo.filePath).externalDirectory abs_dataset_dir, _ = getPathVariants(dataset_dir, self.WorkingDirectory.value) known_keys = {} known_keys['dataset_dir'] = abs_dataset_dir known_keys['nickname'] = rawInfo.nickname # Disconnect to open the 'transaction' if self._opImageOnDiskProvider is not None: self._opImageOnDiskProvider.TransactionSlot.disconnect() self._opFormattedExport.TransactionSlot.disconnect() # Blank the internal path while we manipulate the external path # to avoid invalid intermediate states of ExportPath self._opFormattedExport.OutputInternalPath.setValue("") # use partial formatting to fill in non-coordinate name fields name_format = self.OutputFilenameFormat.value partially_formatted_name = format_known_keys(name_format, known_keys) # Convert to absolute path before configuring the internal op abs_path, _ = getPathVariants(partially_formatted_name, self.WorkingDirectory.value) self._opFormattedExport.OutputFilenameFormat.setValue(abs_path) # use partial formatting on the internal dataset name, too internal_dataset_format = self.OutputInternalPath.value partially_formatted_dataset_name = format_known_keys( internal_dataset_format, known_keys) self._opFormattedExport.OutputInternalPath.setValue( partially_formatted_dataset_name) # Re-connect to finish the 'transaction' self._opFormattedExport.TransactionSlot.connect(self.TransactionSlot) if self._opImageOnDiskProvider is not None: self._opImageOnDiskProvider.TransactionSlot.connect( self.TransactionSlot) self.setupOnDiskView() def execute(self, slot, subindex, roi, result): assert False, "Shouldn't get here" def propagateDirty(self, slot, subindex, roi): # Out input data changed, so we have work to do when we get executed. self.Dirty.setValue(True) def run_export(self): # If we're not dirty, we don't have to do anything. if self.Dirty.value: self.cleanupOnDiskView() self._opFormattedExport.run_export() self.Dirty.setValue(False) self.setupOnDiskView()