def convert_predictions_to_uncertainties( input_path, parsed_export_args ):
"""
Read exported pixel predictions and calculate/export the uncertainties.
input_path: The path to the prediction output file. If hdf5, must include the internal dataset name.
parsed_export_args: The already-parsed cmd-line arguments generated from a DataExportApplet-compatible ArgumentParser.
"""
graph = Graph()
opReader = OpInputDataReader(graph=graph)
opReader.WorkingDirectory.setValue( os.getcwd() )
opReader.FilePath.setValue(input_path)
opUncertainty = OpEnsembleMargin( graph=graph )
opUncertainty.Input.connect( opReader.Output )
opExport = OpFormattedDataExport( graph=graph )
opExport.Input.connect( opUncertainty.Output )
# Apply command-line arguments.
DataExportApplet._configure_operator_with_parsed_args(parsed_export_args, opExport)
last_progress = [-1]
def print_progress(progress_percent):
if progress_percent != last_progress[0]:
last_progress[0] = progress_percent
sys.stdout.write( " {}".format(progress_percent) )
print "Exporting results to : {}".format( opExport.ExportPath.value )
sys.stdout.write("Progress:")
opExport.progressSignal.subscribe(print_progress)
# Begin export
opExport.run_export()
sys.stdout.write("\n")
print "DONE."
def testBasic(self):
graph = Graph()
opExport = OpFormattedDataExport(graph=graph)
data = numpy.random.random((100, 100)).astype(numpy.float32) * 100
data = vigra.taggedView(data, vigra.defaultAxistags('xy'))
opExport.Input.setValue(data)
sub_roi = [(10, 0), (None, 80)]
opExport.RegionStart.setValue(sub_roi[0])
opExport.RegionStop.setValue(sub_roi[1])
opExport.ExportDtype.setValue(numpy.uint8)
opExport.InputMin.setValue(0.0)
opExport.InputMax.setValue(100.0)
opExport.ExportMin.setValue(100)
opExport.ExportMax.setValue(200)
opExport.OutputFormat.setValue('hdf5')
opExport.OutputFilenameFormat.setValue(
self._tmpdir + '/export_x{x_start}-{x_stop}_y{y_start}-{y_stop}')
opExport.OutputInternalPath.setValue('volume/data')
opExport.TransactionSlot.setValue(True)
assert opExport.ImageToExport.ready()
assert opExport.ExportPath.ready()
assert opExport.ImageToExport.meta.drange == (100, 200)
#print "exporting data to: {}".format( opExport.ExportPath.value )
assert opExport.ExportPath.value == self._tmpdir + '/' + 'export_x10-100_y0-80.h5/volume/data'
opExport.run_export()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(opExport.ExportPath.value)
# Compare with the correct subregion and convert dtype.
sub_roi[1] = (100, 80) # Replace 'None' with full extent
expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)]
expected_data = expected_data.astype(numpy.uint8)
expected_data += 100 # see renormalization settings
assert opRead.Output.meta.shape == expected_data.shape
assert opRead.Output.meta.dtype == expected_data.dtype
read_data = opRead.Output[:].wait()
# Due to rounding errors, the actual result and the expected result may differ by 1
# e.g. if the original pixel value was 32.99999999
# Also, must promote to signed values to avoid unsigned rollover
# See issue ( https://github.com/ilastik/lazyflow/issues/165 ).
expected_data_signed = expected_data.astype(numpy.int16)
read_data_signed = expected_data.astype(numpy.int16)
difference_from_expected = expected_data_signed - read_data_signed
assert (numpy.abs(difference_from_expected) <=
1).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def testBasic(self):
graph = Graph()
opExport = OpFormattedDataExport(graph=graph)
data = numpy.random.random( (100,100) ).astype( numpy.float32 ) * 100
data = vigra.taggedView( data, vigra.defaultAxistags('xy') )
opExport.Input.setValue(data)
sub_roi = [(10, 0), (None, 80)]
opExport.RegionStart.setValue( sub_roi[0] )
opExport.RegionStop.setValue( sub_roi[1] )
opExport.ExportDtype.setValue( numpy.uint8 )
opExport.InputMin.setValue( 0.0 )
opExport.InputMax.setValue( 100.0 )
opExport.ExportMin.setValue( 100 )
opExport.ExportMax.setValue( 200 )
opExport.OutputFormat.setValue( 'hdf5' )
opExport.OutputFilenameFormat.setValue( self._tmpdir + '/export_x{x_start}-{x_stop}_y{y_start}-{y_stop}' )
opExport.OutputInternalPath.setValue('volume/data')
opExport.TransactionSlot.setValue( True )
assert opExport.ImageToExport.ready()
assert opExport.ExportPath.ready()
assert opExport.ImageToExport.meta.drange == (100,200)
#print "exporting data to: {}".format( opExport.ExportPath.value )
assert opExport.ExportPath.value == self._tmpdir + '/' + 'export_x10-100_y0-80.h5/volume/data'
opExport.run_export()
opRead = OpInputDataReader( graph=graph )
try:
opRead.FilePath.setValue( opExport.ExportPath.value )
# Compare with the correct subregion and convert dtype.
sub_roi[1] = (100, 80) # Replace 'None' with full extent
expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)]
expected_data = expected_data.astype(numpy.uint8)
expected_data += 100 # see renormalization settings
assert opRead.Output.meta.shape == expected_data.shape
assert opRead.Output.meta.dtype == expected_data.dtype
read_data = opRead.Output[:].wait()
# Due to rounding errors, the actual result and the expected result may differ by 1
# e.g. if the original pixel value was 32.99999999
# Also, must promote to signed values to avoid unsigned rollover
# See issue ( https://github.com/ilastik/lazyflow/issues/165 ).
expected_data_signed = expected_data.astype(numpy.int16)
read_data_signed = expected_data.astype(numpy.int16)
difference_from_expected = expected_data_signed - read_data_signed
assert (numpy.abs(difference_from_expected) <= 1).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def convert_predictions_to_segmentation(input_paths, parsed_export_args):
"""
Read exported pixel predictions and calculate/export the segmentation.
input_path: The path to the prediction output file. If hdf5, must include the internal dataset name.
parsed_export_args: The already-parsed cmd-line arguments generated from a DataExportApplet-compatible ArgumentParser.
"""
graph = Graph()
opReader = OpInputDataReader(graph=graph)
opReader.WorkingDirectory.setValue(os.getcwd())
opArgmaxChannel = OpArgmaxChannel(graph=graph)
opArgmaxChannel.Input.connect(opReader.Output)
opExport = OpFormattedDataExport(graph=graph)
opExport.Input.connect(opArgmaxChannel.Output)
# Apply command-line arguments.
DataExportApplet._configure_operator_with_parsed_args(
parsed_export_args, opExport)
last_progress = [-1]
def print_progress(progress_percent):
if progress_percent != last_progress[0]:
last_progress[0] = progress_percent
sys.stdout.write(" {}".format(progress_percent))
opExport.progressSignal.subscribe(print_progress)
for input_path in input_paths:
opReader.FilePath.setValue(input_path)
input_pathcomp = PathComponents(input_path)
opExport.OutputFilenameFormat.setValue(str(
input_pathcomp.externalPath))
output_path = opExport.ExportPath.value
output_pathcomp = PathComponents(output_path)
output_pathcomp.filenameBase += "_Segmentation"
opExport.OutputFilenameFormat.setValue(
str(output_pathcomp.externalPath))
print("Exporting results to : {}".format(opExport.ExportPath.value))
sys.stdout.write("Progress:")
# Begin export
opExport.run_export()
sys.stdout.write("\n")
print("DONE.")
def convert_predictions_to_segmentation( input_paths, parsed_export_args ):
"""
Read exported pixel predictions and calculate/export the segmentation.
input_path: The path to the prediction output file. If hdf5, must include the internal dataset name.
parsed_export_args: The already-parsed cmd-line arguments generated from a DataExportApplet-compatible ArgumentParser.
"""
graph = Graph()
opReader = OpInputDataReader(graph=graph)
opReader.WorkingDirectory.setValue( os.getcwd() )
opArgmaxChannel = OpArgmaxChannel( graph=graph )
opArgmaxChannel.Input.connect( opReader.Output )
opExport = OpFormattedDataExport( graph=graph )
opExport.Input.connect( opArgmaxChannel.Output )
# Apply command-line arguments.
DataExportApplet._configure_operator_with_parsed_args(parsed_export_args, opExport)
last_progress = [-1]
def print_progress(progress_percent):
if progress_percent != last_progress[0]:
last_progress[0] = progress_percent
sys.stdout.write( " {}".format(progress_percent) )
opExport.progressSignal.subscribe(print_progress)
for input_path in input_paths:
opReader.FilePath.setValue(input_path)
input_pathcomp = PathComponents(input_path)
opExport.OutputFilenameFormat.setValue(str(input_pathcomp.externalPath))
output_path = opExport.ExportPath.value
output_pathcomp = PathComponents( output_path )
output_pathcomp.filenameBase += "_Segmentation"
opExport.OutputFilenameFormat.setValue(str(output_pathcomp.externalPath))
print "Exporting results to : {}".format( opExport.ExportPath.value )
sys.stdout.write("Progress:")
# Begin export
opExport.run_export()
sys.stdout.write("\n")
print "DONE."
def testBasic(self):
graph = Graph()
opExport = OpFormattedDataExport(graph=graph)
data = numpy.random.random( (100,100) ).astype( numpy.float32 ) * 100
data = vigra.taggedView( data, vigra.defaultAxistags('xy') )
opExport.Input.setValue(data)
sub_roi = [(10, 0), (None, 80)]
opExport.RegionStart.setValue( sub_roi[0] )
opExport.RegionStop.setValue( sub_roi[1] )
opExport.ExportDtype.setValue( numpy.uint8 )
opExport.InputMin.setValue( 0.0 )
opExport.InputMax.setValue( 100.0 )
opExport.ExportMin.setValue( 100 )
opExport.ExportMax.setValue( 200 )
opExport.OutputFormat.setValue( 'hdf5' )
opExport.OutputFilenameFormat.setValue( self._tmpdir + '/export_x{x_start}-{x_stop}_y{y_start}-{y_stop}' )
opExport.OutputInternalPath.setValue('volume/data')
opExport.TransactionSlot.setValue( True )
assert opExport.ImageToExport.ready()
assert opExport.ExportPath.ready()
assert opExport.ImageToExport.meta.drange == (100,200)
#print "exporting data to: {}".format( opExport.ExportPath.value )
assert opExport.ExportPath.value == self._tmpdir + '/' + 'export_x10-100_y0-80.h5/volume/data'
opExport.run_export()
opRead = OpInputDataReader( graph=graph )
opRead.FilePath.setValue( opExport.ExportPath.value )
# Compare with the correct subregion and convert dtype.
sub_roi[1] = (100, 80) # Replace 'None' with full extent
expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)]
expected_data = expected_data.astype(numpy.uint8)
expected_data += 100 # see renormalization settings
read_data = opRead.Output[:].wait()
assert (read_data == expected_data).all(), "Read data didn't match exported data!"
class OpDataExport(Operator):
"""
Top-level operator for the export applet.
Mostly a simple wrapper for OpProcessedDataExport, but with extra formatting of the export path based on lane attributes.
"""
TransactionSlot = InputSlot() # To apply all settings in one 'transaction',
# disconnect this slot and reconnect it when all slots are ready
RawData = InputSlot(optional=True) # Display only
FormattedRawData = OutputSlot() # OUTPUT - for overlaying the transformed raw data with the export data.
# The dataset info for the original dataset (raw data)
RawDatasetInfo = InputSlot()
WorkingDirectory = InputSlot() # Non-absolute paths are relative to this directory. If not provided, paths must be absolute.
Input = InputSlot() # The results slot we want to export
# Subregion params
RegionStart = InputSlot(optional=True)
RegionStop = InputSlot(optional=True)
# Normalization params
InputMin = InputSlot(optional=True)
InputMax = InputSlot(optional=True)
ExportMin = InputSlot(optional=True)
ExportMax = InputSlot(optional=True)
ExportDtype = InputSlot(optional=True)
OutputAxisOrder = InputSlot(optional=True)
# File settings
OutputFilenameFormat = InputSlot(value='{dataset_dir}/{nickname}_export') # A format string allowing {dataset_dir} {nickname}, {roi}, {x_start}, {x_stop}, etc.
OutputInternalPath = InputSlot(value='exported_data')
OutputFormat = InputSlot(value='hdf5')
ExportPath = OutputSlot() # Location of the saved file after export is complete.
ConvertedImage = OutputSlot() # Cropped image, not yet re-ordered (useful for guis)
ImageToExport = OutputSlot() # The image that will be exported
ImageOnDisk = OutputSlot() # This slot reads the exported image from disk (after the export is complete)
Dirty = OutputSlot() # Whether or not the result currently matches what's on disk
FormatSelectionIsValid = OutputSlot()
ALL_FORMATS = OpFormattedDataExport.ALL_FORMATS
####
# Simplified block diagram for actual export data and 'live preview' display:
#
# --> ExportPath
# /
# Input -> opFormattedExport --> ImageToExport (live preview)
# |\
# \ --> ConvertedImage
# \
# --> FormatSeletionIsValid
####
# Simplified block diagram for Raw data display:
#
# RegionStart --
# \
# RegionStop --> OpRawSubRegionHelper.RawStart -
# / .RawStop --\
# / \
# RawData ---->---------------------------------> opFormatRaw --> FormattedRawData
####
# Simplified block diagram for "on-disk view" of the exported results file:
#
# opFormattedExport.ImageToExport (for metadata only) -->
# \
# opFormattedExport.ExportPath -------------------------> opImageOnDiskProvider --> ImageOnDisk
def __init__(self, *args, **kwargs):
super( OpDataExport, self ).__init__(*args, **kwargs)
self._opFormattedExport = OpFormattedDataExport( parent=self )
opFormattedExport = self._opFormattedExport
# Forward almost all inputs to the 'real' exporter
opFormattedExport.TransactionSlot.connect( self.TransactionSlot )
opFormattedExport.Input.connect( self.Input )
opFormattedExport.RegionStart.connect( self.RegionStart )
opFormattedExport.RegionStop.connect( self.RegionStop )
opFormattedExport.InputMin.connect( self.InputMin )
opFormattedExport.InputMax.connect( self.InputMax )
opFormattedExport.ExportMin.connect( self.ExportMin )
opFormattedExport.ExportMax.connect( self.ExportMax )
opFormattedExport.ExportDtype.connect( self.ExportDtype )
opFormattedExport.OutputAxisOrder.connect( self.OutputAxisOrder )
opFormattedExport.OutputFormat.connect( self.OutputFormat )
self.ConvertedImage.connect( opFormattedExport.ConvertedImage )
self.ImageToExport.connect( opFormattedExport.ImageToExport )
self.ExportPath.connect( opFormattedExport.ExportPath )
self.FormatSelectionIsValid.connect( opFormattedExport.FormatSelectionIsValid )
self.progressSignal = opFormattedExport.progressSignal
self.Dirty.setValue(True) # Default to Dirty
self._opImageOnDiskProvider = None
# We don't export the raw data, but we connect it to it's own op
# so it can be displayed alongside the data to export in the same viewer.
# This keeps axis order, shape, etc. in sync with the displayed export data.
# Note that we must not modify the channels of the raw data, so it gets passed through a helper.
opHelper = OpRawSubRegionHelper( parent=self )
opHelper.RawImage.connect( self.RawData )
opHelper.ExportStart.connect( self.RegionStart )
opHelper.ExportStop.connect( self.RegionStop )
opFormatRaw = OpFormattedDataExport( parent=self )
opFormatRaw.TransactionSlot.connect( self.TransactionSlot )
opFormatRaw.Input.connect( self.RawData )
opFormatRaw.RegionStart.connect( opHelper.RawStart )
opFormatRaw.RegionStop.connect( opHelper.RawStop )
# Don't normalize the raw data.
#opFormatRaw.InputMin.connect( self.InputMin )
#opFormatRaw.InputMax.connect( self.InputMax )
#opFormatRaw.ExportMin.connect( self.ExportMin )
#opFormatRaw.ExportMax.connect( self.ExportMax )
#opFormatRaw.ExportDtype.connect( self.ExportDtype )
opFormatRaw.OutputAxisOrder.connect( self.OutputAxisOrder )
opFormatRaw.OutputFormat.connect( self.OutputFormat )
self._opFormatRaw = opFormatRaw
self.FormattedRawData.connect( opFormatRaw.ImageToExport )
def cleanupOnDiskView(self):
if self._opImageOnDiskProvider is not None:
self.ImageOnDisk.disconnect()
self._opImageOnDiskProvider.cleanUp()
self._opImageOnDiskProvider = None
def setupOnDiskView(self):
# Set up the output that let's us view the exported file
self._opImageOnDiskProvider = OpImageOnDiskProvider( parent=self )
self._opImageOnDiskProvider.TransactionSlot.connect( self.TransactionSlot )
self._opImageOnDiskProvider.Input.connect( self._opFormattedExport.ImageToExport )
self._opImageOnDiskProvider.WorkingDirectory.connect( self.WorkingDirectory )
self._opImageOnDiskProvider.DatasetPath.connect( self._opFormattedExport.ExportPath )
# Not permitted to make this connection because we can't connect our own output to a child operator.
# Instead, dirty state is copied manually into the child op whenever we change it.
#self._opImageOnDiskProvider.Dirty.connect( self.Dirty )
self.ImageOnDisk.connect( self._opImageOnDiskProvider.Output )
def setupOutputs(self):
self.cleanupOnDiskView()
# FIXME: If RawData becomes unready() at the same time as RawDatasetInfo(), then
# we have no guarantees about which one will trigger setupOutputs() first.
# It is therefore possible for 'RawDatasetInfo' to appear ready() to us,
# even though it's upstream partner is UNready. We are about to get the
# unready() notification, but it will come too late to prevent our
# setupOutputs method from being called.
# Without proper graph setup transaction semantics, we have to use this
# hack as a workaround.
try:
rawInfo = self.RawDatasetInfo.value
except:
for oslot in self.outputs.values():
if oslot.partner is None:
oslot.meta.NOTREADY = True
return
dataset_dir = PathComponents(rawInfo.filePath).externalDirectory
abs_dataset_dir, _ = getPathVariants(dataset_dir, self.WorkingDirectory.value)
known_keys = {}
known_keys['dataset_dir'] = abs_dataset_dir
known_keys['nickname'] = rawInfo.nickname
# Disconnect to open the 'transaction'
if self._opImageOnDiskProvider is not None:
self._opImageOnDiskProvider.TransactionSlot.disconnect()
self._opFormattedExport.TransactionSlot.disconnect()
# Blank the internal path while we manipulate the external path
# to avoid invalid intermediate states of ExportPath
self._opFormattedExport.OutputInternalPath.setValue( "" )
# use partial formatting to fill in non-coordinate name fields
name_format = self.OutputFilenameFormat.value
partially_formatted_name = format_known_keys( name_format, known_keys )
# Convert to absolute path before configuring the internal op
abs_path, _ = getPathVariants( partially_formatted_name, self.WorkingDirectory.value )
self._opFormattedExport.OutputFilenameFormat.setValue( abs_path )
# use partial formatting on the internal dataset name, too
internal_dataset_format = self.OutputInternalPath.value
partially_formatted_dataset_name = format_known_keys( internal_dataset_format, known_keys )
self._opFormattedExport.OutputInternalPath.setValue( partially_formatted_dataset_name )
# Re-connect to finish the 'transaction'
self._opFormattedExport.TransactionSlot.connect( self.TransactionSlot )
if self._opImageOnDiskProvider is not None:
self._opImageOnDiskProvider.TransactionSlot.connect( self.TransactionSlot )
self.setupOnDiskView()
def execute(self, slot, subindex, roi, result):
assert False, "Shouldn't get here"
def propagateDirty(self, slot, subindex, roi):
# Out input data changed, so we have work to do when we get executed.
self.Dirty.setValue(True)
if self._opImageOnDiskProvider:
self._opImageOnDiskProvider.Dirty.setValue( False )
def run_export(self):
# If we're not dirty, we don't have to do anything.
if self.Dirty.value:
self.cleanupOnDiskView()
self._opFormattedExport.run_export()
self.Dirty.setValue( False )
self.setupOnDiskView()
self._opImageOnDiskProvider.Dirty.setValue( False )
class OpDataExport(Operator):
"""
Top-level operator for the export applet.
Mostly a simple wrapper for OpProcessedDataExport, but with extra formatting of the export path based on lane attributes.
"""
TransactionSlot = InputSlot() # To apply all settings in one 'transaction',
# disconnect this slot and reconnect it when all slots are ready
RawData = InputSlot(optional=True) # Display only
FormattedRawData = OutputSlot() # OUTPUT - for overlaying the transformed raw data with the export data.
# The dataset info for the original dataset (raw data)
RawDatasetInfo = InputSlot()
WorkingDirectory = (
InputSlot()
) # Non-absolute paths are relative to this directory. If not provided, paths must be absolute.
Inputs = InputSlot(level=1) # The exportable slots (should all be of the same shape, except for channel)
InputSelection = InputSlot(value=0)
SelectionNames = InputSlot() # A list of names corresponding to the exportable inputs
# Subregion params
RegionStart = InputSlot(optional=True)
RegionStop = InputSlot(optional=True)
# Normalization params
InputMin = InputSlot(optional=True)
InputMax = InputSlot(optional=True)
ExportMin = InputSlot(optional=True)
ExportMax = InputSlot(optional=True)
ExportDtype = InputSlot(optional=True)
OutputAxisOrder = InputSlot(optional=True)
# File settings
OutputFilenameFormat = InputSlot(
value="{dataset_dir}/{nickname}_{result_type}"
) # A format string allowing {dataset_dir} {nickname}, {roi}, {x_start}, {x_stop}, etc.
OutputInternalPath = InputSlot(value="exported_data")
OutputFormat = InputSlot(value="hdf5")
# Only export csv/HDF5 table (don't export volume)
TableOnlyName = InputSlot(value="Table-Only")
TableOnly = InputSlot(value=False)
ExportPath = OutputSlot() # Location of the saved file after export is complete.
ConvertedImage = OutputSlot() # Cropped image, not yet re-ordered (useful for guis)
ImageToExport = OutputSlot() # The image that will be exported
Dirty = OutputSlot() # Whether or not the result currently matches what's on disk
FormatSelectionErrorMsg = OutputSlot()
ALL_FORMATS = OpFormattedDataExport.ALL_FORMATS
####
# Simplified block diagram for actual export data and 'live preview' display:
#
# --> ExportPath
# /
# Input -> opFormattedExport --> ImageToExport (live preview)
# |\
# \ --> ConvertedImage
# \
# --> FormatSeletionIsValid
####
# Simplified block diagram for Raw data display:
#
# RegionStart --
# \
# RegionStop --> OpRawSubRegionHelper.RawStart -
# / .RawStop --\
# / \
# RawData ---->---------------------------------> opFormatRaw --> FormattedRawData
####
def __init__(self, *args, **kwargs):
super(OpDataExport, self).__init__(*args, **kwargs)
self._opFormattedExport = OpFormattedDataExport(parent=self)
opFormattedExport = self._opFormattedExport
# Forward almost all inputs to the 'real' exporter
opFormattedExport.TransactionSlot.connect(self.TransactionSlot)
opFormattedExport.RegionStart.connect(self.RegionStart)
opFormattedExport.RegionStop.connect(self.RegionStop)
opFormattedExport.InputMin.connect(self.InputMin)
opFormattedExport.InputMax.connect(self.InputMax)
opFormattedExport.ExportMin.connect(self.ExportMin)
opFormattedExport.ExportMax.connect(self.ExportMax)
opFormattedExport.ExportDtype.connect(self.ExportDtype)
opFormattedExport.OutputAxisOrder.connect(self.OutputAxisOrder)
opFormattedExport.OutputFormat.connect(self.OutputFormat)
self.ConvertedImage.connect(opFormattedExport.ConvertedImage)
self.ImageToExport.connect(opFormattedExport.ImageToExport)
self.ExportPath.connect(opFormattedExport.ExportPath)
self.FormatSelectionErrorMsg.connect(opFormattedExport.FormatSelectionErrorMsg)
self.progressSignal = opFormattedExport.progressSignal
self.Dirty.setValue(True) # Default to Dirty
# We don't export the raw data, but we connect it to it's own op
# so it can be displayed alongside the data to export in the same viewer.
# This keeps axis order, shape, etc. in sync with the displayed export data.
# Note that we must not modify the channels of the raw data, so it gets passed through a helper.
opHelper = OpRawSubRegionHelper(parent=self)
opHelper.RawImage.connect(self.RawData)
opHelper.ExportStart.connect(self.RegionStart)
opHelper.ExportStop.connect(self.RegionStop)
opFormatRaw = OpFormattedDataExport(parent=self)
opFormatRaw.TransactionSlot.connect(self.TransactionSlot)
opFormatRaw.Input.connect(self.RawData)
opFormatRaw.RegionStart.connect(opHelper.RawStart)
opFormatRaw.RegionStop.connect(opHelper.RawStop)
# Don't normalize the raw data.
# opFormatRaw.InputMin.connect( self.InputMin )
# opFormatRaw.InputMax.connect( self.InputMax )
# opFormatRaw.ExportMin.connect( self.ExportMin )
# opFormatRaw.ExportMax.connect( self.ExportMax )
# opFormatRaw.ExportDtype.connect( self.ExportDtype )
opFormatRaw.OutputAxisOrder.connect(self.OutputAxisOrder)
opFormatRaw.OutputFormat.connect(self.OutputFormat)
self._opFormatRaw = opFormatRaw
self.FormattedRawData.connect(opFormatRaw.ImageToExport)
def setupOutputs(self):
# FIXME: If RawData becomes unready() at the same time as RawDatasetInfo(), then
# we have no guarantees about which one will trigger setupOutputs() first.
# It is therefore possible for 'RawDatasetInfo' to appear ready() to us,
# even though it's upstream partner is UNready. We are about to get the
# unready() notification, but it will come too late to prevent our
# setupOutputs method from being called.
# Without proper graph setup transaction semantics, we have to use this
# hack as a workaround.
try:
rawInfo = self.RawDatasetInfo.value
except:
for oslot in list(self.outputs.values()):
if oslot.upstream_slot is None:
oslot.meta.NOTREADY = True
return
selection_index = self.InputSelection.value
if not self.Inputs[selection_index].ready():
for oslot in list(self.outputs.values()):
if oslot.upstream_slot is None:
oslot.meta.NOTREADY = True
return
self._opFormattedExport.Input.connect(self.Inputs[selection_index])
dataset_dir = str(rawInfo.default_output_dir)
abs_dataset_dir, _ = getPathVariants(dataset_dir, self.WorkingDirectory.value)
known_keys = {}
known_keys["dataset_dir"] = abs_dataset_dir
nickname = rawInfo.nickname.replace("*", "")
if os.path.pathsep in nickname:
nickname = PathComponents(nickname.split(os.path.pathsep)[0]).fileNameBase
known_keys["nickname"] = nickname
result_types = self.SelectionNames.value
known_keys["result_type"] = result_types[selection_index]
self._opFormattedExport.TransactionSlot.disconnect()
# Blank the internal path while we manipulate the external path
# to avoid invalid intermediate states of ExportPath
self._opFormattedExport.OutputInternalPath.setValue("")
# use partial formatting to fill in non-coordinate name fields
name_format = self.OutputFilenameFormat.value
partially_formatted_name = format_known_keys(name_format, known_keys)
# Convert to absolute path before configuring the internal op
abs_path, _ = getPathVariants(partially_formatted_name, self.WorkingDirectory.value)
self._opFormattedExport.OutputFilenameFormat.setValue(abs_path)
# use partial formatting on the internal dataset name, too
internal_dataset_format = self.OutputInternalPath.value
partially_formatted_dataset_name = format_known_keys(internal_dataset_format, known_keys)
self._opFormattedExport.OutputInternalPath.setValue(partially_formatted_dataset_name)
# Re-connect to finish the 'transaction'
self._opFormattedExport.TransactionSlot.connect(self.TransactionSlot)
def execute(self, slot, subindex, roi, result):
assert False, "Shouldn't get here"
def propagateDirty(self, slot, subindex, roi):
# Out input data changed, so we have work to do when we get executed.
self.Dirty.setValue(True)
def run_export(self):
# If Table-Only is disabled or we're not dirty, we don't have to do anything.
if not self.TableOnly.value and self.Dirty.value:
self._opFormattedExport.run_export()
self.Dirty.setValue(False)
def run_export_to_array(self):
# This function can be used to export the results to an in-memory array, instead of to disk
# (Typically used from pure-python clients in batch mode.)
return self._opFormattedExport.run_export_to_array()
class OpDataExport(Operator):
"""
Top-level operator for the export applet.
Mostly a simple wrapper for OpProcessedDataExport, but with extra formatting of the export path based on lane attributes.
"""
TransactionSlot = InputSlot(
) # To apply all settings in one 'transaction',
# disconnect this slot and reconnect it when all slots are ready
RawData = InputSlot(optional=True) # Display only
FormattedRawData = OutputSlot(
) # OUTPUT - for overlaying the transformed raw data with the export data.
# The dataset info for the original dataset (raw data)
RawDatasetInfo = InputSlot()
WorkingDirectory = InputSlot(
) # Non-absolute paths are relative to this directory. If not provided, paths must be absolute.
Input = InputSlot() # The results slot we want to export
# Subregion params
RegionStart = InputSlot(optional=True)
RegionStop = InputSlot(optional=True)
# Normalization params
InputMin = InputSlot(optional=True)
InputMax = InputSlot(optional=True)
ExportMin = InputSlot(optional=True)
ExportMax = InputSlot(optional=True)
ExportDtype = InputSlot(optional=True)
OutputAxisOrder = InputSlot(optional=True)
# File settings
OutputFilenameFormat = InputSlot(
value='{dataset_dir}/{nickname}_export'
) # A format string allowing {dataset_dir} {nickname}, {roi}, {x_start}, {x_stop}, etc.
OutputInternalPath = InputSlot(value='exported_data')
OutputFormat = InputSlot(value='hdf5')
ExportPath = OutputSlot(
) # Location of the saved file after export is complete.
ConvertedImage = OutputSlot(
) # Cropped image, not yet re-ordered (useful for guis)
ImageToExport = OutputSlot() # The image that will be exported
ImageOnDisk = OutputSlot(
) # This slot reads the exported image from disk (after the export is complete)
Dirty = OutputSlot(
) # Whether or not the result currently matches what's on disk
FormatSelectionIsValid = OutputSlot()
####
# Simplified block diagram for actual export data and 'live preview' display:
#
# --> ExportPath
# /
# Input -> opFormattedExport --> ImageToExport (live preview)
# |\
# \ --> ConvertedImage
# \
# --> FormatSeletionIsValid
####
# Simplified block diagram for Raw data display:
#
# RegionStart --
# \
# RegionStop --> OpRawSubRegionHelper.RawStart -
# / .RawStop --\
# / \
# RawData ---->---------------------------------> opFormatRaw --> FormattedRawData
####
# Simplified block diagram for "on-disk view" of the exported results file:
#
# opFormattedExport.ImageToExport (for metadata only) -->
# \
# opFormattedExport.ExportPath -------------------------> opImageOnDiskProvider --> ImageOnDisk
def __init__(self, *args, **kwargs):
super(OpDataExport, self).__init__(*args, **kwargs)
self._opFormattedExport = OpFormattedDataExport(parent=self)
opFormattedExport = self._opFormattedExport
# Forward almost all inputs to the 'real' exporter
opFormattedExport.TransactionSlot.connect(self.TransactionSlot)
opFormattedExport.Input.connect(self.Input)
opFormattedExport.RegionStart.connect(self.RegionStart)
opFormattedExport.RegionStop.connect(self.RegionStop)
opFormattedExport.InputMin.connect(self.InputMin)
opFormattedExport.InputMax.connect(self.InputMax)
opFormattedExport.ExportMin.connect(self.ExportMin)
opFormattedExport.ExportMax.connect(self.ExportMax)
opFormattedExport.ExportDtype.connect(self.ExportDtype)
opFormattedExport.OutputAxisOrder.connect(self.OutputAxisOrder)
opFormattedExport.OutputFormat.connect(self.OutputFormat)
self.ConvertedImage.connect(opFormattedExport.ConvertedImage)
self.ImageToExport.connect(opFormattedExport.ImageToExport)
self.ExportPath.connect(opFormattedExport.ExportPath)
self.FormatSelectionIsValid.connect(
opFormattedExport.FormatSelectionIsValid)
self.progressSignal = opFormattedExport.progressSignal
self.Dirty.setValue(True) # Default to Dirty
self._opImageOnDiskProvider = None
# We don't export the raw data, but we connect it to it's own op
# so it can be displayed alongside the data to export in the same viewer.
# This keeps axis order, shape, etc. in sync with the displayed export data.
# Note that we must not modify the channels of the raw data, so it gets passed throught a helper.
opHelper = OpRawSubRegionHelper(parent=self)
opHelper.RawImage.connect(self.RawData)
opHelper.ExportStop.connect(self.RegionStart)
opHelper.ExportStop.connect(self.RegionStop)
opFormatRaw = OpFormattedDataExport(parent=self)
opFormatRaw.TransactionSlot.connect(self.TransactionSlot)
opFormatRaw.Input.connect(self.RawData)
opFormatRaw.RegionStart.connect(opHelper.RawStart)
opFormatRaw.RegionStop.connect(opHelper.RawStop)
# Don't normalize the raw data.
#opFormatRaw.InputMin.connect( self.InputMin )
#opFormatRaw.InputMax.connect( self.InputMax )
#opFormatRaw.ExportMin.connect( self.ExportMin )
#opFormatRaw.ExportMax.connect( self.ExportMax )
#opFormatRaw.ExportDtype.connect( self.ExportDtype )
opFormatRaw.OutputAxisOrder.connect(self.OutputAxisOrder)
opFormatRaw.OutputFormat.connect(self.OutputFormat)
self._opFormatRaw = opFormatRaw
self.FormattedRawData.connect(opFormatRaw.ImageToExport)
def cleanupOnDiskView(self):
if self._opImageOnDiskProvider is not None:
self.ImageOnDisk.disconnect()
self._opImageOnDiskProvider.cleanUp()
self._opImageOnDiskProvider = None
def setupOnDiskView(self):
# Set up the output that let's us view the exported file
self._opImageOnDiskProvider = OpImageOnDiskProvider(parent=self)
self._opImageOnDiskProvider.TransactionSlot.connect(
self.TransactionSlot)
self._opImageOnDiskProvider.Input.connect(
self._opFormattedExport.ImageToExport)
self._opImageOnDiskProvider.WorkingDirectory.connect(
self.WorkingDirectory)
self._opImageOnDiskProvider.DatasetPath.connect(
self._opFormattedExport.ExportPath)
self._opImageOnDiskProvider.Dirty.connect(self.Dirty)
self.ImageOnDisk.connect(self._opImageOnDiskProvider.Output)
def setupOutputs(self):
self.cleanupOnDiskView()
# FIXME: If RawData becomes unready() at the same time as RawDatasetInfo(), then
# we have no guarantees about which one will trigger setupOutputs() first.
# It is therefore possible for 'RawDatasetInfo' to appear ready() to us,
# even though it's upstream partner is UNready. We are about to get the
# unready() notification, but it will come too late to prevent our
# setupOutputs method from being called.
# Without proper graph setup transaction semantics, we have to use this
# hack as a workaround.
try:
rawInfo = self.RawDatasetInfo.value
except:
for oslot in self.outputs.values():
if oslot.partner is None:
oslot.meta.NOTREADY = True
return
dataset_dir = PathComponents(rawInfo.filePath).externalDirectory
abs_dataset_dir, _ = getPathVariants(dataset_dir,
self.WorkingDirectory.value)
known_keys = {}
known_keys['dataset_dir'] = abs_dataset_dir
known_keys['nickname'] = rawInfo.nickname
# Disconnect to open the 'transaction'
if self._opImageOnDiskProvider is not None:
self._opImageOnDiskProvider.TransactionSlot.disconnect()
self._opFormattedExport.TransactionSlot.disconnect()
# Blank the internal path while we manipulate the external path
# to avoid invalid intermediate states of ExportPath
self._opFormattedExport.OutputInternalPath.setValue("")
# use partial formatting to fill in non-coordinate name fields
name_format = self.OutputFilenameFormat.value
partially_formatted_name = format_known_keys(name_format, known_keys)
# Convert to absolute path before configuring the internal op
abs_path, _ = getPathVariants(partially_formatted_name,
self.WorkingDirectory.value)
self._opFormattedExport.OutputFilenameFormat.setValue(abs_path)
# use partial formatting on the internal dataset name, too
internal_dataset_format = self.OutputInternalPath.value
partially_formatted_dataset_name = format_known_keys(
internal_dataset_format, known_keys)
self._opFormattedExport.OutputInternalPath.setValue(
partially_formatted_dataset_name)
# Re-connect to finish the 'transaction'
self._opFormattedExport.TransactionSlot.connect(self.TransactionSlot)
if self._opImageOnDiskProvider is not None:
self._opImageOnDiskProvider.TransactionSlot.connect(
self.TransactionSlot)
self.setupOnDiskView()
def execute(self, slot, subindex, roi, result):
assert False, "Shouldn't get here"
def propagateDirty(self, slot, subindex, roi):
# Out input data changed, so we have work to do when we get executed.
self.Dirty.setValue(True)
def run_export(self):
# If we're not dirty, we don't have to do anything.
if self.Dirty.value:
self.cleanupOnDiskView()
self._opFormattedExport.run_export()
self.Dirty.setValue(False)
self.setupOnDiskView()
