def _get_template_dataset_infos(self, input_axes=None):
"""
Sometimes the default settings for an input file are not suitable (e.g. the axistags need to be changed).
We assume the LAST non-batch input in the workflow has settings that will work for all batch processing inputs.
Here, we get the DatasetInfo objects from that lane and store them as 'templates' to modify for all batch-processing files.
"""
template_infos = {}
# If there isn't an available dataset to use as a template
if len(self.dataSelectionApplet.topLevelOperator.DatasetGroup) == 0:
num_roles = len(self.dataSelectionApplet.topLevelOperator.DatasetRoles.value)
for role_index in range(num_roles):
template_infos[role_index] = DatasetInfo()
template_infos[role_index].axistags = vigra.defaultAxistags(input_axes)
return template_infos
# Use the LAST non-batch input file as our 'template' for DatasetInfo settings (e.g. axistags)
template_lane = len(self.dataSelectionApplet.topLevelOperator.DatasetGroup) - 1
opDataSelectionTemplateView = self.dataSelectionApplet.topLevelOperator.getLane(template_lane)
for role_index, info_slot in enumerate(opDataSelectionTemplateView.DatasetGroup):
if info_slot.ready():
template_infos[role_index] = info_slot.value
else:
template_infos[role_index] = DatasetInfo()
if input_axes:
# Support the --input_axes arg to override input axis order, same as DataSelection applet.
template_infos[role_index].axistags = vigra.defaultAxistags(input_axes)
return template_infos
def testBasic3DWrongAxes(self):
"""Test if 3D file with intentionally wrong axes is rejected """
for fileName in self.imgFileNames3D:
graph = lazyflow.graph.Graph()
reader = OperatorWrapper(OpDataSelection, graph=graph)
reader.ProjectFile.setValue(self.projectFile)
reader.WorkingDirectory.setValue(os.getcwd())
reader.ProjectDataGroup.setValue('DataSelection/local_data')
info = DatasetInfo()
# Will be read from the filesystem since the data won't be found in the project file.
info.location = DatasetInfo.Location.ProjectInternal
info.filePath = fileName
info.internalPath = ""
info.invertColors = False
info.convertToGrayscale = False
info.axistags = vigra.defaultAxistags('tzyc')
try:
reader.Dataset.setValues([info])
assert False, "Should have thrown an exception!"
except DatasetConstraintError:
pass
except:
assert False, "Should have thrown a DatasetConstraintError!"
def impl():
projFilePath = self.PROJECT_FILE
shell = self.shell
# New project
shell.createAndLoadNewProject(projFilePath, self.workflowClass())
workflow = shell.projectManager.workflow
from ilastik.applets.dataSelection.opDataSelection import DatasetInfo
opDataSelection = workflow.dataSelectionApplet.topLevelOperator
for i, dataFile in enumerate(self.SAMPLE_DATA):
# Add a file
info = DatasetInfo()
info.filePath = dataFile
info.axistags = vigra.defaultAxistags('xyc')
opDataSelection.DatasetGroup.resize(i + 1)
opDataSelection.DatasetGroup[i][0].setValue(info)
# Set some features
opFeatures = workflow.featureSelectionApplet.topLevelOperator
# sigma: 0.3 0.7 1.0 1.6 3.5 5.0 10.0
selections = numpy.array(
[[True, False, False, False, False, False, False],
[True, False, False, False, False, False, False],
[True, False, False, False, False, False, False],
[False, False, False, False, False, False, False],
[False, False, False, False, False, False, False],
[False, False, False, False, False, False, False]])
opFeatures.SelectionMatrix.setValue(selections)
# Save and close
shell.projectManager.saveProject()
shell.ensureNoCurrentProject(assertClean=True)
def impl():
projFilePath = self.PROJECT_FILE
shell = self.shell
# New project
shell.createAndLoadNewProject(projFilePath, self.workflowClass())
workflow = shell.projectManager.workflow
from ilastik.applets.dataSelection.opDataSelection import DatasetInfo
opDataSelection = workflow.dataSelectionApplet.topLevelOperator
for i, dataFile in enumerate(self.SAMPLE_DATA):
# Add a file
info = DatasetInfo()
info.filePath = dataFile
info.axistags = vigra.defaultAxistags('xyc')
opDataSelection.DatasetGroup.resize(i+1)
opDataSelection.DatasetGroup[i][0].setValue(info)
# Set some features
opFeatures = workflow.featureSelectionApplet.topLevelOperator
# sigma: 0.3 0.7 1.0 1.6 3.5 5.0 10.0
selections = numpy.array( [[True, False, False, False, False, False, False],
[True, False, False, False, False, False, False],
[True, False, False, False, False, False, False],
[False, False, False, False, False, False, False],
[False, False, False, False, False, False, False],
[False, False, False, False, False, False, False]] )
opFeatures.SelectionMatrix.setValue(selections)
# Save and close
shell.projectManager.saveProject()
shell.ensureNoCurrentProject(assertClean=True)
def testWeirdAxisInfos(self):
"""
If we add a dataset that has the channel axis in the wrong place,
the operator should automatically transpose it to be last.
"""
weirdAxisFilename = os.path.join(self.workingDir, 'WeirdAxes.npy')
expected_data = numpy.random.random( (3,100,100) )
numpy.save(weirdAxisFilename, expected_data)
info = DatasetInfo()
info.filePath = weirdAxisFilename
info.axistags = vigra.defaultAxistags('cxy')
graph = Graph()
op = OpDataSelectionGroup(graph=graph, forceAxisOrder=False)
op.WorkingDirectory.setValue( self.workingDir )
op.DatasetRoles.setValue( ['RoleA'] )
op.DatasetGroup.resize( 1 )
op.DatasetGroup[0].setValue( info )
assert op.ImageGroup[0].ready()
data_from_op = op.ImageGroup[0][:].wait()
assert data_from_op.dtype == expected_data.dtype
assert data_from_op.shape == expected_data.shape, (data_from_op.shape, expected_data.shape)
assert (data_from_op == expected_data).all()
# op.Image is a synonym for op.ImageGroup[0]
assert op.Image.ready()
assert (op.Image[:].wait() == expected_data).all()
# Ensure that files opened by the inner operators are closed before we exit.
op.DatasetGroup.resize(0)
def test_fake_data_source(self):
graph = lazyflow.graph.Graph()
reader = OperatorWrapper(OpDataSelection, graph=graph,
operator_kwargs={'forceAxisOrder': False})
reader.ProjectFile.setValue(self.projectFile)
reader.WorkingDirectory.setValue(os.getcwd())
reader.ProjectDataGroup.setValue('DataSelection/local_data')
info = DatasetInfo()
# Will be read from the filesystem since the data won't be found in the project file.
info.location = DatasetInfo.Location.ProjectInternal
info.filePath = self.testRawDataFileName
info.internalPath = ""
info.invertColors = False
info.convertToGrayscale = False
# Use *fake* data source
info.realDataSource = False
info.axistags = vigra.defaultAxistags('tczyx')
info.laneShape = self.imgData.shape
info.laneDtype = self.imgData.dtype
reader.Dataset.setValues([info])
# Verify that now data selection operator returns fake data
# with expected shape and type
imgData = reader.Image[0][...].wait()
assert imgData.shape == self.imgData.shape
assert imgData.dtype == self.imgData.dtype
expected_fake_data = numpy.zeros(info.laneShape, dtype=info.laneDtype)
numpy.testing.assert_array_equal(imgData, expected_fake_data)
def testBasic3DWrongAxes(self):
"""Test if 3D file with intentionally wrong axes is rejected """
for fileName in self.imgFileNames3D:
graph = lazyflow.graph.Graph()
reader = OperatorWrapper(OpDataSelection, graph=graph, operator_kwargs={'forceAxisOrder': False})
reader.ProjectFile.setValue(self.projectFile)
reader.WorkingDirectory.setValue(os.getcwd())
reader.ProjectDataGroup.setValue('DataSelection/local_data')
info = DatasetInfo()
# Will be read from the filesystem since the data won't be found in the project file.
info.location = DatasetInfo.Location.ProjectInternal
info.filePath = fileName
info.internalPath = ""
info.invertColors = False
info.convertToGrayscale = False
info.axistags = vigra.defaultAxistags('tzyc')
try:
reader.Dataset.setValues([info])
assert False, "Should have thrown an exception!"
except DatasetConstraintError:
pass
except:
assert False, "Should have thrown a DatasetConstraintError!"
def _get_template_dataset_infos(self, input_axes=None, sequence_axis=None):
"""
Sometimes the default settings for an input file are not suitable (e.g. the axistags need to be changed).
We assume the LAST non-batch input in the workflow has settings that will work for all batch processing inputs.
Here, we get the DatasetInfo objects from that lane and store them as 'templates' to modify for all batch-
processing files.
"""
template_infos = {}
# If there isn't an available dataset to use as a template
if len(self.dataSelectionApplet.topLevelOperator.DatasetGroup) == 0:
num_roles = len(
self.dataSelectionApplet.topLevelOperator.DatasetRoles.value)
for role_index in range(num_roles):
template_infos[role_index] = DatasetInfo()
if input_axes:
template_infos[
role_index].axistags = vigra.defaultAxistags(
input_axes)
if sequence_axis:
template_infos[role_index].sequenceAxis = sequence_axis
return template_infos
# Use the LAST non-batch input file as our 'template' for DatasetInfo settings (e.g. axistags)
template_lane = len(
self.dataSelectionApplet.topLevelOperator.DatasetGroup) - 1
opDataSelectionTemplateView = self.dataSelectionApplet.topLevelOperator.getLane(
template_lane)
for role_index, info_slot in enumerate(
opDataSelectionTemplateView.DatasetGroup):
if info_slot.ready():
template_infos[role_index] = info_slot.value
else:
template_infos[role_index] = DatasetInfo()
if input_axes:
# Support the --input_axes arg to override input axis order, same as DataSelection applet.
template_infos[role_index].axistags = vigra.defaultAxistags(
input_axes)
if sequence_axis:
template_infos[role_index].sequenceAxis = sequence_axis
return template_infos
def append_lane(workflow, input_filepath, axisorder=None):
# Sanity checks
assert isinstance(workflow, PixelClassificationWorkflow)
opPixelClassification = workflow.pcApplet.topLevelOperator
assert opPixelClassification.Classifier.ready()
# If the filepath is a globstring, convert the stack to h5
input_filepath = DataSelectionApplet.convertStacksToH5( [input_filepath], TMP_DIR )[0]
info = DatasetInfo()
info.location = DatasetInfo.Location.FileSystem
info.filePath = input_filepath
comp = PathComponents(input_filepath)
# Convert all (non-url) paths to absolute
# (otherwise they are relative to the project file, which probably isn't what the user meant)
if not isUrl(input_filepath):
comp.externalPath = os.path.abspath(comp.externalPath)
info.filePath = comp.totalPath()
info.nickname = comp.filenameBase
if axisorder:
info.axistags = vigra.defaultAxistags(axisorder)
logger.debug( "adding lane: {}".format( info ) )
opDataSelection = workflow.dataSelectionApplet.topLevelOperator
# Add a lane
num_lanes = len( opDataSelection.DatasetGroup )+1
logger.debug( "num_lanes: {}".format( num_lanes ) )
opDataSelection.DatasetGroup.resize( num_lanes )
# Configure it.
role_index = 0 # raw data
opDataSelection.DatasetGroup[-1][role_index].setValue( info )
# Sanity check
assert len(opPixelClassification.InputImages) == num_lanes
return opPixelClassification
def append_lane(workflow, input_filepath, axisorder=None):
"""
Add a lane to the project file for the given input file.
If axisorder is given, override the default axisorder for
the file and force the project to use the given one.
Globstrings are supported, in which case the files are converted to HDF5 first.
"""
# If the filepath is a globstring, convert the stack to h5
input_filepath = DataSelectionApplet.convertStacksToH5( [input_filepath], tempfile.mkdtemp() )[0]
info = DatasetInfo()
info.location = DatasetInfo.Location.FileSystem
info.filePath = input_filepath
comp = PathComponents(input_filepath)
# Convert all (non-url) paths to absolute
# (otherwise they are relative to the project file, which probably isn't what the user meant)
if not isUrl(input_filepath):
comp.externalPath = os.path.abspath(comp.externalPath)
info.filePath = comp.totalPath()
info.nickname = comp.filenameBase
if axisorder:
info.axistags = vigra.defaultAxistags(axisorder)
logger.debug( "adding lane: {}".format( info ) )
opDataSelection = workflow.dataSelectionApplet.topLevelOperator
# Add a lane
num_lanes = len( opDataSelection.DatasetGroup )+1
logger.debug( "num_lanes: {}".format( num_lanes ) )
opDataSelection.DatasetGroup.resize( num_lanes )
# Configure it.
role_index = 0 # raw data
opDataSelection.DatasetGroup[-1][role_index].setValue( info )
def testNoChannelAxis(self):
"""
If we add a dataset that is missing a channel axis altogether,
the operator should automatically append a channel axis.
"""
noChannelFilename = os.path.join(self.workingDir, 'NoChannelAxis.npy')
noChannelData = numpy.random.random((100, 100))
numpy.save(noChannelFilename, noChannelData)
info = DatasetInfo()
info.filePath = noChannelFilename
info.axistags = vigra.defaultAxistags('xy')
graph = Graph()
op = OpDataSelectionGroup(graph=graph)
op.WorkingDirectory.setValue(self.workingDir)
op.DatasetRoles.setValue(['RoleA'])
op.DatasetGroup.resize(1)
op.DatasetGroup[0].setValue(info)
assert op.ImageGroup[0].ready()
# Note that we expect a channel axis to be appended to the data.
expected_data = noChannelData[:, :, numpy.newaxis]
data_from_op = op.ImageGroup[0][:].wait()
assert data_from_op.dtype == expected_data.dtype
assert data_from_op.shape == expected_data.shape
assert (data_from_op == expected_data).all()
# op.Image is a synonym for op.ImageGroup[0]
assert op.Image.ready()
assert (op.Image[:].wait() == expected_data).all()
# Ensure that files opened by the inner operators are closed before we exit.
op.DatasetGroup.resize(0)
def testNoChannelAxis(self):
"""
If we add a dataset that is missing a channel axis altogether,
the operator should automatically append a channel axis.
"""
noChannelFilename = os.path.join(self.workingDir, 'NoChannelAxis.npy')
noChannelData = numpy.random.random( (100,100) )
numpy.save(noChannelFilename, noChannelData)
info = DatasetInfo()
info.filePath = noChannelFilename
info.axistags = vigra.defaultAxistags('xy')
graph = Graph()
op = OpDataSelectionGroup( graph=graph )
op.WorkingDirectory.setValue( self.workingDir )
op.DatasetRoles.setValue( ['RoleA'] )
op.DatasetGroup.resize( 1 )
op.DatasetGroup[0].setValue( info )
assert op.ImageGroup[0].ready()
# Note that we expect a channel axis to be appended to the data.
expected_data = noChannelData[:,:,numpy.newaxis]
data_from_op = op.ImageGroup[0][:].wait()
assert data_from_op.dtype == expected_data.dtype
assert data_from_op.shape == expected_data.shape
assert (data_from_op == expected_data).all()
# op.Image is a synonym for op.ImageGroup[0]
assert op.Image.ready()
assert (op.Image[:].wait() == expected_data).all()
# Ensure that files opened by the inner operators are closed before we exit.
op.DatasetGroup.resize(0)
def testWeirdAxisInfos(self):
"""
If we add a dataset that has the channel axis in the wrong place,
the operator should automatically transpose it to be last.
"""
weirdAxisFilename = os.path.join(self.workingDir, 'WeirdAxes.npy')
weirdAxisData = numpy.random.random( (3,100,100) )
numpy.save(weirdAxisFilename, weirdAxisData)
info = DatasetInfo()
info.filePath = weirdAxisFilename
info.axistags = vigra.defaultAxistags('cxy')
graph = Graph()
op = OpDataSelectionGroup( graph=graph )
op.WorkingDirectory.setValue( self.workingDir )
op.DatasetRoles.setValue( ['RoleA'] )
op.DatasetGroup.resize( 1 )
op.DatasetGroup[0].setValue( info )
assert op.ImageGroup[0].ready()
# Note that we expect the channel axis to be transposed to be last.
expected_data = weirdAxisData.transpose( 1,2,0 )
data_from_op = op.ImageGroup[0][:].wait()
assert data_from_op.dtype == expected_data.dtype
assert data_from_op.shape == expected_data.shape
assert (data_from_op == expected_data).all()
# op.Image is a synonym for op.ImageGroup[0]
assert op.Image.ready()
assert (op.Image[:].wait() == expected_data).all()
# Ensure that files opened by the inner operators are closed before we exit.
op.DatasetGroup.resize(0)
