def setupOutputs(self):
self.fileNameList = []
globStrings = self.globstring.value
# Parse list into separate globstrings and combine them
for globString in sorted(globStrings.split("//")):
self.fileNameList += sorted(glob.glob(globString))
num_files = len(self.fileNameList)
if len(self.fileNameList) == 0:
self.stack.meta.NOTREADY = True
return
try:
self.info = vigra.impex.ImageInfo(self.fileNameList[0])
self.slices_per_file = vigra.impex.numberImages(self.fileNameList[0])
except RuntimeError:
raise OpStackLoader.FileOpenError(self.fileNameList[0])
slice_shape = self.info.getShape()
X, Y, C = slice_shape
if self.slices_per_file == 1:
# If this is a stack of 2D images, we assume xy slices stacked along z
Z = num_files
shape = (Z, Y, X, C)
axistags = vigra.defaultAxistags('zyxc')
else:
# If it's a stack of 3D volumes, we assume xyz blocks stacked along t
T = num_files
Z = self.slices_per_file
shape = (T, Z, Y, X, C)
axistags = vigra.defaultAxistags('tzyxc')
self.stack.meta.shape = shape
self.stack.meta.axistags = axistags
self.stack.meta.dtype = self.info.getDtype()
def _get_template_dataset_infos(self, input_axes=None):
"""
Sometimes the default settings for an input file are not suitable (e.g. the axistags need to be changed).
We assume the LAST non-batch input in the workflow has settings that will work for all batch processing inputs.
Here, we get the DatasetInfo objects from that lane and store them as 'templates' to modify for all batch-processing files.
"""
template_infos = {}
# If there isn't an available dataset to use as a template
if len(self.dataSelectionApplet.topLevelOperator.DatasetGroup) == 0:
num_roles = len(self.dataSelectionApplet.topLevelOperator.DatasetRoles.value)
for role_index in range(num_roles):
template_infos[role_index] = DatasetInfo()
template_infos[role_index].axistags = vigra.defaultAxistags(input_axes)
return template_infos
# Use the LAST non-batch input file as our 'template' for DatasetInfo settings (e.g. axistags)
template_lane = len(self.dataSelectionApplet.topLevelOperator.DatasetGroup) - 1
opDataSelectionTemplateView = self.dataSelectionApplet.topLevelOperator.getLane(template_lane)
for role_index, info_slot in enumerate(opDataSelectionTemplateView.DatasetGroup):
if info_slot.ready():
template_infos[role_index] = info_slot.value
else:
template_infos[role_index] = DatasetInfo()
if input_axes:
# Support the --input_axes arg to override input axis order, same as DataSelection applet.
template_infos[role_index].axistags = vigra.defaultAxistags(input_axes)
return template_infos
def setUp(self):
self.data2d = numpy.zeros((3, 3, 3))
self.data2d[:, :, 0] = 1
self.data2d[:, :, 1] = 2
self.data2d[:, :, 2] = 3
self.data2d = self.data2d.view(vigra.VigraArray)
self.data2d.axistags = vigra.defaultAxistags("xyc")
self.data3d = numpy.zeros((3, 3, 3, 3))
self.data3d[:, :, :, 0] = 1
self.data3d[:, :, :, 1] = 2
self.data3d[:, :, :, 2] = 3
self.data3d = self.data3d.view(vigra.VigraArray)
self.data3d.axistags = vigra.defaultAxistags("xyzc")
self.data_bad_channel = numpy.zeros((3, 3, 3, 3))
self.data_bad_channel[:, :, 0, :] = 1
self.data_bad_channel[:, :, 1, :] = 2
self.data_bad_channel[:, :, 2, :] = 3
self.data_bad_channel = self.data_bad_channel.view(vigra.VigraArray)
self.data_bad_channel.axistags = vigra.defaultAxistags("xycz")
def setupOutputs(self):
self.fileNameList = self.expandGlobStrings(self.globstring.value)
num_files = len(self.fileNameList)
if len(self.fileNameList) == 0:
self.stack.meta.NOTREADY = True
return
try:
self.info = vigra.impex.ImageInfo(self.fileNameList[0])
self.slices_per_file = vigra.impex.numberImages(self.fileNameList[0])
except RuntimeError as e:
logger.error(str(e))
raise OpStackLoader.FileOpenError(self.fileNameList[0])
slice_shape = self.info.getShape()
X, Y, C = slice_shape
if self.slices_per_file == 1:
# If this is a stack of 2D images, we assume xy slices stacked along z
Z = num_files
shape = (Z, Y, X, C)
axistags = vigra.defaultAxistags('zyxc')
else:
# If it's a stack of 3D volumes, we assume xyz blocks stacked along t
T = num_files
Z = self.slices_per_file
shape = (T, Z, Y, X, C)
axistags = vigra.defaultAxistags('tzyxc')
self.stack.meta.shape = shape
self.stack.meta.axistags = axistags
self.stack.meta.dtype = self.info.getDtype()
def testChangeBlockshape_masked(self):
logger.info("Generating sample data...")
sampleData = numpy.indices((100, 200, 150), dtype=numpy.float32).sum(0)
sampleData = sampleData.view(numpy.ma.masked_array)
sampleData.set_fill_value(numpy.float32(numpy.nan))
sampleData[0] = numpy.ma.masked
graph = Graph()
opData = OpArrayPiper(graph=graph)
opData.Input.meta.has_mask = True
opData.Input.meta.axistags = vigra.defaultAxistags("xyz")
opData.Input.setValue(sampleData)
op = OpCompressedCache(parent=None, graph=graph)
# logger.debug("Setting block shape...")
op.BlockShape.setValue([100, 75, 50])
op.Input.connect(opData.Output)
assert op.Output.ready()
slicing = numpy.s_[0:100, 50:150, 75:150]
expectedData = sampleData[slicing]
# logger.debug("Requesting data...")
readData = op.Output[slicing].wait()
# logger.debug("Checking data...")
assert (
(readData == expectedData).all()
and (readData.mask == expectedData.mask).all()
and (
(readData.fill_value == expectedData.fill_value)
| (numpy.isnan(readData.fill_value) & numpy.isnan(expectedData.fill_value))
).all()
), "Incorrect output!"
# Now change the blockshape and the input and try again...
sampleDataWithChannel = sampleData[..., None]
opData.Input.meta.axistags = vigra.defaultAxistags("xyzc")
opData.Input.setValue(sampleDataWithChannel)
op.BlockShape.setValue([45, 33, 40, 1])
assert op.Output.ready()
slicing = numpy.s_[60:70, 50:110, 60:120, 0:1]
expectedData = sampleDataWithChannel[slicing]
# logger.debug("Requesting data...")
readData = op.Output[slicing].wait()
# logger.debug("Checking data...")
assert (
(readData == expectedData).all()
and (readData.mask == expectedData.mask).all()
and (
(readData.fill_value == expectedData.fill_value)
| (numpy.isnan(readData.fill_value) & numpy.isnan(expectedData.fill_value))
).all()
), "Incorrect output!"
def setupOutputs(self):
# assert len(self.Input.meta.shape) == 1
self.Output.meta.shape = (self.Input.meta.shape[0], 1)
self.Output.meta.dtype = np.float32
self.Output.meta.axistags = vigra.defaultAxistags('tc')
self.Valid.meta.shape = self.Output.meta.shape[:1]
self.Valid.meta.axistags = vigra.defaultAxistags('t')
self.Valid.meta.dtype = np.uint8
def setUp(self):
if 'TRAVIS' in os.environ:
# This test takes a long time, so skip it on Travis-CI.
import nose
raise nose.SkipTest
self.scaleZ = 2
self.scales = [0.3, 0.7, 1, 1.6, 3.5, 5.0, 10.0]
self.featureIds = [ 'GaussianSmoothing', 'LaplacianOfGaussian',\
'GaussianGradientMagnitude',
'DifferenceOfGaussians',
'StructureTensorEigenvalues',
'HessianOfGaussianEigenvalues' ]
#setup the data
self.nx = 50
self.ny = 50
self.nz = 50
self.data3d = numpy.zeros((self.nx, self.ny, self.nz, 1), dtype=numpy.float32)
for i in range(self.data3d.shape[2]):
self.data3d[:, :, i, 0]=i
newRangeZ = self.scaleZ*(self.nz-1)+1
self.data3dInterp = vigra.sampling.resizeVolumeSplineInterpolation(self.data3d.squeeze(), \
shape=(self.nx, self.ny, newRangeZ))
self.data3dInterp = self.data3dInterp.reshape(self.data3dInterp.shape + (1,))
self.data3d = self.data3d.view(vigra.VigraArray)
self.data3d.axistags = vigra.VigraArray.defaultAxistags(4)
self.data3dInterp = self.data3dInterp.view(vigra.VigraArray)
self.data3dInterp.axistags = vigra.VigraArray.defaultAxistags(4)
self.randomData = (numpy.random.random((self.nx, self.ny, self.nz, 1))).astype(numpy.float32)
self.randomDataInterp = vigra.sampling.resizeVolumeSplineInterpolation(self.randomData.squeeze(), \
shape = (self.nx, self.ny, newRangeZ))
self.randomDataInterp = self.randomDataInterp.reshape(self.randomDataInterp.shape+(1,))
self.randomData = self.randomData.view(vigra.VigraArray).astype(numpy.float32)
self.randomData.axistags = vigra.defaultAxistags(4)
self.randomDataInterp = self.randomDataInterp.view(vigra.VigraArray)
self.randomDataInterp.axistags = vigra.defaultAxistags(4)
#data without channels
self.dataNoChannels = self.randomData.squeeze()
self.dataNoChannels = self.dataNoChannels.view(vigra.VigraArray)
self.dataNoChannels.axistags = vigra.defaultAxistags(3, noChannels=True)
#setup the feature selection
rows = 6
cols = 7
self.selectedFeatures = []
#only test 1 feature 1 sigma setup for now
for i in range(rows):
for j in range(cols):
features = numpy.zeros((rows,cols), dtype=bool)
features[i, j]=True
self.selectedFeatures.append(features)
def setupOutputs(self):
window = self.WindowSize.value
self.Output.meta.shape = (self.Input.meta.shape[0], window)
self.Output.meta.axistags = vigra.defaultAxistags('tc')
self.Output.meta.dtype = self.Input.meta.dtype
self.Valid.meta.shape = (self.Input.meta.shape[0],)
self.Valid.meta.axistags = vigra.defaultAxistags('t')
self.Valid.dtype = np.uint8
def setupOutputs(self):
assert (len(self.Input.meta.shape) <= 2 or
np.prod(self.Input.meta.shape[1:]) == 1)
self.Output.meta.shape = (self.Input.meta.shape[0], 1)
self.Output.meta.axistags = vigra.defaultAxistags('tc')
self.Output.meta.dtype = np.float32
self.Valid.meta.shape = (self.Input.meta.shape[0],)
self.Valid.meta.axistags = vigra.defaultAxistags('t')
self.Valid.meta.dtype = np.uint8
def test_2d_vigra_along_z(self):
"""Test if 2d files generated through vigra are recognized correctly"""
# Prepare some data set for this case
data = numpy.random.randint(0, 255, (20, 100, 200, 3)).astype(numpy.uint8)
axistags = vigra.defaultAxistags("yxc")
expected_axistags = vigra.defaultAxistags("zyxc")
h5_op = OpStreamingH5N5SequenceReaderM(graph=self.graph)
n5_op = OpStreamingH5N5SequenceReaderM(graph=self.graph)
tempdir = tempfile.TemporaryDirectory()
try:
for sliceIndex, zSlice in enumerate(data):
testDataH5FileName = f"{tempdir.name}/test-{sliceIndex:02d}.h5"
testDataN5FileName = f"{tempdir.name}/test-{sliceIndex:02d}.n5"
# Write the dataset to an hdf5 and a n5 file
# (Note: Don't use vigra to do this, which may reorder the axes)
h5File = h5py.File(testDataH5FileName)
n5File = z5py.N5File(testDataN5FileName)
try:
h5File.create_group("volume")
n5File.create_group("volume")
h5File["volume"].create_dataset("subvolume", data=zSlice)
n5File["volume"].create_dataset("subvolume", data=zSlice)
# Write the axistags attribute
current_path = "volume/subvolume"
h5File[current_path].attrs["axistags"] = axistags.toJSON()
n5File[current_path].attrs["axistags"] = axistags.toJSON()
finally:
h5File.close()
n5File.close()
# Read the data with an operator
hdf5GlobString = f"{tempdir.name}/test-*.h5/volume/subvolume"
n5GlobString = f"{tempdir.name}/test-*.n5/volume/subvolume"
h5_op.SequenceAxis.setValue("z")
n5_op.SequenceAxis.setValue("z")
h5_op.GlobString.setValue(hdf5GlobString)
n5_op.GlobString.setValue(n5GlobString)
assert h5_op.OutputImage.ready()
assert n5_op.OutputImage.ready()
assert h5_op.OutputImage.meta.axistags == expected_axistags
assert n5_op.OutputImage.meta.axistags == expected_axistags
numpy.testing.assert_array_equal(
h5_op.OutputImage.value[5:10, 50:100, 100:150], data[5:10, 50:100, 100:150]
)
numpy.testing.assert_array_equal(
n5_op.OutputImage.value[5:10, 50:100, 100:150], data[5:10, 50:100, 100:150]
)
finally:
h5_op.cleanUp()
n5_op.cleanUp()
def test_3d_vigra_along_t(self):
"""Test if 3d volumes generated through vigra are recognized correctly"""
# Prepare some data set for this case
data = numpy.random.randint(0, 255, (10, 15, 50, 100, 3)).astype(numpy.uint8)
axistags = vigra.defaultAxistags("zyxc")
expected_axistags = vigra.defaultAxistags("tzyxc")
h5_op = OpStreamingH5N5SequenceReaderS(graph=self.graph)
n5_op = OpStreamingH5N5SequenceReaderS(graph=self.graph)
try:
testDataH5FileName = f"{self.tempdir_normalized_name}/test.h5"
testDataN5FileName = f"{self.tempdir_normalized_name}/test.n5"
# Write the dataset to an hdf5 file
# (Note: Don't use vigra to do this, which may reorder the axes)
h5File = h5py.File(testDataH5FileName)
n5File = z5py.N5File(testDataN5FileName)
try:
h5File.create_group("volumes")
n5File.create_group("volumes")
internalPathString = "subvolume-{sliceIndex:02d}"
for sliceIndex, tSlice in enumerate(data):
subpath = internalPathString.format(sliceIndex=sliceIndex)
h5File["volumes"].create_dataset(subpath, data=tSlice)
n5File["volumes"].create_dataset(subpath, data=tSlice)
# Write the axistags attribute
current_path = "volumes/{}".format(subpath)
h5File[current_path].attrs["axistags"] = axistags.toJSON()
n5File[current_path].attrs["axistags"] = axistags.toJSON()
finally:
h5File.close()
n5File.close()
# Read the data with an operator
hdf5GlobString = f"{testDataH5FileName}/volumes/subvolume-*"
n5GlobString = f"{testDataN5FileName}/volumes/subvolume-*"
h5_op.SequenceAxis.setValue("t")
n5_op.SequenceAxis.setValue("t")
h5_op.GlobString.setValue(hdf5GlobString)
n5_op.GlobString.setValue(n5GlobString)
assert h5_op.OutputImage.ready()
assert n5_op.OutputImage.ready()
assert h5_op.OutputImage.meta.axistags == expected_axistags
assert n5_op.OutputImage.meta.axistags == expected_axistags
numpy.testing.assert_array_equal(h5_op.OutputImage.value, data)
numpy.testing.assert_array_equal(n5_op.OutputImage.value, data)
finally:
h5_op.cleanUp()
n5_op.cleanUp()
def setupOutputs(self):
self.fileNameList = self.expandGlobStrings(self.globstring.value)
num_files = len(self.fileNameList)
if len(self.fileNameList) == 0:
self.stack.meta.NOTREADY = True
return
try:
self.info = vigra.impex.ImageInfo(self.fileNameList[0])
self.slices_per_file = vigra.impex.numberImages(self.fileNameList[0])
except RuntimeError as e:
logger.error(str(e))
raise OpStackLoader.FileOpenError(self.fileNameList[0]) from e
slice_shape = self.info.getShape()
X, Y, C = slice_shape
if self.slices_per_file == 1:
if self.SequenceAxis.ready():
sequence_axis = str(self.SequenceAxis.value)
assert sequence_axis in "tzc"
else:
sequence_axis = "z"
# For stacks of 2D images, we assume xy slices
if sequence_axis == "c":
shape = (X, Y, C * num_files)
axistags = vigra.defaultAxistags("xyc")
else:
shape = (num_files, Y, X, C)
axistags = vigra.defaultAxistags(sequence_axis + "yxc")
else:
if self.SequenceAxis.ready():
sequence_axis = self.SequenceAxis.value
assert sequence_axis in "tzc"
else:
sequence_axis = "t"
if sequence_axis == "z":
axistags = vigra.defaultAxistags("ztyxc")
elif sequence_axis == "t":
axistags = vigra.defaultAxistags("tzyxc")
else:
axistags = vigra.defaultAxistags("czyx")
# For stacks of 3D volumes, we assume xyz blocks stacked along
# sequence_axis
if sequence_axis == "c":
shape = (num_files * C, self.slices_per_file, Y, X)
else:
shape = (num_files, self.slices_per_file, Y, X, C)
self.stack.meta.shape = shape
self.stack.meta.axistags = axistags
self.stack.meta.dtype = self.info.getDtype()
def setUp(self):
self.delta = numpy.zeros((19, 19, 19, 1), dtype=numpy.float32)
self.delta[9, 9, 9, 0]=1
self.delta = self.delta.view(vigra.VigraArray)
self.delta.axistags = vigra.defaultAxistags(4)
self.dataShape = ((100, 100, 100, 1))
self.randomData = (numpy.random.random(self.dataShape) * 100).astype(int)
self.randomData = self.randomData.view(vigra.VigraArray)
self.randomData.axistags = vigra.defaultAxistags(4)
self.anisotropicSigmas = [(3, 3, 1), (1.6, 1.6, 1)]
self.isotropicSigmasTuple = [(3, 3, 3), (1, 1, 1)]
self.isotropicSigmas = [3, 1]
def test1(self):
superpixels = generate_random_voronoi((100,200), 200)
superpixels.axistags = vigra.defaultAxistags('yx')
feature_names = ['edgeregion_edge_regionradii']
rag = Rag( superpixels )
acc = EdgeRegionEdgeAccumulator(rag, feature_names)
features_df = rag.compute_features(None, feature_names, accumulator_set=[acc])
radii = features_df[features_df.columns.values[2:]].values
assert (radii[:,0] >= radii[:,1]).all()
# Transpose superpixels and check again
# Should match (radii are sorted by magnitude).
superpixels.axistags = vigra.defaultAxistags('xy')
rag = Rag( superpixels )
acc = EdgeRegionEdgeAccumulator(rag, feature_names)
transposed_features_df = rag.compute_features(None, feature_names, accumulator_set=[acc])
transposed_radii = transposed_features_df[transposed_features_df.columns.values[2:]].values
assert (transposed_features_df[['sp1', 'sp1']].values == features_df[['sp1', 'sp1']].values).all()
DEBUG = False
if DEBUG:
count_features = rag.compute_features(None, ['standard_edge_count', 'standard_sp_count'])
import pandas as pd
combined_features_df = pd.merge(features_df, transposed_features_df, how='left', on=['sp1', 'sp2'], suffixes=('_orig', '_transposed'))
combined_features_df = pd.merge(combined_features_df, count_features, how='left', on=['sp1', 'sp2'])
problem_rows = np.logical_or(np.isclose(radii[:, 0], transposed_radii[:, 0]) != 1,
np.isclose(radii[:, 1], transposed_radii[:, 1]) != 1)
problem_df = combined_features_df.loc[problem_rows][sorted(list(combined_features_df.columns))]
print(problem_df.transpose())
debug_sp = np.zeros_like(superpixels, dtype=np.uint8)
for sp1 in problem_df['sp1'].values:
debug_sp[superpixels == sp1] = 128
for sp2 in problem_df['sp2'].values:
debug_sp[superpixels == sp2] = 255
vigra.impex.writeImage(debug_sp, '/tmp/debug_sp.png', dtype='NATIVE')
# The first axes should all be close.
# The second axes may differ somewhat in the case of purely linear edges,
# so we allow a higher tolerance.
assert np.isclose(radii[:,0], transposed_radii[:,0]).all()
assert np.isclose(radii[:,1], transposed_radii[:,1], atol=0.001).all()
def setUp(self):
segimg = segImage()
rawimg = np.indices(segimg.shape).sum(0).astype(np.float32)
rawimg = rawimg.view(vigra.VigraArray)
rawimg.axistags = vigra.defaultAxistags('txyzc')
feats = {"Vigra Object Features":
{"Count":{}, "RegionCenter":{}, "Coord<Principal<Kurtosis>>":{}, "Coord<Minimum>":{}, "Coord<Maximum>":{}}
}
g = Graph()
self.featsop = OpRegionFeatures(graph=g)
self.featsop.LabelImage.setValue(segimg)
self.featsop.RawImage.setValue( rawimg )
self.featsop.Features.setValue(feats)
self.assertTrue(self.featsop.Output.ready(), "The output of operator {} was not ready after connections took place.".format(self.featsop))
self._opRegFeatsAdaptOutput = OpAdaptTimeListRoi(graph=g)
self._opRegFeatsAdaptOutput.Input.connect(self.featsop.Output)
self.assertTrue(self._opRegFeatsAdaptOutput.Output.ready(), "The output of operator {} was not ready after connections took place.".format(self._opRegFeatsAdaptOutput))
self.op = OpObjectTrain(graph=g)
self.op.Features.resize(1)
self.op.Features[0].connect(self._opRegFeatsAdaptOutput.Output)
self.op.SelectedFeatures.setValue(feats)
self.op.FixClassifier.setValue(False)
self.op.ForestCount.setValue(self.nRandomForests)
def setupOutputs(self):
numImages = len(self.LabelInputs)
self.PredictionsFromDisk.resize( numImages )
self.NonzeroLabelBlocks.resize( numImages )
self.LabelImages.resize( numImages )
self.PredictionProbabilities.resize( numImages )
self.opClassifier.Images.resize( numImages )
for i in range(numImages):
self._data.append( numpy.zeros(self.dataShape) )
self.NonzeroLabelBlocks[i].meta.shape = (1,)
self.NonzeroLabelBlocks[i].meta.dtype = object
self.LabelImages[i].meta.shape = self.dataShape
self.LabelImages[i].meta.dtype = numpy.float64
# Hard-coded: Two prediction classes
self.PredictionProbabilities[i].meta.shape = self.prediction_shape
self.PredictionProbabilities[i].meta.dtype = numpy.float64
self.PredictionProbabilities[i].meta.axistags = vigra.defaultAxistags('txyzc')
# Classify with random data
self.opClassifier.Images[i].setValue( numpy.random.random(self.dataShape) )
self.Classifier.connect( self.opClassifier.Classifier )
def testPartialAllocate():
nx = 15
ny = 20
nz = 17
nc = 7
stack = vigra.VigraArray((nx, ny, nz, nc), axistags=vigra.defaultAxistags('xyzc'))
stack[...] = numpy.random.rand(nx, ny, nz, nc)
g = Graph()
#assume that the slicer works
slicerX = OpMultiArraySlicer(graph=g)
slicerX.inputs["Input"].setValue(stack)
slicerX.inputs["AxisFlag"].setValue('x')
#insert the x dimension
stackerX = OpMultiArrayStacker(graph=g)
stackerX.inputs["AxisFlag"].setValue('x')
stackerX.inputs["AxisIndex"].setValue(0)
stackerX.inputs["Images"].connect(slicerX.outputs["Slices"])
key = (slice(2, 3, None), slice(15, 18, None), slice(12, 15, None), slice(0, 7, None))
newdata = stackerX.outputs["Output"][key].wait()
substack = stack[key]
print newdata.shape, substack.shape
assert_array_equal(newdata, substack.view(numpy.ndarray))
def setup(self):
graph = Graph()
op = OpCompressedUserLabelArray(graph=graph)
arrayshape = (1,100,100,10,1)
op.inputs["shape"].setValue( arrayshape )
blockshape = (1,10,10,10,1) # Why doesn't this work if blockshape is an ndarray?
op.inputs["blockShape"].setValue( blockshape )
op.eraser.setValue(100)
op.Input.meta.axistags = vigra.defaultAxistags('txyzc')
op.Input.meta.has_mask = True
dummyData = numpy.zeros(arrayshape, dtype=numpy.uint8)
dummyData = numpy.ma.masked_array(dummyData, mask=numpy.ma.getmaskarray(dummyData), fill_value=numpy.uint8(0), shrink=False)
op.Input.setValue( dummyData )
slicing = sl[0:1, 1:15, 2:36, 3:7, 0:1]
inDataShape = slicing2shape(slicing)
inputData = ( 3*numpy.random.random(inDataShape) ).astype(numpy.uint8)
inputData = numpy.ma.masked_array(inputData, mask=numpy.ma.getmaskarray(inputData), fill_value=numpy.uint8(0), shrink=False)
inputData[:, 0] = numpy.ma.masked
op.Input[slicing] = inputData
data = numpy.ma.zeros(arrayshape, dtype=numpy.uint8, fill_value=numpy.uint8(0))
data[slicing] = inputData
self.op = op
self.slicing = slicing
self.inData = inputData
self.data = data
def create_test_files():
tags = vigra.defaultAxistags("zyxc")
tags['x'].resolution = 1.0
tags['y'].resolution = 1.0
tags['z'].resolution = 45.0
tags['c'].description = 'intensity'
with h5py.File(test_data_path, 'w') as f:
f['zeros'] = numpy.zeros( (10, 100, 200, 1), dtype=numpy.uint8 )
f['zeros'].attrs['axistags'] = tags.toJSON()
import ilastik_main
parsed_args, workflow_cmdline_args = ilastik_main.parser.parse_known_args()
parsed_args.new_project = test_project_path
parsed_args.workflow = "Pixel Classification"
parsed_args.headless = True
shell = ilastik_main.main(parsed_args, workflow_cmdline_args)
data_selection_applet = shell.workflow.dataSelectionApplet
# To configure data selection, start with empty cmdline args and manually fill them in
data_selection_args, _ = data_selection_applet.parse_known_cmdline_args([])
data_selection_args.raw_data = [test_data_path + '/zeros']
# Configure
data_selection_applet.configure_operator_with_parsed_args(data_selection_args)
shell.projectManager.saveProject()
return data_selection_applet
def testBasic_Hdf5(self):
data = numpy.random.random( (100,100) ).astype( numpy.float32 )
data = vigra.taggedView( data, vigra.defaultAxistags('xy') )
graph = Graph()
opExport = OpExportSlot(graph=graph)
opExport.Input.setValue(data)
opExport.OutputFormat.setValue( 'hdf5' )
opExport.OutputFilenameFormat.setValue( self._tmpdir + '/test_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}' )
opExport.OutputInternalPath.setValue('volume/data')
opExport.CoordinateOffset.setValue( (10, 20) )
assert opExport.ExportPath.ready()
export_file = PathComponents( opExport.ExportPath.value ).externalPath
assert os.path.split(export_file)[1] == 'test_export_x10-110_y20-120.h5'
#print "exporting data to: {}".format( opExport.ExportPath.value )
opExport.run_export()
opRead = OpInputDataReader( graph=graph )
opRead.FilePath.setValue( opExport.ExportPath.value )
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data).all(), "Read data didn't match exported data!"
opRead.cleanUp()
def setupOutputs(self):
inputAxistags = self.inputs["input"].meta.axistags
inputShape = list(self.inputs["input"].meta.shape)
self.resSl = [slice(0, stop, None) for stop in list(self.inputs["input"].meta.shape)]
if self.order.ready():
self._axisorder = self.order.value
outputTags = vigra.defaultAxistags(self._axisorder)
inputKeys = set(tag.key for tag in inputAxistags)
for outputTag in outputTags:
if outputTag.key not in inputKeys:
# inputAxistags.insert(outputTags.index(tag.key),tag)
# inputShape.insert(outputTags.index(tag.key),1)
self.resSl.insert(outputTags.index(outputTag.key), 0)
outputShape = []
for tag in outputTags:
if tag in inputAxistags:
outputShape += [inputShape[inputAxistags.index(tag.key)]]
else:
outputShape += [1]
self.output.meta.assignFrom(self.input.meta)
self.outputs["output"].meta.dtype = self.inputs["input"].meta.dtype
self.outputs["output"].meta.shape = tuple(outputShape)
self.outputs["output"].meta.axistags = outputTags
if self.output.meta.original_axistags is None:
self.output.meta.original_axistags = copy.copy(inputAxistags)
self.output.meta.original_shape = self.input.meta.shape
assert len(inputAxistags) == len(self.input.meta.shape)
def _write_slice(self, roi, slice_data):
"""
Write the data from the given roi into a slice image.
"""
step_axis = self._volume_axes[0]
input_axes = self.Input.meta.getAxisKeys()
tagged_roi = OrderedDict( zip( input_axes, zip( *roi ) ) )
# e.g. tagged_roi={ 'x':(0,1), 'y':(3,4), 'z':(10,20) }
assert tagged_roi[step_axis][1] - tagged_roi[step_axis][0] == 1,\
"Expected roi to be a single slice."
slice_index = tagged_roi[step_axis][0] + self.SliceIndexOffset.value
filepattern = self.FilepathPattern.value
# If the user didn't provide custom formatting for the slice field,
# auto-format to include zero-padding
if '{slice_index}' in filepattern:
filepattern = filepattern.format( slice_index='{' + 'slice_index:0{}'.format(self._max_slice_digits) + '}' )
formatted_path = filepattern.format( slice_index=slice_index )
squeezed_data = slice_data.squeeze()
squeezed_data = vigra.taggedView(squeezed_data, vigra.defaultAxistags("".join(self._volume_axes[1:])))
assert len(squeezed_data.shape) == len(self._volume_axes)-1
#logger.debug( "Writing slice image for roi: {}".format( roi ) )
logger.debug("Writing slice: {}".format(formatted_path) )
vigra.impex.writeImage( squeezed_data, formatted_path )
def testBasic_2d_Sequence(self):
data = 255 * numpy.random.random((10, 50, 100, 3))
data = data.astype(numpy.uint8)
data = vigra.taggedView(data, vigra.defaultAxistags("zyxc"))
# Must run this through an operator
# Can't use opExport.setValue() because because OpStackWriter can't work with ValueRequests
graph = Graph()
opData = OpBlockedArrayCache(graph=graph)
opData.BlockShape.setValue(data.shape)
opData.Input.setValue(data)
filepattern = self._tmpdir + "/test_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}_z{slice_index}"
opExport = OpExportSlot(graph=graph)
opExport.Input.connect(opData.Output)
opExport.OutputFormat.setValue("png sequence")
opExport.OutputFilenameFormat.setValue(filepattern)
opExport.CoordinateOffset.setValue((10, 20, 30, 0))
opExport.run_export()
export_pattern = opExport.ExportPath.value
globstring = export_pattern.format(slice_index=999)
globstring = globstring.replace("999", "*")
opReader = OpStackLoader(graph=graph)
try:
opReader.globstring.setValue(globstring)
assert opReader.stack.meta.shape == data.shape, "Exported files were of the wrong shape or number."
assert (opReader.stack[:].wait() == data.view(numpy.ndarray)).all(), "Exported data was not correct"
finally:
opReader.cleanUp()
def _createDatasetInFile(self, hdf5File, datasetName, roi):
shape = tuple(roi[1] - roi[0])
chunks = self._description.chunks
if chunks is not None:
# chunks must not be bigger than the data in any dim
chunks = numpy.minimum(chunks, shape)
chunks = tuple(chunks)
compression = self._description.compression
compression_opts = self._description.compression_opts
dtype = self._description.dtype
if dtype == object:
dtype = h5py.special_dtype(vlen=numpy.uint8)
dataset = hdf5File.create_dataset(
datasetName,
shape=shape,
dtype=dtype,
chunks=chunks,
compression=compression,
compression_opts=compression_opts,
)
# Set data attributes
if self._description.drange is not None:
dataset.attrs["drange"] = self._description.drange
if _use_vigra:
dataset.attrs["axistags"] = vigra.defaultAxistags(str(self._description.axes)).toJSON()
def setUp(self):
segimg = segImage()
binimg = (segimg > 0).astype(np.uint8)
labels = {0: np.array([0, 1, 2]), 1: np.array([0, 1, 1, 2])}
rawimg = np.indices(segimg.shape).sum(0).astype(np.float32)
rawimg = rawimg.view(vigra.VigraArray)
rawimg.axistags = vigra.defaultAxistags("txyzc")
g = Graph()
objectExtraction.config.vigra_features = ["Count", "Mean", "Variance", "Skewness"]
objectExtraction.config.selected_features = ["Count", "Mean", "Mean_excl", "Variance"]
self.extrOp = OpObjectExtraction(graph=g)
self.extrOp.BinaryImage.setValue(binimg)
self.extrOp.RawImage.setValue(rawimg)
assert self.extrOp.RegionFeatures.ready()
self.classOp = OpObjectClassification(graph=g)
self.classOp.BinaryImages.resize(1)
self.classOp.BinaryImages.setValues([binimg])
self.classOp.SegmentationImages.resize(1)
self.classOp.SegmentationImages.setValues([segimg])
self.classOp.RawImages.resize(1)
self.classOp.RawImages.setValues([rawimg])
self.classOp.LabelInputs.resize(1)
self.classOp.LabelInputs.setValues([labels])
self.classOp.LabelsAllowedFlags.resize(1)
self.classOp.LabelsAllowedFlags.setValues([True])
self.classOp.ObjectFeatures.connect(self.extrOp.RegionFeatures)
def setUp(self):
segimg = segImage()
binimg = (segimg > 0).astype(np.uint8)
labels = {0: np.array([0, 1, 2]), 1: np.array([0, 1, 1, 2])}
rawimg = np.indices(segimg.shape).sum(0).astype(np.float32)
rawimg = rawimg.view(vigra.VigraArray)
rawimg.axistags = vigra.defaultAxistags("txyzc")
g = Graph()
objectExtraction.config.vigra_features = ["Count", "Mean", "Variance", "Skewness"]
objectExtraction.config.other_features = []
objectExtraction.config.selected_features = ["Count", "Mean", "Mean_excl", "Variance"]
self.extrOp = OpObjectExtraction(graph=g)
self.extrOp.BinaryImage.setValue(binimg)
self.extrOp.RawImage.setValue(rawimg)
assert self.extrOp.RegionFeatures.ready()
self.trainop = OpObjectTrain(graph=g)
self.trainop.Features.resize(1)
self.trainop.Features.connect(self.extrOp.RegionFeatures)
self.trainop.Labels.resize(1)
self.trainop.Labels.setValues([labels])
self.trainop.FixClassifier.setValue(False)
self.trainop.ForestCount.setValue(1)
assert self.trainop.Classifier.ready()
def testBasic_Npy(self):
data = numpy.random.random((100, 100)).astype(numpy.float32)
data = vigra.taggedView(data, vigra.defaultAxistags("xy"))
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue(data)
opExport = OpExportSlot(graph=graph)
opExport.Input.connect(opPiper.Output)
opExport.OutputFormat.setValue("numpy")
opExport.OutputFilenameFormat.setValue(self._tmpdir + "/test_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}")
opExport.CoordinateOffset.setValue((10, 20))
assert opExport.ExportPath.ready()
assert os.path.split(opExport.ExportPath.value)[1] == "test_export_x10-110_y20-120.npy"
# print "exporting data to: {}".format( opExport.ExportPath.value )
opExport.run_export()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(opExport.ExportPath.value)
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def setUp(self):
segimg = segImage()
labels = {0: np.array([0, 1, 2]), 1: np.array([0, 0, 0, 0])}
rawimg = np.indices(segimg.shape).sum(0).astype(np.float32)
rawimg = rawimg.view(vigra.VigraArray)
rawimg.axistags = vigra.defaultAxistags("txyzc")
g = Graph()
objectExtraction.config.selected_features = ["Count"]
self.featsop = OpRegionFeatures(FEATURES, graph=g)
self.featsop.LabelImage.setValue(segimg)
self.featsop.RawImage.setValue(rawimg)
assert self.featsop.Output.ready()
self._opRegFeatsAdaptOutput = OpAdaptTimeListRoi(graph=g)
self._opRegFeatsAdaptOutput.Input.connect(self.featsop.Output)
assert self._opRegFeatsAdaptOutput.Output.ready()
self.trainop = OpObjectTrain(graph=g)
self.trainop.Features.resize(1)
self.trainop.Features[0].connect(self._opRegFeatsAdaptOutput.Output)
self.trainop.Labels.resize(1)
self.trainop.Labels.setValues([labels])
self.trainop.FixClassifier.setValue(False)
self.trainop.ForestCount.setValue(1)
assert self.trainop.Classifier.ready()
self.op = OpObjectPredict(graph=g)
self.op.Classifier.connect(self.trainop.Classifier)
self.op.Features.connect(self._opRegFeatsAdaptOutput.Output)
self.op.LabelsCount.setValue(2)
assert self.op.Predictions.ready()
def setup(self):
graph = Graph()
op = OpCompressedUserLabelArray(graph=graph)
arrayshape = (1,100,100,10,1)
op.inputs["shape"].setValue( arrayshape )
blockshape = (1,10,10,10,1) # Why doesn't this work if blockshape is an ndarray?
op.inputs["blockShape"].setValue( blockshape )
op.eraser.setValue(100)
dummyData = vigra.VigraArray(arrayshape, axistags=vigra.defaultAxistags('txyzc'))
op.Input.setValue( dummyData )
slicing = sl[0:1, 1:15, 2:36, 3:7, 0:1]
inDataShape = slicing2shape(slicing)
inputData = ( 3*numpy.random.random(inDataShape) ).astype(numpy.uint8)
op.Input[slicing] = inputData
data = numpy.zeros(arrayshape, dtype=numpy.uint8)
data[slicing] = inputData
# Sanity check...
assert (op.Output[:].wait()[slicing] == data[slicing]).all()
self.op = op
self.slicing = slicing
self.inData = inputData
self.data = data
def test_basic(self):
graph = lazyflow.graph.Graph()
data = numpy.indices( (100,100), dtype=numpy.uint8 ).sum(0)
data = vigra.taggedView( data, vigra.defaultAxistags('xy') )
opDataProvider = OpArrayCache( graph=graph )
opDataProvider.Input.setValue( data )
op = OpZeroDefault( graph=graph )
op.MetaInput.setValue( data )
# Zero by default
output_data = op.Output[:].wait()
assert (output_data == 0).all()
# Connecting a real input triggers dirty notification
dirty_notification_count = [0]
def handleDirty(*args):
dirty_notification_count[0] += 1
op.Output.notifyDirty( handleDirty )
op.Input.connect( opDataProvider.Output )
assert dirty_notification_count[0] == 1
# Output should provide real data if available
assert ( op.Output[:].wait() == data.view( numpy.ndarray ) ).all()
# Output provides zeros again when the data is no longer available
op.Input.disconnect()
output_data = op.Output[:].wait()
assert (output_data == 0).all()
def testPatchDetection(self):
vol = vigra.taggedView(np.ones((5,5), dtype=np.uint8)*128, axistags=vigra.defaultAxistags('xy'))
vol[2:5,2:5] = 0
expected = np.zeros((5,5))
expected[3:5,3:5] = 1
self.op.PatchSize.setValue(2)
self.op.HaloSize.setValue(1)
self.op.DetectionMethod.setValue('classic')
self.op.InputVolume.setValue(vol)
out = self.op.Output[:].wait()
assert_array_equal(expected[3:5,3:5], out[3:5,3:5])
def handle_updated_axes():
# The user is specifying a new interpretation of the file's axes
updated_axisorder = str(settingsDlg.axesEdit.text())
metadata = opMetadataInjector.Metadata.value.copy()
metadata.axistags = vigra.defaultAxistags(updated_axisorder)
opMetadataInjector.Metadata.setValue(metadata)
if opReorderAxes._invalid_axes:
settingsDlg.buttonBox.button(
QDialogButtonBox.Ok).setEnabled(False)
# Red background
settingsDlg.axesEdit.setStyleSheet(
"QLineEdit { background: rgb(255, 128, 128); selection-background-color: rgb(128, 128, 255); }"
)
def testInvalidDtype(self):
data = 255 * numpy.random.random((50, 100))
data = data.astype(numpy.uint32)
data = vigra.taggedView(data, vigra.defaultAxistags('xy'))
graph = Graph()
opExport = OpExportSlot(graph=graph)
opExport.Input.setValue(data)
opExport.OutputFilenameFormat.setValue(self._tmpdir + '/test_export')
for fmt in ('jpg', 'png', 'pnm', 'bmp'):
opExport.OutputFormat.setValue(fmt)
msg = opExport.FormatSelectionErrorMsg.value
assert msg, "{} supported although it is actually not".format(fmt)
def testMultiThread_masked(self):
logger.info("Generating sample data...")
sampleData = numpy.indices((3, 100, 200, 150, 2),
dtype=numpy.float32).sum(0)
sampleData = sampleData.view(numpy.ma.masked_array)
sampleData.set_fill_value(numpy.float32(numpy.nan))
sampleData[0] = numpy.ma.masked
graph = Graph()
opData = OpArrayPiper(graph=graph)
opData.Input.meta.has_mask = True
opData.Input.meta.axistags = vigra.defaultAxistags("txyzc")
opData.Input.setValue(sampleData)
op = OpCompressedCache(parent=None, graph=graph)
# logger.debug("Setting block shape...")
op.BlockShape.setValue([1, 100, 75, 50, 2])
op.Input.connect(opData.Output)
assert op.Output.ready()
slicing = numpy.s_[0:2, 0:100, 50:150, 75:150, 0:1]
expectedData = sampleData[slicing]
results = {}
def readData(resultIndex):
results[resultIndex] = op.Output[slicing].wait()
threads = []
for i in range(10):
threads.append(
threading.Thread(target=functools.partial(readData, i)))
for th in threads:
th.start()
for th in threads:
th.join()
assert len(results) == len(threads), "Didn't get all results."
# logger.debug("Checking data...")
for i, data in list(results.items()):
assert ((data == expectedData).all()
and (data.mask == expectedData.mask).all()
and ((data.fill_value == expectedData.fill_value)
| (numpy.isnan(data.fill_value)
& numpy.isnan(expectedData.fill_value))).all()
), "Incorrect output for index {}".format(i)
def deserializeFromHdf5(self, hdf5File, projectFilePath, headless=False):
# Check the overall file version
ilastikVersion = hdf5File["ilastikVersion"][()]
# This is the v0.5 import deserializer. Don't work with 0.6 projects (or anything else).
if ilastikVersion != 0.5:
return
# The 'working directory' for the purpose of constructing absolute
# paths from relative paths is the project file's directory.
projectDir = os.path.split(projectFilePath)[0]
self.topLevelOperator.WorkingDirectory.setValue(projectDir)
# Access the top group and the info group
try:
# dataset = hdf5File["DataSets"]["dataItem00"]["data"]
dataDir = hdf5File["DataSets"]
except KeyError:
# If our group (or subgroup) doesn't exist, then make sure the operator is empty
self.topLevelOperator.DatasetGroup.resize(0)
return
self.topLevelOperator.DatasetGroup.resize(len(dataDir))
for index, (datasetDirName,
datasetDir) in enumerate(sorted(dataDir.items())):
# Some older versions of ilastik 0.5 stored the data in tzyxc order.
# Some power-users can enable a command-line flag that tells us to
# transpose the data back to txyzc order when we import the old project.
default_axis_order = ilastik.utility.globals.ImportOptions.default_axis_order
if default_axis_order is not None:
import warnings
# todo:axisorder: this will apply for other old ilastik projects as well... adapt the formulation.
warnings.warn(
"Using a strange axis order to import ilastik 0.5 projects: {}"
.format(default_axis_order))
datasetInfo.axistags = vigra.defaultAxistags(
default_axis_order)
# We'll set up the link to the dataset in the old project file,
# but we'll set the location to ProjectInternal so that it will
# be copied to the new file when the project is saved.
datasetInfo = ProjectInternalDatasetInfo(
inner_path=str(projectFilePath + "/DataSets/" +
datasetDirName + "/data"),
nickname=f"{datasetDirName} (imported from v0.5)",
)
# Give the new info to the operator
self.topLevelOperator.DatasetGroup[index][0].setValue(datasetInfo)
def _get_template_dataset_infos(self, input_axes=None):
"""
Sometimes the default settings for an input file are not suitable (e.g. the axistags need to be changed).
We assume the LAST non-batch input in the workflow has settings that will work for all batch processing inputs.
Here, we get the DatasetInfo objects from that lane and store them as 'templates' to modify for all batch-processing files.
"""
template_infos = {}
# If there isn't an available dataset to use as a template
if len(self.dataSelectionApplet.topLevelOperator.DatasetGroup) == 0:
num_roles = len(
self.dataSelectionApplet.topLevelOperator.DatasetRoles.value)
for role_index in range(num_roles):
template_infos[role_index] = DatasetInfo()
if input_axes:
template_infos[
role_index].axistags = vigra.defaultAxistags(
input_axes)
return template_infos
# Use the LAST non-batch input file as our 'template' for DatasetInfo settings (e.g. axistags)
template_lane = len(
self.dataSelectionApplet.topLevelOperator.DatasetGroup) - 1
opDataSelectionTemplateView = self.dataSelectionApplet.topLevelOperator.getLane(
template_lane)
for role_index, info_slot in enumerate(
opDataSelectionTemplateView.DatasetGroup):
if info_slot.ready():
template_infos[role_index] = info_slot.value
else:
template_infos[role_index] = DatasetInfo()
if input_axes:
# Support the --input_axes arg to override input axis order, same as DataSelection applet.
template_infos[role_index].axistags = vigra.defaultAxistags(
input_axes)
return template_infos
def testIntegerRange(self):
"""
test if the output is in the right integer range
in particular, too large values should be set to max and too small
values to min
"""
v = np.zeros((1, 1, 5), dtype=np.uint8)
v[0, 0, :] = [220, 255, 0, 255, 220]
v = vigra.VigraArray(v, axistags=vigra.defaultAxistags("xyz"), dtype=np.uint8)
m = vigra.VigraArray(v, axistags=vigra.defaultAxistags("xyz"), dtype=np.uint8)
m[:] = 0
m[0, 0, 2] = 1
for interpolationMethod in ["cubic"]:
self.op.InputVolume.setValue(v)
self.op.Missing.setValue(m)
self.op.InterpolationMethod.setValue(interpolationMethod)
self.op.InputVolume.setValue(v)
out = self.op.Output[:].wait().view(np.ndarray)
# natural comparison
self.assertEqual(out[0, 0, 2], 255)
v = np.zeros((1, 1, 5), dtype=np.uint8)
v[0, 0, :] = [220, 255, 0, 255, 220]
v = 255 - vigra.VigraArray(v, axistags=vigra.defaultAxistags("xyz"), dtype=np.uint8)
m = vigra.VigraArray(v, axistags=vigra.defaultAxistags("xyz"), dtype=np.uint8)
m[:] = 0
m[0, 0, 2] = 1
for interpolationMethod in ["cubic"]:
self.op.InputVolume.setValue(v)
self.op.Missing.setValue(m)
self.op.InterpolationMethod.setValue(interpolationMethod)
self.op.InputVolume.setValue(v)
out = self.op.Output[:].wait().view(np.ndarray)
# natural comparison
self.assertEqual(out[0, 0, 2], 0)
def export_from_tiled_volume(tiles_description_json_path, roi,
output_hdf5_path, output_dataset_name):
"""
Export a cutout volume from a TiledVolume into an hdf5 dataset.
Args:
tiles_description_json_path: path to the TiledVolume's json description file.
roi: The (start, stop) corners of the cutout region to export. (Must be tuple-of-tuples.)
output_hdf5_path: The HDF5 file to export to.
output_dataset_name: The name of the HDF5 dataset to write. Will be deleted first if necessary.
"""
if not os.path.exists(tiles_description_json_path):
raise Exception("Description file does not exist: " +
tiles_description_json_path)
start, stop = numpy.array(roi)
shape = tuple(stop - start)
tiled_volume = TiledVolume(tiles_description_json_path)
with Timer() as timer:
result_array = numpy.ndarray(shape, tiled_volume.description.dtype)
logger.info("Reading cutout volume of shape: {}".format(shape))
tiled_volume.read((start, stop), result_out=result_array)
logger.info("Writing data to: {}/{}".format(output_hdf5_path,
output_dataset_name))
with h5py.File(output_hdf5_path, 'a') as output_h5_file:
if output_dataset_name in output_h5_file:
del output_h5_file[output_dataset_name]
dset = output_h5_file.create_dataset(output_dataset_name,
shape,
result_array.dtype,
chunks=True,
data=result_array)
try:
import vigra
except ImportError:
pass
else:
# Attach axistags to the exported dataset, so ilastik
# automatically interprets the volume correctly.
output_axes = tiled_volume.description.output_axes
dset.attrs['axistags'] = vigra.defaultAxistags(
output_axes).toJSON()
logger.info("Exported {:.1e} pixels in {:.1f} seconds.".format(
numpy.prod(shape), timer.seconds()))
def setupOutputs(self):
"""
Load the file specified via our input slot and present its data on the output slot.
"""
if self._memmapFile is not None:
self._memmapFile.close()
fileName = self.FileName.value
try:
# Load the file in read-only "memmap" mode to avoid reading it from disk all at once.
rawLoadedNumpyObject = numpy.load(str(fileName),
mmap_mode="r",
allow_pickle=False)
except (ValueError, IOError):
raise OpNpyFileReader.DatasetReadError(
"Unable to open numpy dataset: {}".format(fileName))
# .npy files:
if isinstance(rawLoadedNumpyObject, numpy.ndarray):
rawNumpyArray = rawLoadedNumpyObject
self._memmapFile = rawNumpyArray._mmap
# .npz files:
elif isinstance(rawLoadedNumpyObject, numpy.lib.npyio.NpzFile):
if self.InternalPath.ready():
try:
rawNumpyArray = rawLoadedNumpyObject[
self.InternalPath.value]
except KeyError:
raise OpNpyFileReader.DatasetReadError(
"InternalPath not found in file. Unable to open numpy npz dataset: "
"{fileName}: {internalPath}".format(
fileName=fileName,
internalPath=self.InternalPath.value))
else:
raise OpNpyFileReader.DatasetReadError(
"InternalPath not given. Unable to open numpy npz dataset: {fileName}"
.format(fileName=fileName))
shape = rawNumpyArray.shape
axisorder = get_default_axisordering(shape)
# Cast to vigra array
self._rawVigraArray = rawNumpyArray.view(vigra.VigraArray)
self._rawVigraArray.axistags = vigra.defaultAxistags(axisorder)
self.Output.meta.dtype = self._rawVigraArray.dtype.type
self.Output.meta.axistags = copy.copy(self._rawVigraArray.axistags)
self.Output.meta.shape = self._rawVigraArray.shape
def test_Roi_default_order(self):
for i in range(self.tests):
self.prepareVolnOp()
shape = self.operator.outputs["output"].meta.shape
roi = [None, None]
roi[1] = [
numpy.random.randint(2, s) if s != 1 else 1 for s in shape
]
roi[0] = [
numpy.random.randint(0, roi[1][i]) if s != 1 else 0
for i, s in enumerate(shape)
]
roi[0] = TinyVector(roi[0])
roi[1] = TinyVector(roi[1])
result = self.operator.outputs["output"](roi[0], roi[1]).wait()
logger.debug(
'------------------------------------------------------')
logger.debug("self.array.shape = " + str(self.array.shape))
logger.debug("type(input) == " +
str(type(self.operator.input.value)))
logger.debug("input.shape == " +
str(self.operator.input.meta.shape))
logger.debug("Input Tags:")
logger.debug(str(self.operator.inputs['input'].meta.axistags))
logger.debug("Output Tags:")
logger.debug(str(self.operator.output.meta.axistags))
logger.debug("roi= " + str(roi))
logger.debug("type(result) == " + str(type(result)))
logger.debug("result.shape == " + str(result.shape))
logger.debug(
'------------------------------------------------------')
# Check the shape
assert len(result.shape) == 5
# Ensure the result came out in volumina order
assert self.operator.outputs[
"output"].meta.axistags == vigra.defaultAxistags('txyzc')
# Check the data
vresult = result.view(vigra.VigraArray)
vresult.axistags = self.operator.outputs[
"output"].meta.axistags
reorderedInput = self.inArray.withAxes(*[
tag.key
for tag in self.operator.outputs["output"].meta.axistags
])
assert numpy.all(
vresult == reorderedInput[roiToSlice(roi[0], roi[1])])
def testBasic_Dvid(self):
if _skip_dvid:
raise nose.SkipTest
# Spin up a mock dvid server to test with.
dvid_dataset, data_uuid, data_name = "datasetA", "abcde", "indices_data"
mockserver_data_file = self._tmpdir + '/mockserver_data.h5'
with H5MockServerDataFile(mockserver_data_file) as test_h5file:
test_h5file.add_node(dvid_dataset, data_uuid)
server_proc, shutdown_event = H5MockServer.create_and_start(
mockserver_data_file,
"localhost",
8000,
same_process=False,
disable_server_logging=True)
try:
data = 255 * numpy.random.random((100, 100, 4))
data = data.astype(numpy.uint8)
data = vigra.taggedView(data, vigra.defaultAxistags('xyc'))
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue(data)
opExport = OpExportSlot(graph=graph)
opExport.Input.connect(opPiper.Output)
opExport.OutputFormat.setValue('dvid')
url = 'http://localhost:8000/api/node/{data_uuid}/{data_name}'.format(
**locals())
opExport.OutputFilenameFormat.setValue(url)
assert opExport.ExportPath.ready()
assert opExport.ExportPath.value == url
opExport.run_export()
try:
opRead = OpInputDataReader(graph=graph)
opRead.FilePath.setValue(opExport.ExportPath.value)
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data
).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
finally:
shutdown_event.set()
server_proc.join()
def setUp(self):
segimg = segImage()
binimg = (segimg > 0).astype(np.uint8)
labels = {0: np.array([0, 1, 2]), 1: np.array([0, 1, 1, 2])}
rawimg = np.indices(segimg.shape).sum(0).astype(np.float32)
rawimg = rawimg.view(vigra.VigraArray)
rawimg.axistags = vigra.defaultAxistags('txyzc')
g = Graph()
features = {"Standard Object Features": {"Count":{}, "RegionCenter":{}, "Coord<Principal<Kurtosis>>":{}, \
"Coord<Maximum>":{}, "Mean":{}, \
"Mean in neighborhood":{"margin":(30, 30, 1)}}}
sel_features = {
"Standard Object Features": {
"Count": {},
"Mean": {},
"Mean in neighborhood": {
"margin": (30, 30, 1)
},
"Variance": {}
}
}
self.extrOp = OpObjectExtraction(graph=g)
self.extrOp.BinaryImage.setValue(binimg)
self.extrOp.RawImage.setValue(rawimg)
self.extrOp.Features.setValue(features)
assert self.extrOp.RegionFeatures.ready()
feats = self.extrOp.RegionFeatures([0, 1]).wait()
assert len(feats) == rawimg.shape[0]
for key in features["Standard Object Features"]:
assert key in feats[0]["Standard Object Features"].keys()
self.trainop = OpObjectTrain(graph=g)
self.trainop.Features.resize(1)
self.trainop.Features.connect(self.extrOp.RegionFeatures)
self.trainop.SelectedFeatures.setValue(sel_features)
self.trainop.LabelsCount.setValue(2)
self.trainop.Labels.resize(1)
self.trainop.Labels.setValues([labels])
self.trainop.FixClassifier.setValue(False)
self.trainop.ForestCount.setValue(1)
assert self.trainop.Classifier.ready()
def test_Roi_default_order(self):
for i in range(self.tests):
self.prepareVolnOp()
shape = self.operator.Output.meta.shape
roi = [None, None]
roi[1] = [
numpy.random.randint(2, s) if s != 1 else 1 for s in shape
]
roi[0] = [
numpy.random.randint(0, roi[1][i]) if s != 1 else 0
for i, s in enumerate(shape)
]
roi[0] = TinyVector(roi[0])
roi[1] = TinyVector(roi[1])
result = self.operator.Output(roi[0], roi[1]).wait()
logger.debug(
"------------------------------------------------------")
logger.debug("self.array.shape = " + str(self.array.shape))
logger.debug("type(input) == " +
str(type(self.operator.Input.value)))
logger.debug("input.shape == " +
str(self.operator.Input.meta.shape))
logger.debug("Input Tags:")
logger.debug(str(self.operator.Input.meta.axistags))
logger.debug("Output Tags:")
logger.debug(str(self.operator.Output.meta.axistags))
logger.debug("roi= " + str(roi))
logger.debug("type(result) == " + str(type(result)))
logger.debug("result.shape == " + str(result.shape))
logger.debug(
"------------------------------------------------------")
# Check the shape
assert len(result.shape) == 5
assert not isinstance(
result, vigra.VigraArray
), "For compatibility with generic code, output should be provided as a plain numpy array."
# Ensure the result came out in volumina order
assert self.operator.Output.meta.axistags == vigra.defaultAxistags(
"tzyxc")
# Check the data
vresult = result.view(vigra.VigraArray)
vresult.axistags = self.operator.Output.meta.axistags
reorderedInput = self.inArray.withAxes(
*[tag.key for tag in self.operator.Output.meta.axistags])
assert numpy.all(
vresult == reorderedInput[roiToSlice(roi[0], roi[1])])
def setupOutputs(self):
"""
Load the file specified via our input slot and present its data on the output slot.
"""
fileName = self.FileName.value
self.mmf = MmfParser.MmfParser(str(fileName))
frameNum = self.mmf.getNumberOfFrames()
self.frame = self.mmf.getFrame(0)
self.Output.meta.dtype = self.frame.dtype.type
self.Output.meta.axistags = vigra.defaultAxistags(AXIS_ORDER)
self.Output.meta.shape = (frameNum, self.frame.shape[0], self.frame.shape[1], 1)
self.Output.meta.ideal_blockshape = (1,) + self.Output.meta.shape[1:]
def prepareVolnOp(self):
tags = random.sample(self.axis, random.randint(2, len(self.axis)))
tagStr = ''
for s in tags:
tagStr += s
axisTags = vigra.defaultAxistags(tagStr)
self.shape = []
for tag in axisTags:
self.shape.append(random.randint(20, 30))
self.array = (numpy.random.rand(*tuple(self.shape)) * 255)
self.array = (float(250) / 255 * self.array + 5).astype(int)
self.inArray = vigra.VigraArray(self.array, axistags=axisTags)
self.operator.inputs["input"].setValue(self.inArray)
def runConvexHull(self, segimg, feats):
# testing 2D Convex Hull features on a 2D image
rawimg = np.indices(segimg.shape).sum(0).astype(np.float32)
rawimg = rawimg.view(vigra.VigraArray)
rawimg.axistags = vigra.defaultAxistags("txyzc")
g = Graph()
self.featsop = OpRegionFeatures(graph=g)
self.featsop.LabelVolume.setValue(segimg)
self.featsop.RawVolume.setValue(rawimg)
self.featsop.Features.setValue(feats)
output = self.featsop.Output([]).wait()
return output
def setup_method(self, method):
self.data2d = numpy.zeros((3, 3, 3))
self.data2d[:, :, 0] = 1
self.data2d[:, :, 1] = 2
self.data2d[:, :, 2] = 3
self.data2d = self.data2d.view(vigra.VigraArray)
self.data2d.axistags = vigra.defaultAxistags("xyc")
self.data3d = numpy.zeros((3, 3, 3, 3))
self.data3d[:, :, :, 0] = 1
self.data3d[:, :, :, 1] = 2
self.data3d[:, :, :, 2] = 3
self.data3d = self.data3d.view(vigra.VigraArray)
self.data3d.axistags = vigra.defaultAxistags("xyzc")
self.data_bad_channel = numpy.zeros((3, 3, 3, 3))
self.data_bad_channel[:, :, 0, :] = 1
self.data_bad_channel[:, :, 1, :] = 2
self.data_bad_channel[:, :, 2, :] = 3
self.data_bad_channel = self.data_bad_channel.view(vigra.VigraArray)
self.data_bad_channel.axistags = vigra.defaultAxistags("xycz")
def put(self, slicing, array):
assert _has_vigra, "Lazyflow SinkSource requires lazyflow and vigra."
taggedArray = array.view(vigra.VigraArray)
taggedArray.axistags = vigra.defaultAxistags("txyzc")
inputTags = self._inputSlot.meta.axistags
inputKeys = [tag.key for tag in inputTags]
transposedArray = taggedArray.withAxes(*inputKeys)
taggedSlicing = dict(list(zip("txyzc", slicing)))
transposedSlicing = ()
for k in inputKeys:
if k in "txyzc":
transposedSlicing += (taggedSlicing[k],)
self._inputSlot[transposedSlicing] = transposedArray.view(np.ndarray)
def get_result_operator(input_data):
"""
Returns the result of the wrapping operator pipeline, which gets
accessed via the GUI.
"""
input_data = vigra.VigraArray(input_data, axistags=vigra.defaultAxistags(AXIS_TAGS))
graph = Graph()
with Pipeline(graph=graph) as get_wsdt:
get_wsdt.add(OpArrayPiper, Input=input_data)
get_wsdt.add(OpCachedWsdt, FreezeCache=False)
wsdt_result = get_wsdt[-1].outputs["Superpixels"][:].wait()
return wsdt_result
def setupOutputs(self):
# Create a RESTfulVolume object to read the description file and do the downloads.
self._volumeObject = RESTfulVolume(self.DescriptionFilePath.value)
self._axes = self._volumeObject.description.axes
outputShape = tuple(self._volumeObject.description.shape)
# If the dataset has no channel axis, add one.
if 'c' not in self._axes:
outputShape += (1, )
self._axes += 'c'
self.Output.meta.shape = outputShape
self.Output.meta.dtype = self._volumeObject.description.dtype
self.Output.meta.axistags = vigra.defaultAxistags(self._axes)
def test_OpStreamingUfmfReader(self):
# Tests the ufmf reader operator, opening a small video that is created on setUp.
self.graph = Graph()
ufmfReader = OpStreamingUfmfReader(graph=self.graph)
ufmfReader.FileName.setValue(self.testUfmfFileName)
output = ufmfReader.Output[:].wait()
# Verify shape, data type, and axis tags
assert output.shape == EXPECTED_SHAPE
assert ufmfReader.Output.meta.dtype == EXPECTED_DTYPE
assert ufmfReader.Output.meta.axistags == vigra.defaultAxistags(
EXPECTED_AXIS_ORDER)
# Clean reader
ufmfReader.cleanUp()
def _volume(nx=64, ny=64, nz=100, method="linear"):
b = vigra.VigraArray(np.ones((nx, ny, nz)), axistags=vigra.defaultAxistags("xyz"))
if method == "linear":
for i in range(b.shape[2]):
b[:, :, i] *= (i + 1) + 50
elif method == "cubic":
s = nz // 3
for z in range(b.shape[2]):
b[:, :, z] = (z - s) ** 2 * z * 150.0 / (nz * (nz - s) ** 2) + 30
elif method == "constant":
b[:] = 124
else:
raise NotImplementedError("unknown method '{}'.".format(method))
return b
def binaryImage():
frameNum = 500
img = np.zeros((frameNum, 100, 100, 1), dtype=np.float32)
for frame in range(frameNum):
img[frame, 0:10, 0:10, 0] = 1
img[frame, 20:30, 20:30, 0] = 1
img[frame, 40:45, 40:45, 0] = 1
img[frame, 60:80, 60:80, 0] = 1
img = img.view(vigra.VigraArray)
img.axistags = vigra.defaultAxistags('txyc')
return img
def setUp(self):
self.graph = lazyflow.graph.Graph()
self._tmpdir = tempfile.mkdtemp()
self._name_pattern = 'test_stack_slice_{slice_index}.tiff'
self._stack_filepattern = os.path.join(self._tmpdir,
self._name_pattern)
# Generate some test data
self.dataShape = (5, 10, 64, 128, 2)
self._axisorder = 'tzyxc'
self.testData = vigra.VigraArray(self.dataShape,
axistags=vigra.defaultAxistags(
self._axisorder),
order='C').astype(numpy.uint8)
self.testData[...] = numpy.indices(self.dataShape).sum(0)
def analyze(image_id, model):
args = app.parse_args([])
args.headless = True
args.project = model
args.readonly = True
shell = app.main(args)
input_data = load_from_s3(image_id)
# run ilastik headless
data = [{
"Raw Data":
PreloadedArrayDatasetInfo(preloaded_array=input_data,
axistags=vigra.defaultAxistags("tzyxc"))
}] # noqa
return shell.workflow.batchProcessingApplet.run_export(
data, export_to_array=True) # noqa
def configure_operator_with_parsed_args(self, parsed_args):
"""
Helper function for headless workflows.
Configures this applet's top-level operator according to the settings provided in ``parsed_args``.
:param parsed_args: Must be an ``argparse.Namespace`` as returned by :py:meth:`parse_known_cmdline_args()`.
"""
role_names = self.topLevelOperator.DatasetRoles.value
role_paths = self.role_paths_from_parsed_args(parsed_args)
for role_index, input_paths in list(role_paths.items()):
# If the user doesn't want image stacks to be copied into the project file,
# we generate hdf5 volumes in a temporary directory and use those files instead.
if parsed_args.preconvert_stacks:
import tempfile
input_paths = self.convertStacksToH5(input_paths, tempfile.gettempdir())
input_infos = [FilesystemDatasetInfo(filepath=p) if p else None for p in input_paths]
if parsed_args.input_axes:
for info in [_f for _f in input_infos if _f]:
info.axistags = vigra.defaultAxistags(parsed_args.input_axes)
opDataSelection = self.topLevelOperator
existing_lanes = len(opDataSelection.DatasetGroup)
opDataSelection.DatasetGroup.resize(max(len(input_infos), existing_lanes))
for lane_index, info in enumerate(input_infos):
if info:
opDataSelection.DatasetGroup[lane_index][role_index].setValue(info)
need_warning = False
for lane_index in range(len(input_infos)):
output_slot = opDataSelection.ImageGroup[lane_index][role_index]
if output_slot.ready() and output_slot.meta.prefer_2d and "z" in output_slot.meta.axistags:
need_warning = True
break
if need_warning:
logger.warning(
"*******************************************************************************************"
)
logger.warning(
"Some of your input data is stored in a format that is not efficient for 3D access patterns."
)
logger.warning("Performance may suffer as a result. For best performance, use a chunked HDF5 volume.")
logger.warning(
"*******************************************************************************************"
)
def setupOutputs(self):
# Read the dataset meta-info from the HDF5 dataset
self._h5N5File = self.H5N5File.value
internalPath = self.InternalPath.value
if internalPath not in self._h5N5File:
raise OpStreamingH5N5Reader.DatasetReadError(internalPath)
dataset = self._h5N5File[internalPath]
try:
# Read the axistags property without actually importing the data
# Throws KeyError if 'axistags' can't be found
axistagsJson = self._h5N5File[internalPath].attrs["axistags"]
axistags = vigra.AxisTags.fromJSON(axistagsJson)
axisorder = "".join(tag.key for tag in axistags)
if "?" in axisorder:
raise KeyError("?")
except KeyError:
# No axistags found.
axisorder = get_default_axisordering(dataset.shape)
axistags = vigra.defaultAxistags(str(axisorder))
assert len(axistags) == len(dataset.shape), f"Mismatch between shape {dataset.shape} and axisorder {axisorder}"
# Configure our slot meta-info
self.OutputImage.meta.dtype = dataset.dtype.type
self.OutputImage.meta.shape = dataset.shape
self.OutputImage.meta.axistags = axistags
# If the dataset specifies a datarange, add it to the slot metadata
if "drange" in self._h5N5File[internalPath].attrs:
self.OutputImage.meta.drange = tuple(self._h5N5File[internalPath].attrs["drange"])
# Same for display_mode
if "display_mode" in self._h5N5File[internalPath].attrs:
self.OutputImage.meta.display_mode = str(self._h5N5File[internalPath].attrs["display_mode"])
total_volume = numpy.prod(numpy.array(self._h5N5File[internalPath].shape))
chunks = self._h5N5File[internalPath].chunks
if not chunks and total_volume > 1e8:
self.OutputImage.meta.inefficient_format = True
logger.warning(
f"This dataset ({self._h5N5File.filename}{internalPath}) is NOT chunked. "
f"Performance for 3D access patterns will be bad!"
)
if chunks:
self.OutputImage.meta.ideal_blockshape = chunks
def __init__(
self,
*,
laneShape: Tuple,
laneDtype: type,
default_tags: AxisTags,
allowLabels: bool = True,
subvolume_roi: Tuple = None,
axistags: AxisTags = None,
display_mode: str = "default",
nickname: str = "",
normalizeDisplay: bool = None,
drange: Tuple[Number, Number] = None,
guess_tags_for_singleton_axes: bool = False,
):
self.laneShape = laneShape
self.laneDtype = laneDtype
if isinstance(self.laneDtype, numpy.dtype):
self.laneDtype = numpy.typeDict[self.laneDtype.name]
self.allowLabels = allowLabels
self.subvolume_roi = subvolume_roi
self.axistags = axistags
self.drange = drange
self.display_mode = display_mode # choices: default, grayscale, rgba, random-colortable, binary-mask.
self.nickname = nickname
self.normalizeDisplay = (self.drange is not None) if normalizeDisplay is None else normalizeDisplay
self.axistags = axistags or default_tags
if len(self.axistags) != len(self.laneShape):
if not guess_tags_for_singleton_axes:
raise UnsuitedAxistagsException(self.axistags, self.laneShape)
default_keys = [tag.key for tag in default_tags]
tagged_shape = dict(zip(default_keys, self.laneShape))
squeezed_shape = {k: v for k, v in tagged_shape.items() if v != 1}
requested_keys = [tag.key for tag in axistags]
if set(requested_keys).issubset(set(default_keys)) and set(default_keys) - set(requested_keys) == set("c"):
self.axistags = default_tags # allow missing 'c' in axistags; not sure if this is a good idea
elif len(requested_keys) == len(squeezed_shape):
dummy_axes = [key for key in "ctzxy" if key not in requested_keys]
out_axes = ""
for k, v in tagged_shape.items():
if v > 1:
out_axes += requested_keys.pop(0)
else:
out_axes += dummy_axes.pop(0)
self.axistags = vigra.defaultAxistags(out_axes)
else:
raise UnsuitedAxistagsException(requested_keys, self.laneShape)
self.legacy_datasetId = self.generate_id()
def reorder_axes(input_arr: numpy.ndarray, *, from_axes_tags: str,
to_axes_tags: str):
if isinstance(from_axes_tags, AxisTags):
from_axes_tags = "".join(from_axes_tags.keys())
if isinstance(to_axes_tags, AxisTags):
to_axes_tags = "".join(to_axes_tags.keys())
op = OpReorderAxes(graph=Graph())
tagged_arr = vigra.VigraArray(
input_arr, axistags=vigra.defaultAxistags(from_axes_tags))
op.Input.setValue(tagged_arr)
op.AxisOrder.setValue(to_axes_tags)
return op.Output([]).wait()
def setUp(self):
self.dataShape = (1, 100, 100, 10, 1)
self.data = (numpy.random.random(self.dataShape) * 100).astype(int)
self.data = self.data.view(vigra.VigraArray)
self.data.axistags = vigra.defaultAxistags('txyzc')
graph = Graph()
opProvider = OpArrayPiperWithAccessCount(graph=graph)
opProvider.Input.setValue(self.data)
self.opProvider = opProvider
opCache = OpArrayCache(graph=graph)
opCache.Input.connect(opProvider.Output)
opCache.blockShape.setValue((10, 10, 10, 10, 10))
opCache.fixAtCurrent.setValue(False)
self.opCache = opCache
