def __init__(self, *args, **kwargs):
super(OpNansheExtractF0Cached, self).__init__(*args, **kwargs)
self.opExtractF0 = OpNansheExtractF0(parent=self, cache_f0=True)
self.opExtractF0.HalfWindowSize.connect(self.HalfWindowSize)
self.opExtractF0.WhichQuantile.connect(self.WhichQuantile)
self.opExtractF0.TemporalSmoothingGaussianFilterStdev.connect(
self.TemporalSmoothingGaussianFilterStdev)
self.opExtractF0.TemporalSmoothingGaussianFilterWindowSize.connect(
self.TemporalSmoothingGaussianFilterWindowSize)
self.opExtractF0.SpatialSmoothingGaussianFilterStdev.connect(
self.SpatialSmoothingGaussianFilterStdev)
self.opExtractF0.SpatialSmoothingGaussianFilterWindowSize.connect(
self.SpatialSmoothingGaussianFilterWindowSize)
self.opExtractF0.BiasEnabled.connect(self.BiasEnabled)
self.opExtractF0.Bias.connect(self.Bias)
self.opCache_dF_F = OpBlockedArrayCache(parent=self)
self.opCache_dF_F.fixAtCurrent.setValue(False)
self.opCache_F0 = OpBlockedArrayCache(parent=self)
self.opCache_F0.fixAtCurrent.setValue(False)
self.opExtractF0.Input.connect(self.Input)
self.opCache_F0.Input.connect(self.opExtractF0.F0)
self.opCache_dF_F.Input.connect(self.opExtractF0.dF_F)
self.F0.connect(self.opExtractF0.F0)
self.dF_F.connect(self.opExtractF0.dF_F)
def testCleanup(self):
try:
CacheMemoryManager().disable()
op = OpBlockedArrayCache(graph=self.opProvider.graph)
op.Input.connect(self.opProvider.Output)
s = self.opProvider.Output.meta.shape
op.innerBlockShape.setValue(s)
op.outerBlockShape.setValue(s)
op.fixAtCurrent.setValue(False)
x = op.Output[...].wait()
op.Input.disconnect()
op.cleanUp()
r = weakref.ref(op)
del op
gc.collect()
ref = r()
if ref is not None:
for i, o in enumerate(gc.get_referrers(ref)):
print "Object", i, ":", type(o), ":", o
assert r() is None, "OpBlockedArrayCache was not cleaned up correctly"
finally:
CacheMemoryManager().enable()
def testCleanup(self):
try:
CacheMemoryManager().disable()
op = OpBlockedArrayCache(graph=self.opProvider.graph)
op.Input.connect(self.opProvider.Output)
s = self.opProvider.Output.meta.shape
op.innerBlockShape.setValue(s)
op.outerBlockShape.setValue(s)
op.fixAtCurrent.setValue(False)
x = op.Output[...].wait()
op.Input.disconnect()
op.cleanUp()
r = weakref.ref(op)
del op
gc.collect()
ref = r()
if ref is not None:
for i, o in enumerate(gc.get_referrers(ref)):
print "Object", i, ":", type(o), ":", o
assert r(
) is None, "OpBlockedArrayCache was not cleaned up correctly"
finally:
CacheMemoryManager().enable()
def __init__(self, *args, **kwargs):
super(OpDLClassification, self).__init__(*args, **kwargs)
self.predict = OpPixelwiseClassifierPredict(parent=self)
self.predict.name = "OpClassifierPredict"
self.predict.Image.connect(self.InputImage)
self.predict.Classifier.connect(self.Classifier)
self.predict.LabelsCount.connect(self.NumClasses)
self.PredictionProbabilities.connect(self.predict.PMaps)
self.prediction_cache = OpBlockedArrayCache(parent=self)
self.prediction_cache.name = "BlockedArrayCache"
self.prediction_cache.inputs["Input"].connect(self.predict.PMaps)
self.prediction_cache.BlockShape.connect(self.BlockShape)
self.prediction_cache.inputs["fixAtCurrent"].connect(self.FreezePredictions)
self.CachedPredictionProbabilities.connect(self.prediction_cache.Output)
self.opPredictionSlicer = OpMultiArraySlicer2(parent=self)
self.opPredictionSlicer.name = "opPredictionSlicer"
self.opPredictionSlicer.Input.connect(self.prediction_cache.Output)
self.opPredictionSlicer.AxisFlag.setValue("c")
self.PredictionProbabilityChannels.connect(self.opPredictionSlicer.Slices)
self.opSegmentor = OpMaxChannelIndicatorOperator(parent=self) # IMPROVEME: this "max channel" works for binary classification, but not when we have more than 2 classes
self.opSegmentor.Input.connect(self.prediction_cache.Output)
self.Segmentation.connect(self.opSegmentor.Output)
self.opSegmentationSlicer = OpMultiArraySlicer2(parent=self)
self.opSegmentationSlicer.name = "opSegmentationSlicer"
self.opSegmentationSlicer.Input.connect(self.opSegmentor.Output)
self.opSegmentationSlicer.AxisFlag.setValue("c")
self.SegmentationChannels.connect(self.opSegmentationSlicer.Slices)
def testBlockedCacheHandling(self, cacheMemoryManager):
n, k = 10, 5
vol = np.zeros((n,) * 5, dtype=np.uint8)
vol = vigra.taggedView(vol, axistags="txyzc")
g = Graph()
pipe = OpArrayPiperWithAccessCount(graph=g)
cache = OpBlockedArrayCache(graph=g)
# restrict cache memory to 0 Byte
Memory.setAvailableRamCaches(0)
# set to frequent cleanup
cacheMemoryManager.setRefreshInterval(0.01)
cacheMemoryManager.enable()
cache.BlockShape.setValue((k,) * 5)
cache.Input.connect(pipe.Output)
pipe.Input.setValue(vol)
a = pipe.accessCount
cache.Output[...].wait()
b = pipe.accessCount
assert b > a, "did not cache"
# let the manager clean up
cacheMemoryManager.enable()
time.sleep(0.5)
gc.collect()
cache.Output[...].wait()
c = pipe.accessCount
assert c > b, "did not clean up"
def setupOutputs(self):
self.shape = self.inputs["Input"].meta.shape
self._outerShapes = self.inputs["outerBlockShape"].value
self._innerShapes = self.inputs["innerBlockShape"].value
for blockshape in self._innerShapes + self._outerShapes:
if len(blockshape) != len(self.Input.meta.shape):
self.Output.meta.NOTREADY = True
return
# FIXME: This is wrong: Shouldn't it actually compare the new inner block shape with the old one?
if len(self._innerShapes) != len(self._innerOps):
# Clean up previous inner operators
for slot in self.InnerOutputs:
slot.disconnect()
for o in self._innerOps:
o.cleanUp()
self._innerOps = []
for i, innershape in enumerate(self._innerShapes):
op = OpBlockedArrayCache(parent=self)
op.inputs["fixAtCurrent"].connect(self.inputs["fixAtCurrent"])
op.BypassModeEnabled.connect(self.BypassModeEnabled)
op.CompressionEnabled.connect(self.CompressionEnabled)
self._innerOps.append(op)
op.inputs["Input"].connect(self.inputs["Input"])
# Forward "value changed" notifications to our own output
op.Output.notifyValueChanged(self.Output._sig_value_changed)
for i, innershape in enumerate(self._innerShapes):
op = self._innerOps[i]
op.inputs["innerBlockShape"].setValue(innershape)
op.inputs["outerBlockShape"].setValue(self._outerShapes[i])
self.Output.meta.assignFrom(self.Input.meta)
# Estimate ram usage
ram_per_pixel = 0
if self.Output.meta.dtype == object or self.Output.meta.dtype == numpy.object_:
ram_per_pixel = sys.getsizeof(None)
elif numpy.issubdtype(self.Output.meta.dtype, numpy.dtype):
ram_per_pixel = self.Output.meta.dtype().nbytes
tagged_shape = self.Output.meta.getTaggedShape()
if 'c' in tagged_shape:
ram_per_pixel *= float(tagged_shape['c'])
if self.Output.meta.ram_usage_per_requested_pixel is not None:
ram_per_pixel = max(ram_per_pixel,
self.Output.meta.ram_usage_per_requested_pixel)
self.Output.meta.ram_usage_per_requested_pixel = ram_per_pixel
# We also provide direct access to each of our inner cache outputs.
self.InnerOutputs.resize(len(self._innerOps))
for i, slot in enumerate(self.InnerOutputs):
slot.connect(self._innerOps[i].Output)
def __init__(self):
# Set memory and number of threads here
#lazyflow.request.Request.reset_thread_pool(2)
#Memory.setAvailableRam(500*1024**2)
binary_img = binaryImage()
raw_img = rawImage()
g = Graph()
# Reorder axis operators
self.op5Raw = OpReorderAxes(graph=g)
self.op5Raw.AxisOrder.setValue("txyzc")
#self.op5Raw.Input.connect(self.opReaderRaw.OutputImage)#self.opReaderRaw.OutputImage)
self.op5Raw.Input.setValue(raw_img)
self.op5Binary = OpReorderAxes(graph=g)
self.op5Binary.AxisOrder.setValue("txyzc")
#self.op5Binary.Input.connect(self.opReaderBinary.OutputImage)
self.op5Binary.Input.setValue(binary_img)
# Cache operators
self.opCacheRaw = OpBlockedArrayCache(graph=g)
self.opCacheRaw.Input.connect(self.op5Raw.Output)
self.opCacheRaw.BlockShape.setValue((1, ) +
self.op5Raw.Output.meta.shape[1:])
self.opCacheBinary = OpBlockedArrayCache(graph=g)
self.opCacheBinary.Input.connect(self.op5Binary.Output)
self.opCacheBinary.BlockShape.setValue(
(1, ) + self.op5Binary.Output.meta.shape[1:])
# Label volume operator
self.opLabel = OpLabelVolume(graph=g)
self.opLabel.Input.connect(self.op5Binary.Output)
#self.opLabel.Input.connect(self.opCacheBinary.Output)
# Object extraction
self.opObjectExtraction = OpObjectExtraction(graph=g)
self.opObjectExtraction.RawImage.connect(self.op5Raw.Output)
self.opObjectExtraction.BinaryImage.connect(self.op5Binary.Output)
self.opObjectExtraction.Features.setValue(FEATURES)
# Simplified object features operator (No overhead)
self.opObjectFeaturesSimp = OpObjectFeaturesSimplified(graph=g)
self.opObjectFeaturesSimp.RawVol.connect(self.opCacheRaw.Output)
self.opObjectFeaturesSimp.BinaryVol.connect(self.opCacheBinary.Output)
def __init__(self, *args, **kwargs):
super(OpCachedTiledVolumeReader, self).__init__(*args, **kwargs)
self._opReader = OpTiledVolumeReader(parent=self)
self._opReader.DescriptionFilePath.connect(self.DescriptionFilePath)
self.UncachedOutput.connect(self._opReader.Output)
self._opCache = OpBlockedArrayCache(parent=self)
self._opCache.Input.connect(self._opReader.Output)
self._opCache.fixAtCurrent.setValue(False)
self.CachedOutput.connect(self._opCache.Output)
def __init__(self, *args, **kwargs):
super(OpColorizeLabelImageCached, self).__init__(*args, **kwargs)
self.opColorizeLabelImage = OpColorizeLabelImage(parent=self)
self.opColorizeLabelImage.NumColors.connect(self.NumColors)
self.opCache = OpBlockedArrayCache(parent=self)
self.opCache.fixAtCurrent.setValue(False)
self.opColorizeLabelImage.Input.connect(self.Input)
self.opCache.Input.connect(self.opColorizeLabelImage.Output)
self.Output.connect(self.opCache.Output)
def setupOutputs(self):
self.shape = self.inputs["Input"].meta.shape
for blockshape in self.BlockShape.value:
if len(blockshape) != len(self.Input.meta.shape):
self.Output.meta.NOTREADY = True
return
if self._blockshapes != self.BlockShape.value:
# Clean up previous inner operators
for slot in self.InnerOutputs:
slot.disconnect()
for o in self._innerOps:
o.cleanUp()
self._innerOps = []
self._blockshapes = self.BlockShape.value
for i, innershape in enumerate(self._blockshapes):
op = OpBlockedArrayCache(parent=self)
op.inputs["fixAtCurrent"].connect(self.inputs["fixAtCurrent"])
op.BypassModeEnabled.connect(self.BypassModeEnabled)
op.CompressionEnabled.connect(self.CompressionEnabled)
self._innerOps.append(op)
op.inputs["Input"].connect(self.inputs["Input"])
# Forward "value changed" notifications to our own output
op.Output.notifyValueChanged(self.Output._sig_value_changed)
for i, innershape in enumerate(self._blockshapes):
op = self._innerOps[i]
op.inputs["BlockShape"].setValue(innershape)
self.Output.meta.assignFrom(self.Input.meta)
# Estimate ram usage
ram_per_pixel = get_ram_per_element(self.Output.meta.dtype)
tagged_shape = self.Output.meta.getTaggedShape()
if "c" in tagged_shape:
ram_per_pixel *= float(tagged_shape["c"])
if self.Output.meta.ram_usage_per_requested_pixel is not None:
ram_per_pixel = max(ram_per_pixel,
self.Output.meta.ram_usage_per_requested_pixel)
self.Output.meta.ram_usage_per_requested_pixel = ram_per_pixel
# We also provide direct access to each of our inner cache outputs.
self.InnerOutputs.resize(len(self._innerOps))
for i, slot in enumerate(self.InnerOutputs):
slot.connect(self._innerOps[i].Output)
def __init__(self, *args, **kwargs):
super(OpConvertTypeCached, self).__init__(*args, **kwargs)
self.opConvertType = OpConvertType(parent=self)
self.opConvertType.Dtype.connect(self.Dtype)
self.opCache = OpBlockedArrayCache(parent=self)
self.opCache.fixAtCurrent.setValue(False)
self.opConvertType.Input.connect(self.Input)
self.opCache.Input.connect(self.opConvertType.Output)
self.Output.connect(self.opCache.Output)
def __init__(self, *args, **kwargs):
super(OpMeanProjectionCached, self).__init__(*args, **kwargs)
self.opMeanProjection = OpMeanProjection(parent=self)
self.opMeanProjection.Axis.connect(self.Axis)
self.opCache = OpBlockedArrayCache(parent=self)
self.opCache.fixAtCurrent.setValue(False)
self.opMeanProjection.Input.connect(self.Input)
self.opCache.Input.connect(self.opMeanProjection.Output)
self.Output.connect(self.opCache.Output)
def __init__(self, *args, **kwargs):
super(OpNansheWaveletTransformCached, self).__init__(*args, **kwargs)
self.opWaveletTransform = OpNansheWaveletTransform(parent=self)
self.opWaveletTransform.Scale.connect(self.Scale)
self.opCache = OpBlockedArrayCache(parent=self)
self.opCache.fixAtCurrent.setValue(False)
self.opWaveletTransform.Input.connect(self.Input)
self.opCache.Input.connect(self.opWaveletTransform.Output)
self.Output.connect(self.opCache.Output)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.RESTfulReader = OpRESTfulPrecomputedChunkedVolumeReaderNoCache(parent=self)
self.RESTfulReader.BaseUrl.connect(self.BaseUrl)
self.AvailableScales.connect(self.RESTfulReader.AvailableScales)
self.RESTfulReader.Scale.backpropagate_values = True
self.RESTfulReader.Scale.connect(self.Scale)
self.cache = OpBlockedArrayCache(parent=self)
self.cache.name = "input_image_cache"
self.cache.fixAtCurrent.connect(self.fixAtCurrent)
self.cache.Input.connect(self.RESTfulReader.Output)
self.Output.connect(self.cache.Output)
def __init__(self, *args, **kwargs):
super( OpNansheRemoveZeroedLinesCached, self ).__init__( *args, **kwargs )
self.opRemoveZeroedLines = OpNansheRemoveZeroedLines(parent=self)
self.opRemoveZeroedLines.ErosionShape.connect(self.ErosionShape)
self.opRemoveZeroedLines.DilationShape.connect(self.DilationShape)
self.opCache = OpBlockedArrayCache(parent=self)
self.opCache.fixAtCurrent.setValue(False)
self.opRemoveZeroedLines.Input.connect( self.Input )
self.opCache.Input.connect( self.opRemoveZeroedLines.Output )
self.Output.connect( self.opCache.Output )
def setup_method(self, method):
self.dataShape = (1, 100, 100, 10, 1)
self.data = numpy.random.randint(0, 256, size=self.dataShape)
self.data = self.data.astype(numpy.uint32)
self.data = self.data.view(vigra.VigraArray)
self.data.axistags = vigra.defaultAxistags("txyzc")
graph = Graph()
opProvider = OpArrayPiperWithAccessCount(graph=graph)
opProvider.Input.setValue(self.data)
self.opProvider = opProvider
opCache = OpBlockedArrayCache(graph=graph)
opCache.Input.connect(opProvider.Output)
opCache.BlockShape.setValue((20, 20, 20, 20, 20))
opCache.fixAtCurrent.setValue(False)
self.opCache = opCache
def testBasic(self):
features = numpy.indices((100, 100)).astype(numpy.float32) + 0.5
features = numpy.rollaxis(features, 0, 3)
features = vigra.taggedView(features, "xyc")
labels = numpy.zeros((100, 100, 1), dtype=numpy.uint8)
labels = vigra.taggedView(labels, "xyc")
labels[10, 10] = 1
labels[10, 11] = 1
labels[20, 20] = 2
labels[20, 21] = 2
graph = Graph()
# Use a cache for the labels so we can control the ideal_blockshape
# This ensures that the blockwise behavior is tested, even though we're
# testing with tiny data that would normally fall into a single block.
opLabelCache = OpBlockedArrayCache(graph=graph)
opLabelCache.BlockShape.setValue((10, 10, 1))
opLabelCache.Input.setValue(labels)
opFeatureMatrixCache = OpFeatureMatrixCache(graph=graph)
opFeatureMatrixCache.LabelImage.connect(opLabelCache.Output)
opFeatureMatrixCache.FeatureImage.setValue(features)
labels_and_features = opFeatureMatrixCache.LabelAndFeatureMatrix.value
assert labels_and_features.shape == (
0, 3), "Empty feature matrix has wrong shape: {}".format(
labels_and_features.shape)
opFeatureMatrixCache.LabelImage.setDirty(numpy.s_[10:11, 10:12])
opFeatureMatrixCache.LabelImage.setDirty(numpy.s_[20:21, 20:22])
opFeatureMatrixCache.LabelImage.setDirty(numpy.s_[30:31, 30:32])
labels_and_features = opFeatureMatrixCache.LabelAndFeatureMatrix.value
assert labels_and_features.shape == (4, 3)
assert (labels_and_features[:, 0] == 1).sum() == 2
assert (labels_and_features[:, 0] == 2).sum() == 2
# Can't check for equality because feature blocks can be in a random order.
# Just check that all features are present, regardless of order.
for feature_vec in [[10.5, 10.5], [10.5, 11.5], [20.5, 20.5],
[20.5, 21.5]]:
assert feature_vec in labels_and_features[:, 1:]
def __init__(self, *args, **kwargs):
super( OpNansheEstimateF0Cached, self ).__init__( *args, **kwargs )
self.opEstimateF0 = OpNansheEstimateF0(parent=self)
self.opEstimateF0.HalfWindowSize.connect(self.HalfWindowSize)
self.opEstimateF0.WhichQuantile.connect(self.WhichQuantile)
self.opEstimateF0.TemporalSmoothingGaussianFilterStdev.connect(self.TemporalSmoothingGaussianFilterStdev)
self.opEstimateF0.TemporalSmoothingGaussianFilterWindowSize.connect(self.TemporalSmoothingGaussianFilterWindowSize)
self.opEstimateF0.SpatialSmoothingGaussianFilterStdev.connect(self.SpatialSmoothingGaussianFilterStdev)
self.opEstimateF0.SpatialSmoothingGaussianFilterWindowSize.connect(self.SpatialSmoothingGaussianFilterWindowSize)
self.opCache_F0 = OpBlockedArrayCache(parent=self)
self.opCache_F0.fixAtCurrent.setValue(False)
self.opEstimateF0.Input.connect( self.Input )
self.opCache_F0.Input.connect( self.opEstimateF0.Output )
self.Output.connect( self.opCache_F0.Output )
def setup_method(self, method):
self.dataShape = (1, 100, 100, 10, 1)
self.data = (numpy.random.random(self.dataShape) * 100).astype(int)
self.data = numpy.ma.masked_array(
self.data, mask=numpy.ma.getmaskarray(self.data), fill_value=numpy.iinfo(int).max, shrink=False
)
self.data[:, 0] = numpy.ma.masked
graph = Graph()
opProvider = OpArrayPiperWithAccessCount(graph=graph)
opProvider.Input.meta.axistags = vigra.defaultAxistags("txyzc")
opProvider.Input.meta.has_mask = True
opProvider.Input.setValue(self.data)
self.opProvider = opProvider
opCache = OpBlockedArrayCache(graph=graph)
opCache.Input.connect(opProvider.Output)
opCache.BlockShape.setValue((20, 20, 20, 20, 20))
opCache.fixAtCurrent.setValue(False)
self.opCache = opCache
def __init__(self, *args, **kwargs):
super(OpNNClassification, self).__init__(*args, **kwargs)
self.predict = OpPixelwiseClassifierPredict(parent=self)
self.predict.name = "OpClassifierPredict"
self.predict.Image.connect(self.InputImage)
self.predict.Classifier.connect(self.Classifier)
self.predict.LabelsCount.connect(self.NumClasses)
self.PredictionProbabilities.connect(self.predict.PMaps)
self.prediction_cache = OpBlockedArrayCache(parent=self)
self.prediction_cache.name = "BlockedArrayCache"
self.prediction_cache.inputs["Input"].connect(self.predict.PMaps)
self.prediction_cache.BlockShape.connect(self.BlockShape)
self.prediction_cache.inputs["fixAtCurrent"].connect(self.FreezePredictions)
self.CachedPredictionProbabilities.connect(self.prediction_cache.Output)
self.opPredictionSlicer = OpMultiArraySlicer2(parent=self)
self.opPredictionSlicer.name = "opPredictionSlicer"
self.opPredictionSlicer.Input.connect(self.prediction_cache.Output)
self.opPredictionSlicer.AxisFlag.setValue("c")
self.PredictionProbabilityChannels.connect(self.opPredictionSlicer.Slices)
def setupOutputs(self):
self.shape = self.inputs["Input"].meta.shape
self._outerShapes = self.inputs["outerBlockShape"].value
self._innerShapes = self.inputs["innerBlockShape"].value
# FIXME: This is wrong: Shouldn't it actually compare the new inner block shape with the old one?
if len(self._innerShapes) != len(self._innerOps):
# Clean up previous inner operators
for slot in self.InnerOutputs:
slot.disconnect()
for o in self._innerOps:
o.cleanUp()
self._innerOps = []
for i, innershape in enumerate(self._innerShapes):
op = OpBlockedArrayCache(parent=self)
op.inputs["fixAtCurrent"].connect(self.inputs["fixAtCurrent"])
self._innerOps.append(op)
op.inputs["Input"].connect(self.inputs["Input"])
# Forward "value changed" notifications to our own output
op.Output.notifyValueChanged(self.Output._sig_value_changed)
for i, innershape in enumerate(self._innerShapes):
op = self._innerOps[i]
op.inputs["innerBlockShape"].setValue(innershape)
op.inputs["outerBlockShape"].setValue(self._outerShapes[i])
self.Output.meta.assignFrom(self.Input.meta)
# We also provide direct access to each of our inner cache outputs.
self.InnerOutputs.resize(len(self._innerOps))
for i, slot in enumerate(self.InnerOutputs):
slot.connect(self._innerOps[i].Output)
def testBadMemoryConditions(self):
"""
TestCacheMemoryManager.testBadMemoryConditions
This test is a proof of the proposition in
https://github.com/ilastik/lazyflow/issue/185
which states that, given certain memory constraints, the cache
cleanup strategy in use is inefficient. An advanced strategy
should pass the test.
"""
mgr = _CacheMemoryManager()
mgr.setRefreshInterval(0.01)
mgr.enable()
d = 2
tags = "xy"
shape = (999,) * d
blockshape = (333,) * d
# restrict memory for computation to one block (including fudge
# factor 2 of bigRequestStreamer)
cacheMem = np.prod(shape)
Memory.setAvailableRam(np.prod(blockshape) * 2 + cacheMem)
# restrict cache memory to the whole volume
Memory.setAvailableRamCaches(cacheMem)
# to ease observation, do everything single threaded
Request.reset_thread_pool(num_workers=1)
x = np.zeros(shape, dtype=np.uint8)
x = vigra.taggedView(x, axistags=tags)
g = Graph()
pipe = OpArrayPiperWithAccessCount(graph=g)
pipe.Input.setValue(x)
pipe.Output.meta.ideal_blockshape = blockshape
# simulate BlockedArrayCache behaviour without caching
# cache = OpSplitRequestsBlockwise(True, graph=g)
# cache.BlockShape.setValue(blockshape)
# cache.Input.connect(pipe.Output)
cache = OpBlockedArrayCache(graph=g)
cache.Input.connect(pipe.Output)
cache.BlockShape.setValue(blockshape)
op = OpEnlarge(graph=g)
op.Input.connect(cache.Output)
split = OpSplitRequestsBlockwise(True, graph=g)
split.BlockShape.setValue(blockshape)
split.Input.connect(op.Output)
streamer = BigRequestStreamer(split.Output, [(0,) * len(shape), shape])
streamer.execute()
# in the worst case, we have 4*4 + 4*6 + 9 = 49 requests to pipe
# in the best case, we have 9
np.testing.assert_equal(pipe.accessCount, 9)
def import_labeling_layer(labelLayer, labelingSlots, parent_widget=None):
"""
Prompt the user for layer import settings, and perform the layer import.
:param labelLayer: The top label layer source
:param labelingSlots: An instance of LabelingGui.LabelingSlots
:param parent_widget: The Qt GUI parent object
"""
writeSeeds = labelingSlots.labelInput
assert isinstance(
writeSeeds,
lazyflow.graph.Slot), "slot is of type %r" % (type(writeSeeds))
opLabels = writeSeeds.getRealOperator()
assert isinstance(opLabels, lazyflow.graph.Operator
), "slot's operator is of type %r" % (type(opLabels))
fileNames = ImageFileDialog(
parent_widget,
preferences_group="labeling",
preferences_setting="recently imported").getSelectedPaths()
fileNames = list(map(str, fileNames))
if not fileNames:
return
try:
# Initialize operators
opImport = OpInputDataReader(parent=opLabels.parent)
opCache = OpBlockedArrayCache(parent=opLabels.parent)
opMetadataInjector = OpMetadataInjector(parent=opLabels.parent)
opReorderAxes = OpReorderAxes(parent=opLabels.parent)
# Set up the pipeline as follows:
#
# opImport --> (opCache) --> opMetadataInjector --------> opReorderAxes --(inject via setInSlot)--> labelInput
# / /
# User-specified axisorder labelInput.meta.axistags
opImport.FilePath.setValue(fileNames[0] if len(fileNames) ==
1 else os.path.pathsep.join(fileNames))
assert opImport.Output.ready()
maxLabels = len(labelingSlots.labelNames.value)
# We don't bother with counting the label pixels
# (and caching the data) if it's big (1 GB)
if numpy.prod(opImport.Output.meta.shape) > 1e9:
reading_slot = opImport.Output
# For huge data, we don't go through and search for the pixel values,
# because that takes an annoyingly long amount of time.
# Instead, we make the reasonable assumption that the input labels are already 1,2,3..N
# and we don't tell the user what the label pixel counts are.
unique_read_labels = numpy.array(list(range(maxLabels + 1)))
readLabelCounts = numpy.array([-1] * (maxLabels + 1))
labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
else:
opCache.Input.connect(opImport.Output)
opCache.CompressionEnabled.setValue(True)
assert opCache.Output.ready()
reading_slot = opCache.Output
# We'll show a little window with a busy indicator while the data is loading
busy_dlg = QProgressDialog(parent=parent_widget)
busy_dlg.setLabelText("Scanning Label Data...")
busy_dlg.setCancelButton(None)
busy_dlg.setMinimum(100)
busy_dlg.setMaximum(100)
def close_busy_dlg(*args):
QApplication.postEvent(busy_dlg, QCloseEvent())
# Load the data from file into our cache
# When it's done loading, close the progress dialog.
req = reading_slot[:]
req.notify_finished(close_busy_dlg)
req.notify_failed(close_busy_dlg)
req.submit()
busy_dlg.exec_()
readData = req.result
# Can't use return_counts feature because that requires numpy >= 1.9
# unique_read_labels, readLabelCounts = numpy.unique(readData, return_counts=True)
# This does the same as the above, albeit slower, and probably with more ram.
bincounts = chunked_bincount(readData)
unique_read_labels = bincounts.nonzero()[0].astype(readData.dtype,
copy=False)
readLabelCounts = bincounts[unique_read_labels]
labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
del readData
opMetadataInjector.Input.connect(reading_slot)
metadata = reading_slot.meta.copy()
opMetadataInjector.Metadata.setValue(metadata)
opReorderAxes.Input.connect(opMetadataInjector.Output)
# Transpose the axes for assignment to the labeling operator.
opReorderAxes.AxisOrder.setValue(writeSeeds.meta.getAxisKeys())
# Ask the user how to interpret the data.
settingsDlg = LabelImportOptionsDlg(parent_widget, fileNames,
opMetadataInjector.Output,
labelingSlots.labelInput,
labelInfo)
def handle_updated_axes():
# The user is specifying a new interpretation of the file's axes
updated_axisorder = str(settingsDlg.axesEdit.text())
metadata = opMetadataInjector.Metadata.value.copy()
metadata.axistags = vigra.defaultAxistags(updated_axisorder)
opMetadataInjector.Metadata.setValue(metadata)
if opReorderAxes._invalid_axes:
settingsDlg.buttonBox.button(
QDialogButtonBox.Ok).setEnabled(False)
# Red background
settingsDlg.axesEdit.setStyleSheet(
"QLineEdit { background: rgb(255, 128, 128); selection-background-color: rgb(128, 128, 255); }"
)
settingsDlg.axesEdit.editingFinished.connect(handle_updated_axes)
# Initialize
handle_updated_axes()
dlg_result = settingsDlg.exec_()
if dlg_result != LabelImportOptionsDlg.Accepted:
return
# Get user's chosen label mapping from dlg
labelMapping = settingsDlg.labelMapping
# Get user's chosen offsets, ordered by the 'write seeds' slot
axes_5d = opReorderAxes.Output.meta.getAxisKeys()
tagged_offsets = collections.OrderedDict(
list(zip(axes_5d, [0] * len(axes_5d))))
tagged_offsets.update(
dict(
list(
zip(opReorderAxes.Output.meta.getAxisKeys(),
settingsDlg.imageOffsets))))
imageOffsets = list(tagged_offsets.values())
# Optimization if mapping is identity
if list(labelMapping.keys()) == list(labelMapping.values()):
labelMapping = None
# If the data was already cached, this will be fast.
label_data = opReorderAxes.Output[:].wait()
# Map input labels to output labels
if labelMapping:
# There are other ways to do a relabeling (e.g skimage.segmentation.relabel_sequential)
# But this supports potentially huge values of unique_read_labels (in the billions),
# without needing GB of RAM.
mapping_indexes = numpy.searchsorted(unique_read_labels,
label_data)
new_labels = numpy.array(
[labelMapping[x] for x in unique_read_labels])
label_data[:] = new_labels[mapping_indexes]
label_roi = numpy.array(roiFromShape(opReorderAxes.Output.meta.shape))
label_roi += imageOffsets
label_slice = roiToSlice(*label_roi)
writeSeeds[label_slice] = label_data
finally:
opReorderAxes.cleanUp()
opMetadataInjector.cleanUp()
opCache.cleanUp()
opImport.cleanUp()
def import_labeling_layer(labelLayer, labelingSlots, parent_widget=None):
"""
Prompt the user for layer import settings, and perform the layer import.
:param labelLayer: The top label layer source
:param labelingSlots: An instance of LabelingGui.LabelingSlots
:param parent_widget: The Qt GUI parent object
"""
writeSeeds = labelingSlots.labelInput
assert isinstance(
writeSeeds,
lazyflow.graph.Slot), "slot is of type %r" % (type(writeSeeds))
opLabels = writeSeeds.getRealOperator()
assert isinstance(opLabels, lazyflow.graph.Operator
), "slot's operator is of type %r" % (type(opLabels))
recentlyImported = PreferencesManager().get('labeling',
'recently imported')
mostRecentProjectPath = PreferencesManager().get('shell',
'recently opened')
mostRecentImageFile = PreferencesManager().get('DataSelection',
'recent image')
if recentlyImported:
defaultDirectory = os.path.split(recentlyImported)[0]
elif mostRecentProjectPath:
defaultDirectory = os.path.split(mostRecentProjectPath)[0]
elif mostRecentImageFile:
defaultDirectory = os.path.split(mostRecentImageFile)[0]
else:
defaultDirectory = os.path.expanduser('~')
fileNames = DataSelectionGui.getImageFileNamesToOpen(
parent_widget, defaultDirectory)
fileNames = map(str, fileNames)
if not fileNames:
return
PreferencesManager().set('labeling', 'recently imported', fileNames[0])
try:
# Initialize operators
opImport = OpInputDataReader(parent=opLabels.parent)
opCache = OpBlockedArrayCache(parent=opLabels.parent)
opMetadataInjector = OpMetadataInjector(parent=opLabels.parent)
opReorderAxes = OpReorderAxes(parent=opLabels.parent)
# Set up the pipeline as follows:
#
# opImport --> opCache --> opMetadataInjector --------> opReorderAxes --(inject via setInSlot)--> labelInput
# / /
# User-specified axisorder labelInput.meta.axistags
opImport.WorkingDirectory.setValue(defaultDirectory)
opImport.FilePath.setValue(fileNames[0] if len(fileNames) ==
1 else os.path.pathsep.join(fileNames))
assert opImport.Output.ready()
opCache.Input.connect(opImport.Output)
opCache.CompressionEnabled.setValue(True)
assert opCache.Output.ready()
opMetadataInjector.Input.connect(opCache.Output)
metadata = opCache.Output.meta.copy()
opMetadataInjector.Metadata.setValue(metadata)
opReorderAxes.Input.connect(opMetadataInjector.Output)
# Transpose the axes for assignment to the labeling operator.
opReorderAxes.AxisOrder.setValue(writeSeeds.meta.getAxisKeys())
# We'll show a little window with a busy indicator while the data is loading
busy_dlg = QProgressDialog(parent=parent_widget)
busy_dlg.setLabelText("Importing Label Data...")
busy_dlg.setCancelButton(None)
busy_dlg.setMinimum(100)
busy_dlg.setMaximum(100)
def close_busy_dlg(*args):
QApplication.postEvent(busy_dlg, QCloseEvent())
# Load the data from file into our cache
# When it's done loading, close the progress dialog.
req = opCache.Output[:]
req.notify_finished(close_busy_dlg)
req.notify_failed(close_busy_dlg)
req.submit()
busy_dlg.exec_()
readData = req.result
maxLabels = len(labelingSlots.labelNames.value)
# Can't use return_counts feature because that requires numpy >= 1.9
#unique_read_labels, readLabelCounts = numpy.unique(readData, return_counts=True)
# This does the same as the above, albeit slower, and probably with more ram.
unique_read_labels = numpy.sort(vigra.analysis.unique(readData))
readLabelCounts = vigra_bincount(readData)[unique_read_labels]
labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
del readData
# Ask the user how to interpret the data.
settingsDlg = LabelImportOptionsDlg(parent_widget, fileNames,
opMetadataInjector.Output,
labelingSlots.labelInput,
labelInfo)
def handle_updated_axes():
# The user is specifying a new interpretation of the file's axes
updated_axisorder = str(settingsDlg.axesEdit.text())
metadata = opMetadataInjector.Metadata.value.copy()
metadata.axistags = vigra.defaultAxistags(updated_axisorder)
opMetadataInjector.Metadata.setValue(metadata)
if opReorderAxes._invalid_axes:
settingsDlg.buttonBox.button(
QDialogButtonBox.Ok).setEnabled(False)
# Red background
settingsDlg.axesEdit.setStyleSheet(
"QLineEdit { background: rgb(255, 128, 128);"
"selection-background-color: rgb(128, 128, 255); }")
settingsDlg.axesEdit.editingFinished.connect(handle_updated_axes)
# Initialize
handle_updated_axes()
dlg_result = settingsDlg.exec_()
if dlg_result != LabelImportOptionsDlg.Accepted:
return
# Get user's chosen label mapping from dlg
labelMapping = settingsDlg.labelMapping
# Get user's chosen offsets, ordered by the 'write seeds' slot
axes_5d = opReorderAxes.Output.meta.getAxisKeys()
tagged_offsets = collections.OrderedDict(
zip(axes_5d, [0] * len(axes_5d)))
tagged_offsets.update(
dict(
zip(opReorderAxes.Output.meta.getAxisKeys(),
settingsDlg.imageOffsets)))
imageOffsets = tagged_offsets.values()
# Optimization if mapping is identity
if labelMapping.keys() == labelMapping.values():
labelMapping = None
# This will be fast (it's already cached)
label_data = opReorderAxes.Output[:].wait()
# Map input labels to output labels
if labelMapping:
# There are other ways to do a relabeling (e.g skimage.segmentation.relabel_sequential)
# But this supports potentially huge values of unique_read_labels (in the billions),
# without needing GB of RAM.
mapping_indexes = numpy.searchsorted(unique_read_labels,
label_data)
new_labels = numpy.array(
[labelMapping[x] for x in unique_read_labels])
label_data[:] = new_labels[mapping_indexes]
label_roi = numpy.array(roiFromShape(opReorderAxes.Output.meta.shape))
label_roi += imageOffsets
label_slice = roiToSlice(*label_roi)
writeSeeds[label_slice] = label_data
finally:
opReorderAxes.cleanUp()
opMetadataInjector.cleanUp()
opCache.cleanUp()
opImport.cleanUp()
def import_labeling_layer(labelLayer, labelingSlots, parent_widget=None):
"""
Prompt the user for layer import settings, and perform the layer import.
:param labelLayer: The top label layer source
:param labelingSlots: An instance of LabelingGui.LabelingSlots
:param parent_widget: The Qt GUI parent object
"""
writeSeeds = labelingSlots.labelInput
assert isinstance(writeSeeds, lazyflow.graph.Slot), "slot is of type %r" % (type(writeSeeds))
opLabels = writeSeeds.getRealOperator()
assert isinstance(opLabels, lazyflow.graph.Operator), "slot's operator is of type %r" % (type(opLabels))
recentlyImported = PreferencesManager().get('labeling', 'recently imported')
mostRecentProjectPath = PreferencesManager().get('shell', 'recently opened')
mostRecentImageFile = PreferencesManager().get( 'DataSelection', 'recent image' )
if recentlyImported:
defaultDirectory = os.path.split(recentlyImported)[0]
elif mostRecentProjectPath:
defaultDirectory = os.path.split(mostRecentProjectPath)[0]
elif mostRecentImageFile:
defaultDirectory = os.path.split(mostRecentImageFile)[0]
else:
defaultDirectory = os.path.expanduser('~')
fileNames = DataSelectionGui.getImageFileNamesToOpen(parent_widget, defaultDirectory)
fileNames = list(map(str, fileNames))
if not fileNames:
return
PreferencesManager().set('labeling', 'recently imported', fileNames[0])
try:
# Initialize operators
opImport = OpInputDataReader( parent=opLabels.parent )
opCache = OpBlockedArrayCache( parent=opLabels.parent )
opMetadataInjector = OpMetadataInjector( parent=opLabels.parent )
opReorderAxes = OpReorderAxes( parent=opLabels.parent )
# Set up the pipeline as follows:
#
# opImport --> (opCache) --> opMetadataInjector --------> opReorderAxes --(inject via setInSlot)--> labelInput
# / /
# User-specified axisorder labelInput.meta.axistags
opImport.WorkingDirectory.setValue(defaultDirectory)
opImport.FilePath.setValue(fileNames[0] if len(fileNames) == 1 else
os.path.pathsep.join(fileNames))
assert opImport.Output.ready()
maxLabels = len(labelingSlots.labelNames.value)
# We don't bother with counting the label pixels
# (and caching the data) if it's big (1 GB)
if numpy.prod(opImport.Output.meta.shape) > 1e9:
reading_slot = opImport.Output
# For huge data, we don't go through and search for the pixel values,
# because that takes an annoyingly long amount of time.
# Instead, we make the reasonable assumption that the input labels are already 1,2,3..N
# and we don't tell the user what the label pixel counts are.
unique_read_labels = numpy.array(list(range(maxLabels+1)))
readLabelCounts = numpy.array([-1]*(maxLabels+1))
labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
else:
opCache.Input.connect( opImport.Output )
opCache.CompressionEnabled.setValue(True)
assert opCache.Output.ready()
reading_slot = opCache.Output
# We'll show a little window with a busy indicator while the data is loading
busy_dlg = QProgressDialog(parent=parent_widget)
busy_dlg.setLabelText("Scanning Label Data...")
busy_dlg.setCancelButton(None)
busy_dlg.setMinimum(100)
busy_dlg.setMaximum(100)
def close_busy_dlg(*args):
QApplication.postEvent(busy_dlg, QCloseEvent())
# Load the data from file into our cache
# When it's done loading, close the progress dialog.
req = reading_slot[:]
req.notify_finished( close_busy_dlg )
req.notify_failed( close_busy_dlg )
req.submit()
busy_dlg.exec_()
readData = req.result
# Can't use return_counts feature because that requires numpy >= 1.9
#unique_read_labels, readLabelCounts = numpy.unique(readData, return_counts=True)
# This does the same as the above, albeit slower, and probably with more ram.
bincounts = chunked_bincount(readData)
unique_read_labels = bincounts.nonzero()[0].astype(readData.dtype, copy=False)
readLabelCounts = bincounts[unique_read_labels]
labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
del readData
opMetadataInjector.Input.connect( reading_slot )
metadata = reading_slot.meta.copy()
opMetadataInjector.Metadata.setValue( metadata )
opReorderAxes.Input.connect( opMetadataInjector.Output )
# Transpose the axes for assignment to the labeling operator.
opReorderAxes.AxisOrder.setValue( writeSeeds.meta.getAxisKeys() )
# Ask the user how to interpret the data.
settingsDlg = LabelImportOptionsDlg( parent_widget,
fileNames, opMetadataInjector.Output,
labelingSlots.labelInput, labelInfo )
def handle_updated_axes():
# The user is specifying a new interpretation of the file's axes
updated_axisorder = str(settingsDlg.axesEdit.text())
metadata = opMetadataInjector.Metadata.value.copy()
metadata.axistags = vigra.defaultAxistags(updated_axisorder)
opMetadataInjector.Metadata.setValue( metadata )
if opReorderAxes._invalid_axes:
settingsDlg.buttonBox.button(QDialogButtonBox.Ok).setEnabled(False)
# Red background
settingsDlg.axesEdit.setStyleSheet("QLineEdit { background: rgb(255, 128, 128);"
"selection-background-color: rgb(128, 128, 255); }")
settingsDlg.axesEdit.editingFinished.connect( handle_updated_axes )
# Initialize
handle_updated_axes()
dlg_result = settingsDlg.exec_()
if dlg_result != LabelImportOptionsDlg.Accepted:
return
# Get user's chosen label mapping from dlg
labelMapping = settingsDlg.labelMapping
# Get user's chosen offsets, ordered by the 'write seeds' slot
axes_5d = opReorderAxes.Output.meta.getAxisKeys()
tagged_offsets = collections.OrderedDict( list(zip( axes_5d, [0]*len(axes_5d) )) )
tagged_offsets.update( dict( list(zip( opReorderAxes.Output.meta.getAxisKeys(), settingsDlg.imageOffsets )) ) )
imageOffsets = list(tagged_offsets.values())
# Optimization if mapping is identity
if list(labelMapping.keys()) == list(labelMapping.values()):
labelMapping = None
# If the data was already cached, this will be fast.
label_data = opReorderAxes.Output[:].wait()
# Map input labels to output labels
if labelMapping:
# There are other ways to do a relabeling (e.g skimage.segmentation.relabel_sequential)
# But this supports potentially huge values of unique_read_labels (in the billions),
# without needing GB of RAM.
mapping_indexes = numpy.searchsorted(unique_read_labels, label_data)
new_labels = numpy.array([labelMapping[x] for x in unique_read_labels])
label_data[:] = new_labels[mapping_indexes]
label_roi = numpy.array( roiFromShape(opReorderAxes.Output.meta.shape) )
label_roi += imageOffsets
label_slice = roiToSlice(*label_roi)
writeSeeds[label_slice] = label_data
finally:
opReorderAxes.cleanUp()
opMetadataInjector.cleanUp()
opCache.cleanUp()
opImport.cleanUp()
