コード例 #1
0
    def __init__(self, *args, **kwargs):
        super(OpNansheExtractF0Cached, self).__init__(*args, **kwargs)

        self.opExtractF0 = OpNansheExtractF0(parent=self, cache_f0=True)

        self.opExtractF0.HalfWindowSize.connect(self.HalfWindowSize)
        self.opExtractF0.WhichQuantile.connect(self.WhichQuantile)
        self.opExtractF0.TemporalSmoothingGaussianFilterStdev.connect(
            self.TemporalSmoothingGaussianFilterStdev)
        self.opExtractF0.TemporalSmoothingGaussianFilterWindowSize.connect(
            self.TemporalSmoothingGaussianFilterWindowSize)
        self.opExtractF0.SpatialSmoothingGaussianFilterStdev.connect(
            self.SpatialSmoothingGaussianFilterStdev)
        self.opExtractF0.SpatialSmoothingGaussianFilterWindowSize.connect(
            self.SpatialSmoothingGaussianFilterWindowSize)
        self.opExtractF0.BiasEnabled.connect(self.BiasEnabled)
        self.opExtractF0.Bias.connect(self.Bias)

        self.opCache_dF_F = OpBlockedArrayCache(parent=self)
        self.opCache_dF_F.fixAtCurrent.setValue(False)

        self.opCache_F0 = OpBlockedArrayCache(parent=self)
        self.opCache_F0.fixAtCurrent.setValue(False)

        self.opExtractF0.Input.connect(self.Input)
        self.opCache_F0.Input.connect(self.opExtractF0.F0)
        self.opCache_dF_F.Input.connect(self.opExtractF0.dF_F)

        self.F0.connect(self.opExtractF0.F0)
        self.dF_F.connect(self.opExtractF0.dF_F)
コード例 #2
0
    def testCleanup(self):
        try:
            CacheMemoryManager().disable()

            op = OpBlockedArrayCache(graph=self.opProvider.graph)
            op.Input.connect(self.opProvider.Output)
            s = self.opProvider.Output.meta.shape
            op.innerBlockShape.setValue(s)
            op.outerBlockShape.setValue(s)
            op.fixAtCurrent.setValue(False)
            x = op.Output[...].wait()
            op.Input.disconnect()
            op.cleanUp()

            r = weakref.ref(op)
            del op
            gc.collect()
            ref = r()
            if ref is not None:
                for i, o in enumerate(gc.get_referrers(ref)):
                    print "Object", i, ":", type(o), ":", o
                
            assert r() is None, "OpBlockedArrayCache was not cleaned up correctly"
        finally:
            CacheMemoryManager().enable()
コード例 #3
0
    def testCleanup(self):
        try:
            CacheMemoryManager().disable()

            op = OpBlockedArrayCache(graph=self.opProvider.graph)
            op.Input.connect(self.opProvider.Output)
            s = self.opProvider.Output.meta.shape
            op.innerBlockShape.setValue(s)
            op.outerBlockShape.setValue(s)
            op.fixAtCurrent.setValue(False)
            x = op.Output[...].wait()
            op.Input.disconnect()
            op.cleanUp()

            r = weakref.ref(op)
            del op
            gc.collect()
            ref = r()
            if ref is not None:
                for i, o in enumerate(gc.get_referrers(ref)):
                    print "Object", i, ":", type(o), ":", o

            assert r(
            ) is None, "OpBlockedArrayCache was not cleaned up correctly"
        finally:
            CacheMemoryManager().enable()
コード例 #4
0
    def __init__(self, *args, **kwargs):

        super(OpDLClassification, self).__init__(*args, **kwargs)

        self.predict = OpPixelwiseClassifierPredict(parent=self)
        self.predict.name = "OpClassifierPredict"
        self.predict.Image.connect(self.InputImage)
        self.predict.Classifier.connect(self.Classifier)
        self.predict.LabelsCount.connect(self.NumClasses)
        self.PredictionProbabilities.connect(self.predict.PMaps)

        self.prediction_cache = OpBlockedArrayCache(parent=self)
        self.prediction_cache.name = "BlockedArrayCache"
        self.prediction_cache.inputs["Input"].connect(self.predict.PMaps)
        self.prediction_cache.BlockShape.connect(self.BlockShape)
        self.prediction_cache.inputs["fixAtCurrent"].connect(self.FreezePredictions)
        self.CachedPredictionProbabilities.connect(self.prediction_cache.Output)

        self.opPredictionSlicer = OpMultiArraySlicer2(parent=self)
        self.opPredictionSlicer.name = "opPredictionSlicer"
        self.opPredictionSlicer.Input.connect(self.prediction_cache.Output)
        self.opPredictionSlicer.AxisFlag.setValue("c")
        self.PredictionProbabilityChannels.connect(self.opPredictionSlicer.Slices)

        self.opSegmentor = OpMaxChannelIndicatorOperator(parent=self)  # IMPROVEME: this "max channel" works for binary classification, but not when we have more than 2 classes
        self.opSegmentor.Input.connect(self.prediction_cache.Output)
        self.Segmentation.connect(self.opSegmentor.Output)

        self.opSegmentationSlicer = OpMultiArraySlicer2(parent=self)
        self.opSegmentationSlicer.name = "opSegmentationSlicer"
        self.opSegmentationSlicer.Input.connect(self.opSegmentor.Output)
        self.opSegmentationSlicer.AxisFlag.setValue("c")
        self.SegmentationChannels.connect(self.opSegmentationSlicer.Slices)
コード例 #5
0
    def testBlockedCacheHandling(self, cacheMemoryManager):
        n, k = 10, 5
        vol = np.zeros((n,) * 5, dtype=np.uint8)
        vol = vigra.taggedView(vol, axistags="txyzc")

        g = Graph()
        pipe = OpArrayPiperWithAccessCount(graph=g)
        cache = OpBlockedArrayCache(graph=g)

        # restrict cache memory to 0 Byte
        Memory.setAvailableRamCaches(0)

        # set to frequent cleanup
        cacheMemoryManager.setRefreshInterval(0.01)
        cacheMemoryManager.enable()

        cache.BlockShape.setValue((k,) * 5)
        cache.Input.connect(pipe.Output)
        pipe.Input.setValue(vol)

        a = pipe.accessCount
        cache.Output[...].wait()
        b = pipe.accessCount
        assert b > a, "did not cache"

        # let the manager clean up
        cacheMemoryManager.enable()
        time.sleep(0.5)
        gc.collect()

        cache.Output[...].wait()
        c = pipe.accessCount
        assert c > b, "did not clean up"
コード例 #6
0
    def setupOutputs(self):
        self.shape = self.inputs["Input"].meta.shape
        self._outerShapes = self.inputs["outerBlockShape"].value
        self._innerShapes = self.inputs["innerBlockShape"].value

        for blockshape in self._innerShapes + self._outerShapes:
            if len(blockshape) != len(self.Input.meta.shape):
                self.Output.meta.NOTREADY = True
                return

        # FIXME: This is wrong: Shouldn't it actually compare the new inner block shape with the old one?
        if len(self._innerShapes) != len(self._innerOps):
            # Clean up previous inner operators
            for slot in self.InnerOutputs:
                slot.disconnect()
            for o in self._innerOps:
                o.cleanUp()

            self._innerOps = []

            for i, innershape in enumerate(self._innerShapes):
                op = OpBlockedArrayCache(parent=self)
                op.inputs["fixAtCurrent"].connect(self.inputs["fixAtCurrent"])
                op.BypassModeEnabled.connect(self.BypassModeEnabled)
                op.CompressionEnabled.connect(self.CompressionEnabled)
                self._innerOps.append(op)

                op.inputs["Input"].connect(self.inputs["Input"])

                # Forward "value changed" notifications to our own output
                op.Output.notifyValueChanged(self.Output._sig_value_changed)

        for i, innershape in enumerate(self._innerShapes):
            op = self._innerOps[i]
            op.inputs["innerBlockShape"].setValue(innershape)
            op.inputs["outerBlockShape"].setValue(self._outerShapes[i])

        self.Output.meta.assignFrom(self.Input.meta)

        # Estimate ram usage
        ram_per_pixel = 0
        if self.Output.meta.dtype == object or self.Output.meta.dtype == numpy.object_:
            ram_per_pixel = sys.getsizeof(None)
        elif numpy.issubdtype(self.Output.meta.dtype, numpy.dtype):
            ram_per_pixel = self.Output.meta.dtype().nbytes

        tagged_shape = self.Output.meta.getTaggedShape()
        if 'c' in tagged_shape:
            ram_per_pixel *= float(tagged_shape['c'])

        if self.Output.meta.ram_usage_per_requested_pixel is not None:
            ram_per_pixel = max(ram_per_pixel,
                                self.Output.meta.ram_usage_per_requested_pixel)

        self.Output.meta.ram_usage_per_requested_pixel = ram_per_pixel

        # We also provide direct access to each of our inner cache outputs.
        self.InnerOutputs.resize(len(self._innerOps))
        for i, slot in enumerate(self.InnerOutputs):
            slot.connect(self._innerOps[i].Output)
コード例 #7
0
    def __init__(self):
        # Set memory and number of threads here
        #lazyflow.request.Request.reset_thread_pool(2)
        #Memory.setAvailableRam(500*1024**2)

        binary_img = binaryImage()
        raw_img = rawImage()

        g = Graph()

        # Reorder axis operators
        self.op5Raw = OpReorderAxes(graph=g)
        self.op5Raw.AxisOrder.setValue("txyzc")
        #self.op5Raw.Input.connect(self.opReaderRaw.OutputImage)#self.opReaderRaw.OutputImage)
        self.op5Raw.Input.setValue(raw_img)

        self.op5Binary = OpReorderAxes(graph=g)
        self.op5Binary.AxisOrder.setValue("txyzc")
        #self.op5Binary.Input.connect(self.opReaderBinary.OutputImage)
        self.op5Binary.Input.setValue(binary_img)

        # Cache operators
        self.opCacheRaw = OpBlockedArrayCache(graph=g)
        self.opCacheRaw.Input.connect(self.op5Raw.Output)
        self.opCacheRaw.BlockShape.setValue((1, ) +
                                            self.op5Raw.Output.meta.shape[1:])

        self.opCacheBinary = OpBlockedArrayCache(graph=g)
        self.opCacheBinary.Input.connect(self.op5Binary.Output)
        self.opCacheBinary.BlockShape.setValue(
            (1, ) + self.op5Binary.Output.meta.shape[1:])

        # Label volume operator
        self.opLabel = OpLabelVolume(graph=g)
        self.opLabel.Input.connect(self.op5Binary.Output)
        #self.opLabel.Input.connect(self.opCacheBinary.Output)

        # Object extraction
        self.opObjectExtraction = OpObjectExtraction(graph=g)
        self.opObjectExtraction.RawImage.connect(self.op5Raw.Output)
        self.opObjectExtraction.BinaryImage.connect(self.op5Binary.Output)
        self.opObjectExtraction.Features.setValue(FEATURES)

        # Simplified object features operator (No overhead)
        self.opObjectFeaturesSimp = OpObjectFeaturesSimplified(graph=g)
        self.opObjectFeaturesSimp.RawVol.connect(self.opCacheRaw.Output)
        self.opObjectFeaturesSimp.BinaryVol.connect(self.opCacheBinary.Output)
コード例 #8
0
    def __init__(self, *args, **kwargs):
        super(OpCachedTiledVolumeReader, self).__init__(*args, **kwargs)
        self._opReader = OpTiledVolumeReader(parent=self)
        self._opReader.DescriptionFilePath.connect(self.DescriptionFilePath)

        self.UncachedOutput.connect(self._opReader.Output)

        self._opCache = OpBlockedArrayCache(parent=self)
        self._opCache.Input.connect(self._opReader.Output)
        self._opCache.fixAtCurrent.setValue(False)

        self.CachedOutput.connect(self._opCache.Output)
コード例 #9
0
    def __init__(self, *args, **kwargs):
        super(OpColorizeLabelImageCached, self).__init__(*args, **kwargs)

        self.opColorizeLabelImage = OpColorizeLabelImage(parent=self)
        self.opColorizeLabelImage.NumColors.connect(self.NumColors)

        self.opCache = OpBlockedArrayCache(parent=self)
        self.opCache.fixAtCurrent.setValue(False)

        self.opColorizeLabelImage.Input.connect(self.Input)
        self.opCache.Input.connect(self.opColorizeLabelImage.Output)
        self.Output.connect(self.opCache.Output)
コード例 #10
0
    def setupOutputs(self):
        self.shape = self.inputs["Input"].meta.shape

        for blockshape in self.BlockShape.value:
            if len(blockshape) != len(self.Input.meta.shape):
                self.Output.meta.NOTREADY = True
                return

        if self._blockshapes != self.BlockShape.value:
            # Clean up previous inner operators
            for slot in self.InnerOutputs:
                slot.disconnect()
            for o in self._innerOps:
                o.cleanUp()

            self._innerOps = []
            self._blockshapes = self.BlockShape.value

            for i, innershape in enumerate(self._blockshapes):
                op = OpBlockedArrayCache(parent=self)
                op.inputs["fixAtCurrent"].connect(self.inputs["fixAtCurrent"])
                op.BypassModeEnabled.connect(self.BypassModeEnabled)
                op.CompressionEnabled.connect(self.CompressionEnabled)
                self._innerOps.append(op)

                op.inputs["Input"].connect(self.inputs["Input"])

                # Forward "value changed" notifications to our own output
                op.Output.notifyValueChanged(self.Output._sig_value_changed)

        for i, innershape in enumerate(self._blockshapes):
            op = self._innerOps[i]
            op.inputs["BlockShape"].setValue(innershape)

        self.Output.meta.assignFrom(self.Input.meta)

        # Estimate ram usage
        ram_per_pixel = get_ram_per_element(self.Output.meta.dtype)

        tagged_shape = self.Output.meta.getTaggedShape()
        if "c" in tagged_shape:
            ram_per_pixel *= float(tagged_shape["c"])

        if self.Output.meta.ram_usage_per_requested_pixel is not None:
            ram_per_pixel = max(ram_per_pixel,
                                self.Output.meta.ram_usage_per_requested_pixel)

        self.Output.meta.ram_usage_per_requested_pixel = ram_per_pixel

        # We also provide direct access to each of our inner cache outputs.
        self.InnerOutputs.resize(len(self._innerOps))
        for i, slot in enumerate(self.InnerOutputs):
            slot.connect(self._innerOps[i].Output)
コード例 #11
0
    def __init__(self, *args, **kwargs):
        super(OpConvertTypeCached, self).__init__(*args, **kwargs)

        self.opConvertType = OpConvertType(parent=self)

        self.opConvertType.Dtype.connect(self.Dtype)

        self.opCache = OpBlockedArrayCache(parent=self)
        self.opCache.fixAtCurrent.setValue(False)

        self.opConvertType.Input.connect(self.Input)
        self.opCache.Input.connect(self.opConvertType.Output)
        self.Output.connect(self.opCache.Output)
コード例 #12
0
    def __init__(self, *args, **kwargs):
        super(OpMeanProjectionCached, self).__init__(*args, **kwargs)

        self.opMeanProjection = OpMeanProjection(parent=self)

        self.opMeanProjection.Axis.connect(self.Axis)

        self.opCache = OpBlockedArrayCache(parent=self)
        self.opCache.fixAtCurrent.setValue(False)

        self.opMeanProjection.Input.connect(self.Input)
        self.opCache.Input.connect(self.opMeanProjection.Output)
        self.Output.connect(self.opCache.Output)
コード例 #13
0
    def __init__(self, *args, **kwargs):
        super(OpNansheWaveletTransformCached, self).__init__(*args, **kwargs)

        self.opWaveletTransform = OpNansheWaveletTransform(parent=self)

        self.opWaveletTransform.Scale.connect(self.Scale)

        self.opCache = OpBlockedArrayCache(parent=self)
        self.opCache.fixAtCurrent.setValue(False)

        self.opWaveletTransform.Input.connect(self.Input)
        self.opCache.Input.connect(self.opWaveletTransform.Output)
        self.Output.connect(self.opCache.Output)
コード例 #14
0
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.RESTfulReader = OpRESTfulPrecomputedChunkedVolumeReaderNoCache(parent=self)
        self.RESTfulReader.BaseUrl.connect(self.BaseUrl)
        self.AvailableScales.connect(self.RESTfulReader.AvailableScales)
        self.RESTfulReader.Scale.backpropagate_values = True
        self.RESTfulReader.Scale.connect(self.Scale)


        self.cache = OpBlockedArrayCache(parent=self)
        self.cache.name = "input_image_cache"
        self.cache.fixAtCurrent.connect(self.fixAtCurrent)
        self.cache.Input.connect(self.RESTfulReader.Output)
        self.Output.connect(self.cache.Output)
コード例 #15
0
    def __init__(self, *args, **kwargs):
        super( OpNansheRemoveZeroedLinesCached, self ).__init__( *args, **kwargs )

        self.opRemoveZeroedLines = OpNansheRemoveZeroedLines(parent=self)

        self.opRemoveZeroedLines.ErosionShape.connect(self.ErosionShape)
        self.opRemoveZeroedLines.DilationShape.connect(self.DilationShape)


        self.opCache = OpBlockedArrayCache(parent=self)
        self.opCache.fixAtCurrent.setValue(False)

        self.opRemoveZeroedLines.Input.connect( self.Input )
        self.opCache.Input.connect( self.opRemoveZeroedLines.Output )
        self.Output.connect( self.opCache.Output )
コード例 #16
0
    def setup_method(self, method):
        self.dataShape = (1, 100, 100, 10, 1)
        self.data = numpy.random.randint(0, 256, size=self.dataShape)
        self.data = self.data.astype(numpy.uint32)
        self.data = self.data.view(vigra.VigraArray)
        self.data.axistags = vigra.defaultAxistags("txyzc")

        graph = Graph()
        opProvider = OpArrayPiperWithAccessCount(graph=graph)
        opProvider.Input.setValue(self.data)
        self.opProvider = opProvider

        opCache = OpBlockedArrayCache(graph=graph)
        opCache.Input.connect(opProvider.Output)
        opCache.BlockShape.setValue((20, 20, 20, 20, 20))
        opCache.fixAtCurrent.setValue(False)
        self.opCache = opCache
コード例 #17
0
    def testBasic(self):
        features = numpy.indices((100, 100)).astype(numpy.float32) + 0.5
        features = numpy.rollaxis(features, 0, 3)
        features = vigra.taggedView(features, "xyc")

        labels = numpy.zeros((100, 100, 1), dtype=numpy.uint8)
        labels = vigra.taggedView(labels, "xyc")

        labels[10, 10] = 1
        labels[10, 11] = 1
        labels[20, 20] = 2
        labels[20, 21] = 2

        graph = Graph()
        # Use a cache for the labels so we can control the ideal_blockshape
        # This ensures that the blockwise behavior is tested, even though we're
        # testing with tiny data that would normally fall into a single block.
        opLabelCache = OpBlockedArrayCache(graph=graph)
        opLabelCache.BlockShape.setValue((10, 10, 1))
        opLabelCache.Input.setValue(labels)

        opFeatureMatrixCache = OpFeatureMatrixCache(graph=graph)
        opFeatureMatrixCache.LabelImage.connect(opLabelCache.Output)
        opFeatureMatrixCache.FeatureImage.setValue(features)

        labels_and_features = opFeatureMatrixCache.LabelAndFeatureMatrix.value
        assert labels_and_features.shape == (
            0, 3), "Empty feature matrix has wrong shape: {}".format(
                labels_and_features.shape)

        opFeatureMatrixCache.LabelImage.setDirty(numpy.s_[10:11, 10:12])
        opFeatureMatrixCache.LabelImage.setDirty(numpy.s_[20:21, 20:22])
        opFeatureMatrixCache.LabelImage.setDirty(numpy.s_[30:31, 30:32])

        labels_and_features = opFeatureMatrixCache.LabelAndFeatureMatrix.value
        assert labels_and_features.shape == (4, 3)
        assert (labels_and_features[:, 0] == 1).sum() == 2
        assert (labels_and_features[:, 0] == 2).sum() == 2

        # Can't check for equality because feature blocks can be in a random order.
        # Just check that all features are present, regardless of order.
        for feature_vec in [[10.5, 10.5], [10.5, 11.5], [20.5, 20.5],
                            [20.5, 21.5]]:
            assert feature_vec in labels_and_features[:, 1:]
コード例 #18
0
    def __init__(self, *args, **kwargs):
        super( OpNansheEstimateF0Cached, self ).__init__( *args, **kwargs )

        self.opEstimateF0 = OpNansheEstimateF0(parent=self)

        self.opEstimateF0.HalfWindowSize.connect(self.HalfWindowSize)
        self.opEstimateF0.WhichQuantile.connect(self.WhichQuantile)
        self.opEstimateF0.TemporalSmoothingGaussianFilterStdev.connect(self.TemporalSmoothingGaussianFilterStdev)
        self.opEstimateF0.TemporalSmoothingGaussianFilterWindowSize.connect(self.TemporalSmoothingGaussianFilterWindowSize)
        self.opEstimateF0.SpatialSmoothingGaussianFilterStdev.connect(self.SpatialSmoothingGaussianFilterStdev)
        self.opEstimateF0.SpatialSmoothingGaussianFilterWindowSize.connect(self.SpatialSmoothingGaussianFilterWindowSize)

        self.opCache_F0 = OpBlockedArrayCache(parent=self)
        self.opCache_F0.fixAtCurrent.setValue(False)

        self.opEstimateF0.Input.connect( self.Input )
        self.opCache_F0.Input.connect( self.opEstimateF0.Output )

        self.Output.connect( self.opCache_F0.Output )
コード例 #19
0
    def setup_method(self, method):
        self.dataShape = (1, 100, 100, 10, 1)
        self.data = (numpy.random.random(self.dataShape) * 100).astype(int)
        self.data = numpy.ma.masked_array(
            self.data, mask=numpy.ma.getmaskarray(self.data), fill_value=numpy.iinfo(int).max, shrink=False
        )
        self.data[:, 0] = numpy.ma.masked

        graph = Graph()
        opProvider = OpArrayPiperWithAccessCount(graph=graph)
        opProvider.Input.meta.axistags = vigra.defaultAxistags("txyzc")
        opProvider.Input.meta.has_mask = True
        opProvider.Input.setValue(self.data)
        self.opProvider = opProvider

        opCache = OpBlockedArrayCache(graph=graph)
        opCache.Input.connect(opProvider.Output)
        opCache.BlockShape.setValue((20, 20, 20, 20, 20))
        opCache.fixAtCurrent.setValue(False)
        self.opCache = opCache
コード例 #20
0
    def __init__(self, *args, **kwargs):

        super(OpNNClassification, self).__init__(*args, **kwargs)

        self.predict = OpPixelwiseClassifierPredict(parent=self)
        self.predict.name = "OpClassifierPredict"
        self.predict.Image.connect(self.InputImage)
        self.predict.Classifier.connect(self.Classifier)
        self.predict.LabelsCount.connect(self.NumClasses)
        self.PredictionProbabilities.connect(self.predict.PMaps)

        self.prediction_cache = OpBlockedArrayCache(parent=self)
        self.prediction_cache.name = "BlockedArrayCache"
        self.prediction_cache.inputs["Input"].connect(self.predict.PMaps)
        self.prediction_cache.BlockShape.connect(self.BlockShape)
        self.prediction_cache.inputs["fixAtCurrent"].connect(self.FreezePredictions)
        self.CachedPredictionProbabilities.connect(self.prediction_cache.Output)

        self.opPredictionSlicer = OpMultiArraySlicer2(parent=self)
        self.opPredictionSlicer.name = "opPredictionSlicer"
        self.opPredictionSlicer.Input.connect(self.prediction_cache.Output)
        self.opPredictionSlicer.AxisFlag.setValue("c")
        self.PredictionProbabilityChannels.connect(self.opPredictionSlicer.Slices)
コード例 #21
0
    def setupOutputs(self):
        self.shape = self.inputs["Input"].meta.shape
        self._outerShapes = self.inputs["outerBlockShape"].value
        self._innerShapes = self.inputs["innerBlockShape"].value

        # FIXME: This is wrong: Shouldn't it actually compare the new inner block shape with the old one?
        if len(self._innerShapes) != len(self._innerOps):
            # Clean up previous inner operators
            for slot in self.InnerOutputs:
                slot.disconnect()
            for o in self._innerOps:
                o.cleanUp()

            self._innerOps = []

            for i, innershape in enumerate(self._innerShapes):
                op = OpBlockedArrayCache(parent=self)
                op.inputs["fixAtCurrent"].connect(self.inputs["fixAtCurrent"])
                self._innerOps.append(op)

                op.inputs["Input"].connect(self.inputs["Input"])

                # Forward "value changed" notifications to our own output
                op.Output.notifyValueChanged(self.Output._sig_value_changed)

        for i, innershape in enumerate(self._innerShapes):
            op = self._innerOps[i]
            op.inputs["innerBlockShape"].setValue(innershape)
            op.inputs["outerBlockShape"].setValue(self._outerShapes[i])

        self.Output.meta.assignFrom(self.Input.meta)

        # We also provide direct access to each of our inner cache outputs.
        self.InnerOutputs.resize(len(self._innerOps))
        for i, slot in enumerate(self.InnerOutputs):
            slot.connect(self._innerOps[i].Output)
コード例 #22
0
    def testBadMemoryConditions(self):
        """
        TestCacheMemoryManager.testBadMemoryConditions

        This test is a proof of the proposition in
            https://github.com/ilastik/lazyflow/issue/185
        which states that, given certain memory constraints, the cache
        cleanup strategy in use is inefficient. An advanced strategy
        should pass the test.
        """

        mgr = _CacheMemoryManager()
        mgr.setRefreshInterval(0.01)
        mgr.enable()

        d = 2
        tags = "xy"

        shape = (999,) * d
        blockshape = (333,) * d

        # restrict memory for computation to one block (including fudge
        # factor 2 of bigRequestStreamer)
        cacheMem = np.prod(shape)
        Memory.setAvailableRam(np.prod(blockshape) * 2 + cacheMem)

        # restrict cache memory to the whole volume
        Memory.setAvailableRamCaches(cacheMem)

        # to ease observation, do everything single threaded
        Request.reset_thread_pool(num_workers=1)

        x = np.zeros(shape, dtype=np.uint8)
        x = vigra.taggedView(x, axistags=tags)

        g = Graph()
        pipe = OpArrayPiperWithAccessCount(graph=g)
        pipe.Input.setValue(x)
        pipe.Output.meta.ideal_blockshape = blockshape

        # simulate BlockedArrayCache behaviour without caching
        # cache = OpSplitRequestsBlockwise(True, graph=g)
        # cache.BlockShape.setValue(blockshape)
        # cache.Input.connect(pipe.Output)

        cache = OpBlockedArrayCache(graph=g)
        cache.Input.connect(pipe.Output)
        cache.BlockShape.setValue(blockshape)

        op = OpEnlarge(graph=g)
        op.Input.connect(cache.Output)

        split = OpSplitRequestsBlockwise(True, graph=g)
        split.BlockShape.setValue(blockshape)
        split.Input.connect(op.Output)
        streamer = BigRequestStreamer(split.Output, [(0,) * len(shape), shape])
        streamer.execute()

        # in the worst case, we have 4*4 + 4*6 + 9 = 49 requests to pipe
        # in the best case, we have 9
        np.testing.assert_equal(pipe.accessCount, 9)
コード例 #23
0
def import_labeling_layer(labelLayer, labelingSlots, parent_widget=None):
    """
    Prompt the user for layer import settings, and perform the layer import.
    :param labelLayer: The top label layer source
    :param labelingSlots: An instance of LabelingGui.LabelingSlots
    :param parent_widget: The Qt GUI parent object
    """
    writeSeeds = labelingSlots.labelInput
    assert isinstance(
        writeSeeds,
        lazyflow.graph.Slot), "slot is of type %r" % (type(writeSeeds))
    opLabels = writeSeeds.getRealOperator()
    assert isinstance(opLabels, lazyflow.graph.Operator
                      ), "slot's operator is of type %r" % (type(opLabels))

    fileNames = ImageFileDialog(
        parent_widget,
        preferences_group="labeling",
        preferences_setting="recently imported").getSelectedPaths()
    fileNames = list(map(str, fileNames))

    if not fileNames:
        return

    try:
        # Initialize operators
        opImport = OpInputDataReader(parent=opLabels.parent)
        opCache = OpBlockedArrayCache(parent=opLabels.parent)
        opMetadataInjector = OpMetadataInjector(parent=opLabels.parent)
        opReorderAxes = OpReorderAxes(parent=opLabels.parent)

        # Set up the pipeline as follows:
        #
        #   opImport --> (opCache) --> opMetadataInjector --------> opReorderAxes --(inject via setInSlot)--> labelInput
        #                             /                            /
        #     User-specified axisorder    labelInput.meta.axistags
        opImport.FilePath.setValue(fileNames[0] if len(fileNames) ==
                                   1 else os.path.pathsep.join(fileNames))
        assert opImport.Output.ready()

        maxLabels = len(labelingSlots.labelNames.value)

        # We don't bother with counting the label pixels
        # (and caching the data) if it's big (1 GB)
        if numpy.prod(opImport.Output.meta.shape) > 1e9:
            reading_slot = opImport.Output

            # For huge data, we don't go through and search for the pixel values,
            # because that takes an annoyingly long amount of time.
            # Instead, we make the reasonable assumption that the input labels are already 1,2,3..N
            # and we don't tell the user what the label pixel counts are.
            unique_read_labels = numpy.array(list(range(maxLabels + 1)))
            readLabelCounts = numpy.array([-1] * (maxLabels + 1))
            labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
        else:
            opCache.Input.connect(opImport.Output)
            opCache.CompressionEnabled.setValue(True)
            assert opCache.Output.ready()
            reading_slot = opCache.Output

            # We'll show a little window with a busy indicator while the data is loading
            busy_dlg = QProgressDialog(parent=parent_widget)
            busy_dlg.setLabelText("Scanning Label Data...")
            busy_dlg.setCancelButton(None)
            busy_dlg.setMinimum(100)
            busy_dlg.setMaximum(100)

            def close_busy_dlg(*args):
                QApplication.postEvent(busy_dlg, QCloseEvent())

            # Load the data from file into our cache
            # When it's done loading, close the progress dialog.
            req = reading_slot[:]
            req.notify_finished(close_busy_dlg)
            req.notify_failed(close_busy_dlg)
            req.submit()
            busy_dlg.exec_()

            readData = req.result

            # Can't use return_counts feature because that requires numpy >= 1.9
            # unique_read_labels, readLabelCounts = numpy.unique(readData, return_counts=True)

            # This does the same as the above, albeit slower, and probably with more ram.
            bincounts = chunked_bincount(readData)
            unique_read_labels = bincounts.nonzero()[0].astype(readData.dtype,
                                                               copy=False)
            readLabelCounts = bincounts[unique_read_labels]

            labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
            del readData

        opMetadataInjector.Input.connect(reading_slot)
        metadata = reading_slot.meta.copy()
        opMetadataInjector.Metadata.setValue(metadata)
        opReorderAxes.Input.connect(opMetadataInjector.Output)

        # Transpose the axes for assignment to the labeling operator.
        opReorderAxes.AxisOrder.setValue(writeSeeds.meta.getAxisKeys())

        # Ask the user how to interpret the data.
        settingsDlg = LabelImportOptionsDlg(parent_widget, fileNames,
                                            opMetadataInjector.Output,
                                            labelingSlots.labelInput,
                                            labelInfo)

        def handle_updated_axes():
            # The user is specifying a new interpretation of the file's axes
            updated_axisorder = str(settingsDlg.axesEdit.text())
            metadata = opMetadataInjector.Metadata.value.copy()
            metadata.axistags = vigra.defaultAxistags(updated_axisorder)
            opMetadataInjector.Metadata.setValue(metadata)

            if opReorderAxes._invalid_axes:
                settingsDlg.buttonBox.button(
                    QDialogButtonBox.Ok).setEnabled(False)
                # Red background
                settingsDlg.axesEdit.setStyleSheet(
                    "QLineEdit { background: rgb(255, 128, 128); selection-background-color: rgb(128, 128, 255); }"
                )

        settingsDlg.axesEdit.editingFinished.connect(handle_updated_axes)

        # Initialize
        handle_updated_axes()

        dlg_result = settingsDlg.exec_()
        if dlg_result != LabelImportOptionsDlg.Accepted:
            return

        # Get user's chosen label mapping from dlg
        labelMapping = settingsDlg.labelMapping

        # Get user's chosen offsets, ordered by the 'write seeds' slot
        axes_5d = opReorderAxes.Output.meta.getAxisKeys()
        tagged_offsets = collections.OrderedDict(
            list(zip(axes_5d, [0] * len(axes_5d))))
        tagged_offsets.update(
            dict(
                list(
                    zip(opReorderAxes.Output.meta.getAxisKeys(),
                        settingsDlg.imageOffsets))))
        imageOffsets = list(tagged_offsets.values())

        # Optimization if mapping is identity
        if list(labelMapping.keys()) == list(labelMapping.values()):
            labelMapping = None

        # If the data was already cached, this will be fast.
        label_data = opReorderAxes.Output[:].wait()

        # Map input labels to output labels
        if labelMapping:
            # There are other ways to do a relabeling (e.g skimage.segmentation.relabel_sequential)
            # But this supports potentially huge values of unique_read_labels (in the billions),
            # without needing GB of RAM.
            mapping_indexes = numpy.searchsorted(unique_read_labels,
                                                 label_data)
            new_labels = numpy.array(
                [labelMapping[x] for x in unique_read_labels])
            label_data[:] = new_labels[mapping_indexes]

        label_roi = numpy.array(roiFromShape(opReorderAxes.Output.meta.shape))
        label_roi += imageOffsets
        label_slice = roiToSlice(*label_roi)
        writeSeeds[label_slice] = label_data

    finally:
        opReorderAxes.cleanUp()
        opMetadataInjector.cleanUp()
        opCache.cleanUp()
        opImport.cleanUp()
コード例 #24
0
ファイル: labelingImport.py プロジェクト: nagyist/ilastik
def import_labeling_layer(labelLayer, labelingSlots, parent_widget=None):
    """
    Prompt the user for layer import settings, and perform the layer import.
    :param labelLayer: The top label layer source
    :param labelingSlots: An instance of LabelingGui.LabelingSlots
    :param parent_widget: The Qt GUI parent object
    """
    writeSeeds = labelingSlots.labelInput
    assert isinstance(
        writeSeeds,
        lazyflow.graph.Slot), "slot is of type %r" % (type(writeSeeds))
    opLabels = writeSeeds.getRealOperator()
    assert isinstance(opLabels, lazyflow.graph.Operator
                      ), "slot's operator is of type %r" % (type(opLabels))

    recentlyImported = PreferencesManager().get('labeling',
                                                'recently imported')
    mostRecentProjectPath = PreferencesManager().get('shell',
                                                     'recently opened')
    mostRecentImageFile = PreferencesManager().get('DataSelection',
                                                   'recent image')
    if recentlyImported:
        defaultDirectory = os.path.split(recentlyImported)[0]
    elif mostRecentProjectPath:
        defaultDirectory = os.path.split(mostRecentProjectPath)[0]
    elif mostRecentImageFile:
        defaultDirectory = os.path.split(mostRecentImageFile)[0]
    else:
        defaultDirectory = os.path.expanduser('~')

    fileNames = DataSelectionGui.getImageFileNamesToOpen(
        parent_widget, defaultDirectory)
    fileNames = map(str, fileNames)

    if not fileNames:
        return

    PreferencesManager().set('labeling', 'recently imported', fileNames[0])

    try:
        # Initialize operators
        opImport = OpInputDataReader(parent=opLabels.parent)
        opCache = OpBlockedArrayCache(parent=opLabels.parent)
        opMetadataInjector = OpMetadataInjector(parent=opLabels.parent)
        opReorderAxes = OpReorderAxes(parent=opLabels.parent)

        # Set up the pipeline as follows:
        #
        #   opImport --> opCache --> opMetadataInjector --------> opReorderAxes --(inject via setInSlot)--> labelInput
        #                           /                            /
        #   User-specified axisorder    labelInput.meta.axistags

        opImport.WorkingDirectory.setValue(defaultDirectory)
        opImport.FilePath.setValue(fileNames[0] if len(fileNames) ==
                                   1 else os.path.pathsep.join(fileNames))
        assert opImport.Output.ready()

        opCache.Input.connect(opImport.Output)
        opCache.CompressionEnabled.setValue(True)
        assert opCache.Output.ready()

        opMetadataInjector.Input.connect(opCache.Output)
        metadata = opCache.Output.meta.copy()
        opMetadataInjector.Metadata.setValue(metadata)
        opReorderAxes.Input.connect(opMetadataInjector.Output)

        # Transpose the axes for assignment to the labeling operator.
        opReorderAxes.AxisOrder.setValue(writeSeeds.meta.getAxisKeys())

        # We'll show a little window with a busy indicator while the data is loading
        busy_dlg = QProgressDialog(parent=parent_widget)
        busy_dlg.setLabelText("Importing Label Data...")
        busy_dlg.setCancelButton(None)
        busy_dlg.setMinimum(100)
        busy_dlg.setMaximum(100)

        def close_busy_dlg(*args):
            QApplication.postEvent(busy_dlg, QCloseEvent())

        # Load the data from file into our cache
        # When it's done loading, close the progress dialog.
        req = opCache.Output[:]
        req.notify_finished(close_busy_dlg)
        req.notify_failed(close_busy_dlg)
        req.submit()
        busy_dlg.exec_()

        readData = req.result

        maxLabels = len(labelingSlots.labelNames.value)

        # Can't use return_counts feature because that requires numpy >= 1.9
        #unique_read_labels, readLabelCounts = numpy.unique(readData, return_counts=True)

        # This does the same as the above, albeit slower, and probably with more ram.
        unique_read_labels = numpy.sort(vigra.analysis.unique(readData))
        readLabelCounts = vigra_bincount(readData)[unique_read_labels]

        labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
        del readData

        # Ask the user how to interpret the data.
        settingsDlg = LabelImportOptionsDlg(parent_widget, fileNames,
                                            opMetadataInjector.Output,
                                            labelingSlots.labelInput,
                                            labelInfo)

        def handle_updated_axes():
            # The user is specifying a new interpretation of the file's axes
            updated_axisorder = str(settingsDlg.axesEdit.text())
            metadata = opMetadataInjector.Metadata.value.copy()
            metadata.axistags = vigra.defaultAxistags(updated_axisorder)
            opMetadataInjector.Metadata.setValue(metadata)

            if opReorderAxes._invalid_axes:
                settingsDlg.buttonBox.button(
                    QDialogButtonBox.Ok).setEnabled(False)
                # Red background
                settingsDlg.axesEdit.setStyleSheet(
                    "QLineEdit { background: rgb(255, 128, 128);"
                    "selection-background-color: rgb(128, 128, 255); }")

        settingsDlg.axesEdit.editingFinished.connect(handle_updated_axes)

        # Initialize
        handle_updated_axes()

        dlg_result = settingsDlg.exec_()
        if dlg_result != LabelImportOptionsDlg.Accepted:
            return

        # Get user's chosen label mapping from dlg
        labelMapping = settingsDlg.labelMapping

        # Get user's chosen offsets, ordered by the 'write seeds' slot
        axes_5d = opReorderAxes.Output.meta.getAxisKeys()
        tagged_offsets = collections.OrderedDict(
            zip(axes_5d, [0] * len(axes_5d)))
        tagged_offsets.update(
            dict(
                zip(opReorderAxes.Output.meta.getAxisKeys(),
                    settingsDlg.imageOffsets)))
        imageOffsets = tagged_offsets.values()

        # Optimization if mapping is identity
        if labelMapping.keys() == labelMapping.values():
            labelMapping = None

        # This will be fast (it's already cached)
        label_data = opReorderAxes.Output[:].wait()

        # Map input labels to output labels
        if labelMapping:
            # There are other ways to do a relabeling (e.g skimage.segmentation.relabel_sequential)
            # But this supports potentially huge values of unique_read_labels (in the billions),
            # without needing GB of RAM.
            mapping_indexes = numpy.searchsorted(unique_read_labels,
                                                 label_data)
            new_labels = numpy.array(
                [labelMapping[x] for x in unique_read_labels])
            label_data[:] = new_labels[mapping_indexes]

        label_roi = numpy.array(roiFromShape(opReorderAxes.Output.meta.shape))
        label_roi += imageOffsets
        label_slice = roiToSlice(*label_roi)
        writeSeeds[label_slice] = label_data

    finally:
        opReorderAxes.cleanUp()
        opMetadataInjector.cleanUp()
        opCache.cleanUp()
        opImport.cleanUp()
コード例 #25
0
ファイル: labelingImport.py プロジェクト: ilastik/ilastik
def import_labeling_layer(labelLayer, labelingSlots, parent_widget=None):
    """
    Prompt the user for layer import settings, and perform the layer import.
    :param labelLayer: The top label layer source
    :param labelingSlots: An instance of LabelingGui.LabelingSlots
    :param parent_widget: The Qt GUI parent object
    """
    writeSeeds = labelingSlots.labelInput
    assert isinstance(writeSeeds, lazyflow.graph.Slot), "slot is of type %r" % (type(writeSeeds))
    opLabels = writeSeeds.getRealOperator()
    assert isinstance(opLabels, lazyflow.graph.Operator), "slot's operator is of type %r" % (type(opLabels))


    recentlyImported = PreferencesManager().get('labeling', 'recently imported')
    mostRecentProjectPath = PreferencesManager().get('shell', 'recently opened')
    mostRecentImageFile = PreferencesManager().get( 'DataSelection', 'recent image' )
    if recentlyImported:
        defaultDirectory = os.path.split(recentlyImported)[0]
    elif mostRecentProjectPath:
        defaultDirectory = os.path.split(mostRecentProjectPath)[0]
    elif mostRecentImageFile:
        defaultDirectory = os.path.split(mostRecentImageFile)[0]
    else:
        defaultDirectory = os.path.expanduser('~')

    fileNames = DataSelectionGui.getImageFileNamesToOpen(parent_widget, defaultDirectory)
    fileNames = list(map(str, fileNames))

    if not fileNames:
        return

    PreferencesManager().set('labeling', 'recently imported', fileNames[0])

    try:
        # Initialize operators
        opImport = OpInputDataReader( parent=opLabels.parent )
        opCache = OpBlockedArrayCache( parent=opLabels.parent )
        opMetadataInjector = OpMetadataInjector( parent=opLabels.parent )
        opReorderAxes = OpReorderAxes( parent=opLabels.parent )
    
        # Set up the pipeline as follows:
        #
        #   opImport --> (opCache) --> opMetadataInjector --------> opReorderAxes --(inject via setInSlot)--> labelInput
        #                             /                            /
        #     User-specified axisorder    labelInput.meta.axistags
    
        opImport.WorkingDirectory.setValue(defaultDirectory)
        opImport.FilePath.setValue(fileNames[0] if len(fileNames) == 1 else
                                   os.path.pathsep.join(fileNames))
        assert opImport.Output.ready()

        maxLabels = len(labelingSlots.labelNames.value)
    
        # We don't bother with counting the label pixels
        # (and caching the data) if it's big (1 GB)
        if numpy.prod(opImport.Output.meta.shape) > 1e9:
            reading_slot = opImport.Output
            
            # For huge data, we don't go through and search for the pixel values,
            # because that takes an annoyingly long amount of time.
            # Instead, we make the reasonable assumption that the input labels are already 1,2,3..N
            # and we don't tell the user what the label pixel counts are.
            unique_read_labels = numpy.array(list(range(maxLabels+1)))
            readLabelCounts = numpy.array([-1]*(maxLabels+1))
            labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
        else:    
            opCache.Input.connect( opImport.Output )
            opCache.CompressionEnabled.setValue(True)
            assert opCache.Output.ready()
            reading_slot = opCache.Output

            # We'll show a little window with a busy indicator while the data is loading
            busy_dlg = QProgressDialog(parent=parent_widget)
            busy_dlg.setLabelText("Scanning Label Data...")
            busy_dlg.setCancelButton(None)
            busy_dlg.setMinimum(100)
            busy_dlg.setMaximum(100)
            def close_busy_dlg(*args):
                QApplication.postEvent(busy_dlg, QCloseEvent())
        
            # Load the data from file into our cache
            # When it's done loading, close the progress dialog.
            req = reading_slot[:]
            req.notify_finished( close_busy_dlg )
            req.notify_failed( close_busy_dlg )
            req.submit()
            busy_dlg.exec_()
    
            readData = req.result

            # Can't use return_counts feature because that requires numpy >= 1.9
            #unique_read_labels, readLabelCounts = numpy.unique(readData, return_counts=True)
    
            # This does the same as the above, albeit slower, and probably with more ram.
            bincounts = chunked_bincount(readData)
            unique_read_labels = bincounts.nonzero()[0].astype(readData.dtype, copy=False)
            readLabelCounts = bincounts[unique_read_labels]
    
            labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
            del readData
    
        opMetadataInjector.Input.connect( reading_slot )
        metadata = reading_slot.meta.copy()
        opMetadataInjector.Metadata.setValue( metadata )
        opReorderAxes.Input.connect( opMetadataInjector.Output )

        # Transpose the axes for assignment to the labeling operator.
        opReorderAxes.AxisOrder.setValue( writeSeeds.meta.getAxisKeys() )

        # Ask the user how to interpret the data.
        settingsDlg = LabelImportOptionsDlg( parent_widget,
                                             fileNames, opMetadataInjector.Output,
                                             labelingSlots.labelInput, labelInfo )

        def handle_updated_axes():
            # The user is specifying a new interpretation of the file's axes
            updated_axisorder = str(settingsDlg.axesEdit.text())
            metadata = opMetadataInjector.Metadata.value.copy()
            metadata.axistags = vigra.defaultAxistags(updated_axisorder)
            opMetadataInjector.Metadata.setValue( metadata )
            
            if opReorderAxes._invalid_axes:
                settingsDlg.buttonBox.button(QDialogButtonBox.Ok).setEnabled(False)
                # Red background
                settingsDlg.axesEdit.setStyleSheet("QLineEdit { background: rgb(255, 128, 128);"
                                                   "selection-background-color: rgb(128, 128, 255); }")
        settingsDlg.axesEdit.editingFinished.connect( handle_updated_axes )
        
        # Initialize
        handle_updated_axes()

        dlg_result = settingsDlg.exec_()
        if dlg_result != LabelImportOptionsDlg.Accepted:
            return

        # Get user's chosen label mapping from dlg
        labelMapping = settingsDlg.labelMapping    

        # Get user's chosen offsets, ordered by the 'write seeds' slot
        axes_5d = opReorderAxes.Output.meta.getAxisKeys()
        tagged_offsets = collections.OrderedDict( list(zip( axes_5d, [0]*len(axes_5d) )) )
        tagged_offsets.update( dict( list(zip( opReorderAxes.Output.meta.getAxisKeys(), settingsDlg.imageOffsets )) ) )
        imageOffsets = list(tagged_offsets.values())

        # Optimization if mapping is identity
        if list(labelMapping.keys()) == list(labelMapping.values()):
            labelMapping = None

        # If the data was already cached, this will be fast.
        label_data = opReorderAxes.Output[:].wait()
        
        # Map input labels to output labels
        if labelMapping:
            # There are other ways to do a relabeling (e.g skimage.segmentation.relabel_sequential)
            # But this supports potentially huge values of unique_read_labels (in the billions),
            # without needing GB of RAM.
            mapping_indexes = numpy.searchsorted(unique_read_labels, label_data)
            new_labels = numpy.array([labelMapping[x] for x in unique_read_labels])
            label_data[:] = new_labels[mapping_indexes]

        label_roi = numpy.array( roiFromShape(opReorderAxes.Output.meta.shape) )
        label_roi += imageOffsets
        label_slice = roiToSlice(*label_roi)
        writeSeeds[label_slice] = label_data

    finally:
        opReorderAxes.cleanUp()
        opMetadataInjector.cleanUp()
        opCache.cleanUp()
        opImport.cleanUp()