class TestOpArrayPiperWithAccessCount7(object): def setUp(self): self.graph = Graph() self.operator_identity_1 = OpArrayPiperWithAccessCount( graph=self.graph) self.operator_identity_2 = OpArrayPiperWithAccessCount( graph=self.graph) self.operator_identity_1.Input.meta.axistags = vigra.AxisTags("txyzc") self.operator_identity_2.Input.meta.axistags = vigra.AxisTags("txyzc") def test1(self): # Explicitly set has_mask for the input self.operator_identity_1.Input.meta.has_mask = True self.operator_identity_1.Output.meta.has_mask = True # Generate a dataset and grab chunks of it from the operator. The result should be the same as above. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False) # Try to connect the compatible operators. self.operator_identity_2.Input.connect(self.operator_identity_1.Output) self.operator_identity_1.Input.setValue(data) output = self.operator_identity_2.Output[None].wait() assert (self.operator_identity_1.accessCount == 1) assert (self.operator_identity_2.accessCount == 1) assert ((data == output).all()) assert (data.mask.shape == output.mask.shape) assert ((data.mask == output.mask).all()) def test2(self): # Generate a dataset and grab chunks of it from the operator. The result should be the same as above. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False) # Try to connect the compatible operators. self.operator_identity_1.Input.setValue(data) self.operator_identity_2.Input.connect(self.operator_identity_1.Output) output = self.operator_identity_2.Output[None].wait() assert (self.operator_identity_1.accessCount == 1) assert (self.operator_identity_2.accessCount == 1) assert ((data == output).all()) assert (data.mask.shape == output.mask.shape) assert ((data.mask == output.mask).all()) def tearDown(self): # Take down operators self.operator_identity_2.Input.disconnect() self.operator_identity_2.Output.disconnect() self.operator_identity_2.cleanUp() self.operator_identity_1.Input.disconnect() self.operator_identity_1.Output.disconnect() self.operator_identity_1.cleanUp()
def setUp(self): self.graph = Graph() self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph) self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc") self.operator_identity.Input.meta.has_mask = True
def setup_method(self, method): self.graph = Graph() self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph) self.operator_identity.Input.allow_mask = False self.operator_identity.Output.allow_mask = False self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc")
def setup_method(self, method): self.graph = Graph() self.operator_identity_1 = OpArrayPiperWithAccessCount(graph=self.graph) self.operator_identity_2 = OpArrayPiperWithAccessCount(graph=self.graph) self.operator_identity_1.Input.meta.axistags = vigra.AxisTags("txyzc") self.operator_identity_2.Input.meta.axistags = vigra.AxisTags("txyzc")
class TestOpArrayPiperWithAccessCount4(object): def setup_method(self, method): self.graph = Graph() self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph) self.operator_identity.Input.allow_mask = False self.operator_identity.Output.allow_mask = False self.operator_identity.Input.meta.has_mask = False self.operator_identity.Output.meta.has_mask = False self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc") @nose.tools.raises(AllowMaskException) def test1(self): # Generate a random dataset and see if it we get the right masking from the operator. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False) # Provide input read all output. try: self.operator_identity.Input.setValue(data) except AssertionError as e: raise AllowMaskException(str(e)) @nose.tools.raises(AllowMaskException) def test2(self): # Generate a dataset and grab chunks of it from the operator. The result should be the same as above. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False) # Create array to store results. Don't keep original data. output = data.copy() output[:] = 0 output[:] = numpy.ma.nomask # Provide input and grab chunks. try: self.operator_identity.Input.setValue(data) except AssertionError as e: raise AllowMaskException(str(e)) @nose.tools.raises(AllowMaskException) def test3(self): # Generate a random dataset and see if it we get the right masking from the operator. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False) # Provide input read all output. try: self.operator_identity.Input.setValue(numpy.zeros_like(data)) except AssertionError as e: raise AllowMaskException(str(e)) def teardown_method(self, method): # Take down operators self.operator_identity.Input.disconnect() self.operator_identity.Output.disconnect() self.operator_identity.cleanUp()
class TestOpArrayPiperWithAccessCount5(object): def setUp(self): self.graph = Graph() self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph) self.operator_identity.Input.allow_mask = False self.operator_identity.Output.allow_mask = False self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc") @nose.tools.raises(AllowMaskException) def test1(self): # Generate a random dataset and see if it we get the right masking from the operator. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False) # Provide input read all output. try: self.operator_identity.Input.setValue(data) except AssertionError as e: raise AllowMaskException(str(e)) @nose.tools.raises(AllowMaskException) def test2(self): # Generate a dataset and grab chunks of it from the operator. The result should be the same as above. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False) # Provide input and grab chunks. try: self.operator_identity.Input.setValue(data) except AssertionError as e: raise AllowMaskException(str(e)) @nose.tools.raises(AllowMaskException) def test3(self): # Generate a random dataset and see if it we get the right masking from the operator. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False) # Provide input read all output. try: self.operator_identity.Input.setValue(numpy.zeros_like(data)) except AssertionError as e: raise AllowMaskException(str(e)) def tearDown(self): # Take down operators self.operator_identity.Input.disconnect() self.operator_identity.Output.disconnect() self.operator_identity.cleanUp()
def setup_method(self, method): self.graph = Graph() self.operator_identity_1 = OpArrayPiperWithAccessCount( graph=self.graph) self.operator_identity_2 = OpArrayPiperWithAccessCount( graph=self.graph) self.operator_identity_1.Input.meta.axistags = vigra.AxisTags("txyzc") self.operator_identity_2.Input.meta.axistags = vigra.AxisTags("txyzc")
def setUp(self): self.graph = Graph() self.operator_identity_1 = OpArrayPiperWithAccessCount(graph=self.graph) self.operator_identity_2 = OpArrayPiperWithAccessCount(graph=self.graph) self.operator_identity_2.Input.allow_mask = False self.operator_identity_2.Output.allow_mask = False self.operator_identity_1.Input.meta.axistags = vigra.AxisTags("txyzc") self.operator_identity_2.Input.meta.axistags = vigra.AxisTags("txyzc")
class TestOpArrayPiperWithAccessCount7(object): def setup_method(self, method): self.graph = Graph() self.operator_identity_1 = OpArrayPiperWithAccessCount(graph=self.graph) self.operator_identity_2 = OpArrayPiperWithAccessCount(graph=self.graph) self.operator_identity_1.Input.meta.axistags = vigra.AxisTags("txyzc") self.operator_identity_2.Input.meta.axistags = vigra.AxisTags("txyzc") def test1(self): # Explicitly set has_mask for the input self.operator_identity_1.Input.meta.has_mask = True self.operator_identity_1.Output.meta.has_mask = True # Generate a dataset and grab chunks of it from the operator. The result should be the same as above. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False) # Try to connect the compatible operators. self.operator_identity_2.Input.connect(self.operator_identity_1.Output) self.operator_identity_1.Input.setValue(data) output = self.operator_identity_2.Output[None].wait() assert self.operator_identity_1.accessCount == 1 assert self.operator_identity_2.accessCount == 1 assert (data == output).all() assert data.mask.shape == output.mask.shape assert (data.mask == output.mask).all() def test2(self): # Generate a dataset and grab chunks of it from the operator. The result should be the same as above. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False) # Try to connect the compatible operators. self.operator_identity_1.Input.setValue(data) self.operator_identity_2.Input.connect(self.operator_identity_1.Output) output = self.operator_identity_2.Output[None].wait() assert self.operator_identity_1.accessCount == 1 assert self.operator_identity_2.accessCount == 1 assert (data == output).all() assert data.mask.shape == output.mask.shape assert (data.mask == output.mask).all() def teardown_method(self, method): # Take down operators self.operator_identity_2.Input.disconnect() self.operator_identity_2.Output.disconnect() self.operator_identity_2.cleanUp() self.operator_identity_1.Input.disconnect() self.operator_identity_1.Output.disconnect() self.operator_identity_1.cleanUp()
class TestOpArrayPiperWithAccessCount6(object): def setUp(self): self.graph = Graph() self.operator_identity_1 = OpArrayPiperWithAccessCount( graph=self.graph) self.operator_identity_2 = OpArrayPiperWithAccessCount( graph=self.graph) self.operator_identity_2.Input.allow_mask = False self.operator_identity_2.Output.allow_mask = False self.operator_identity_1.Input.meta.axistags = vigra.AxisTags("txyzc") self.operator_identity_2.Input.meta.axistags = vigra.AxisTags("txyzc") @nose.tools.raises(AllowMaskException) def test1(self): # Explicitly set has_mask for the input self.operator_identity_1.Input.meta.has_mask = True self.operator_identity_1.Output.meta.has_mask = True # Try to connect the incompatible operators. try: self.operator_identity_2.Input.connect( self.operator_identity_1.Output) except AssertionError as e: raise AllowMaskException(str(e)) @nose.tools.raises(AllowMaskException) def test2(self): # Generate a dataset and grab chunks of it from the operator. The result should be the same as above. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False) # Implicitly set has_mask for the input by setting the value. self.operator_identity_1.Input.setValue(data) # Try to connect the incompatible operators. try: self.operator_identity_2.Input.connect( self.operator_identity_1.Output) except AssertionError as e: raise AllowMaskException(str(e)) def tearDown(self): # Take down operators self.operator_identity_2.Input.disconnect() self.operator_identity_2.Output.disconnect() self.operator_identity_2.cleanUp() self.operator_identity_1.Input.disconnect() self.operator_identity_1.Output.disconnect() self.operator_identity_1.cleanUp()
def testBasic(self): graph = Graph() opDataProvider = OpArrayPiperWithAccessCount(graph=graph) opCache = OpUnblockedArrayCache(graph=graph) data = np.random.random((100, 100, 100)).astype(np.float32) opDataProvider.Input.setValue(vigra.taggedView(data, 'zyx')) opCache.Input.connect(opDataProvider.Output) assert opCache.CleanBlocks.value == [] roi = ((30, 30, 30), (50, 50, 50)) cache_data = opCache.Output(*roi).wait() assert (cache_data == data[roiToSlice(*roi)]).all() assert opDataProvider.accessCount == 1 assert opCache.CleanBlocks.value == [roiToSlice(*roi)] # Request the same data a second time. # Access count should not change. cache_data = opCache.Output(*roi).wait() assert (cache_data == data[roiToSlice(*roi)]).all() assert opDataProvider.accessCount == 1 assert opCache.CleanBlocks.value == [roiToSlice(*roi)] # Now invalidate a part of the data # The cache will discard it, so the access count should increase. opDataProvider.Input.setDirty((30, 30, 30), (31, 31, 31)) assert opCache.CleanBlocks.value == [] cache_data = opCache.Output(*roi).wait() assert (cache_data == data[roiToSlice(*roi)]).all() assert opDataProvider.accessCount == 2 # Repeat this next part just for safety for _ in range(10): # Make sure the cache is empty opDataProvider.Input.setDirty((30, 30, 30), (31, 31, 31)) opDataProvider.accessCount = 0 # Create many requests for the same data. # Upstream data should only be accessed ONCE. pool = RequestPool() for _ in range(10): pool.add(opCache.Output(*roi)) pool.wait() assert opDataProvider.accessCount == 1 # Also, make sure requests for INNER rois of stored blocks are also serviced from memory opDataProvider.accessCount = 0 inner_roi = ((35, 35, 35), (45, 45, 45)) cache_data = opCache.Output(*inner_roi).wait() assert (cache_data == data[roiToSlice(*inner_roi)]).all() assert opDataProvider.accessCount == 0 assert opCache.CleanBlocks.value == [roiToSlice(*roi)]
def testBasic(self): graph = Graph() opDataProvider = OpArrayPiperWithAccessCount(graph=graph) opCache = OpUnblockedArrayCache(graph=graph) data = np.random.random((100, 100, 100)).astype(np.float32) opDataProvider.Input.setValue(vigra.taggedView(data, "zyx")) opCache.Input.connect(opDataProvider.Output) assert opCache.CleanBlocks.value == [] roi = ((30, 30, 30), (50, 50, 50)) cache_data = opCache.Output(*roi).wait() assert (cache_data == data[roiToSlice(*roi)]).all() assert opDataProvider.accessCount == 1 assert opCache.CleanBlocks.value == [roiToSlice(*roi)] # Request the same data a second time. # Access count should not change. cache_data = opCache.Output(*roi).wait() assert (cache_data == data[roiToSlice(*roi)]).all() assert opDataProvider.accessCount == 1 assert opCache.CleanBlocks.value == [roiToSlice(*roi)] # Now invalidate a part of the data # The cache will discard it, so the access count should increase. opDataProvider.Input.setDirty((30, 30, 30), (31, 31, 31)) assert opCache.CleanBlocks.value == [] cache_data = opCache.Output(*roi).wait() assert (cache_data == data[roiToSlice(*roi)]).all() assert opDataProvider.accessCount == 2 # Repeat this next part just for safety for _ in range(10): # Make sure the cache is empty opDataProvider.Input.setDirty((30, 30, 30), (31, 31, 31)) opDataProvider.accessCount = 0 # Create many requests for the same data. # Upstream data should only be accessed ONCE. pool = RequestPool() for _ in range(10): pool.add(opCache.Output(*roi)) pool.wait() assert opDataProvider.accessCount == 1 # Also, make sure requests for INNER rois of stored blocks are also serviced from memory opDataProvider.accessCount = 0 inner_roi = ((35, 35, 35), (45, 45, 45)) cache_data = opCache.Output(*inner_roi).wait() assert (cache_data == data[roiToSlice(*inner_roi)]).all() assert opDataProvider.accessCount == 0 assert opCache.CleanBlocks.value == [roiToSlice(*roi)]
def setUp(self): self.graph = Graph() self.operator_identity_1 = OpArrayPiperWithAccessCount( graph=self.graph) self.operator_identity_2 = OpArrayPiperWithAccessCount( graph=self.graph) self.operator_identity_2.Input.allow_mask = False self.operator_identity_2.Output.allow_mask = False self.operator_identity_1.Input.meta.axistags = vigra.AxisTags("txyzc") self.operator_identity_2.Input.meta.axistags = vigra.AxisTags("txyzc")
class TestOpArrayPiperWithAccessCount(object): def setup_method(self, method): self.graph = Graph() self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph) self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc") def test1(self): # Generate a random dataset and see if it we get the right masking from the operator. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) # Provide input read all output. self.operator_identity.Input.setValue(data) output = self.operator_identity.Output[None].wait() assert self.operator_identity.accessCount == 1 assert (data == output).all() def test2(self): # Generate a dataset and grab chunks of it from the operator. The result should be the same as above. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) # Create array to store results. Don't keep original data. output = data.copy() output[:] = 0 # Provide input and grab chunks. self.operator_identity.Input.setValue(data) output[:2] = self.operator_identity.Output[:2].wait() output[2:] = self.operator_identity.Output[2:].wait() assert self.operator_identity.accessCount == 2 assert (data == output).all() def test3(self): # Generate a random dataset and see if it we get the right masking from the operator. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) # Provide input read all output. self.operator_identity.Input.setValue(numpy.zeros_like(data)) output = self.operator_identity.Output[None].wait() assert self.operator_identity.accessCount == 1 assert (output == 0).all() def teardown_method(self, method): # Take down operators self.operator_identity.Input.disconnect() self.operator_identity.Output.disconnect() self.operator_identity.cleanUp()
class TestOpArrayPiperWithAccessCount6(object): def setUp(self): self.graph = Graph() self.operator_identity_1 = OpArrayPiperWithAccessCount(graph=self.graph) self.operator_identity_2 = OpArrayPiperWithAccessCount(graph=self.graph) self.operator_identity_2.Input.allow_mask = False self.operator_identity_2.Output.allow_mask = False self.operator_identity_1.Input.meta.axistags = vigra.AxisTags("txyzc") self.operator_identity_2.Input.meta.axistags = vigra.AxisTags("txyzc") @nose.tools.raises(AllowMaskException) def test1(self): # Explicitly set has_mask for the input self.operator_identity_1.Input.meta.has_mask = True self.operator_identity_1.Output.meta.has_mask = True # Try to connect the incompatible operators. try: self.operator_identity_2.Input.connect(self.operator_identity_1.Output) except AssertionError as e: raise AllowMaskException(str(e)) @nose.tools.raises(AllowMaskException) def test2(self): # Generate a dataset and grab chunks of it from the operator. The result should be the same as above. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array( data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False ) # Implicitly set has_mask for the input by setting the value. self.operator_identity_1.Input.setValue(data) # Try to connect the incompatible operators. try: self.operator_identity_2.Input.connect(self.operator_identity_1.Output) except AssertionError as e: raise AllowMaskException(str(e)) def tearDown(self): # Take down operators self.operator_identity_2.Input.disconnect() self.operator_identity_2.Output.disconnect() self.operator_identity_2.cleanUp() self.operator_identity_1.Input.disconnect() self.operator_identity_1.Output.disconnect() self.operator_identity_1.cleanUp()
class TestOpArrayPiperWithAccessCount(object): def setUp(self): self.graph = Graph() self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph) self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc") def test1(self): # Generate a random dataset and see if it we get the right masking from the operator. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) # Provide input read all output. self.operator_identity.Input.setValue(data) output = self.operator_identity.Output[None].wait() assert (self.operator_identity.accessCount == 1) assert ((data == output).all()) def test2(self): # Generate a dataset and grab chunks of it from the operator. The result should be the same as above. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) # Create array to store results. Don't keep original data. output = data.copy() output[:] = 0 # Provide input and grab chunks. self.operator_identity.Input.setValue(data) output[:2] = self.operator_identity.Output[:2].wait() output[2:] = self.operator_identity.Output[2:].wait() assert (self.operator_identity.accessCount == 2) assert ((data == output).all()) def test3(self): # Generate a random dataset and see if it we get the right masking from the operator. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) # Provide input read all output. self.operator_identity.Input.setValue(numpy.zeros_like(data)) output = self.operator_identity.Output[None].wait() assert (self.operator_identity.accessCount == 1) assert ((output == 0).all()) def tearDown(self): # Take down operators self.operator_identity.Input.disconnect() self.operator_identity.Output.disconnect() self.operator_identity.cleanUp()
def testBlockedCacheHandling(self, cacheMemoryManager): n, k = 10, 5 vol = np.zeros((n,) * 5, dtype=np.uint8) vol = vigra.taggedView(vol, axistags="txyzc") g = Graph() pipe = OpArrayPiperWithAccessCount(graph=g) cache = OpBlockedArrayCache(graph=g) # restrict cache memory to 0 Byte Memory.setAvailableRamCaches(0) # set to frequent cleanup cacheMemoryManager.setRefreshInterval(0.01) cacheMemoryManager.enable() cache.BlockShape.setValue((k,) * 5) cache.Input.connect(pipe.Output) pipe.Input.setValue(vol) a = pipe.accessCount cache.Output[...].wait() b = pipe.accessCount assert b > a, "did not cache" # let the manager clean up cacheMemoryManager.enable() time.sleep(0.5) gc.collect() cache.Output[...].wait() c = pipe.accessCount assert c > b, "did not clean up"
def testSetInSlot(self): graph = Graph() opDataProvider = OpArrayPiperWithAccessCount(graph=graph) opCache = OpUnblockedArrayCache(graph=graph) data = np.random.random((100, 100, 100)).astype(np.float32) opDataProvider.Input.setValue(vigra.taggedView(data, 'zyx')) opCache.Input.connect(opDataProvider.Output) # Write two blocks opCache.Input[10:20, 10:20, 10:20] = 2 * np.ones( (10, 10, 10), dtype=np.float32) opCache.Input[20:30, 20:30, 20:30] = 3 * np.ones( (10, 10, 10), dtype=np.float32) # They should be valid now assert opCache.CleanBlocks.value == [ tuple(make_key[10:20, 10:20, 10:20]), tuple(make_key[20:30, 20:30, 20:30]) ] assert (opCache.Output[10:20, 10:20, 10:20].wait() == 2).all() assert (opCache.Output[20:30, 20:30, 20:30].wait() == 3).all() # Mark a big dirty area opDataProvider.Input.setDirty((10, 10, 10), (40, 40, 40)) # No blocks left in the cache assert opCache.CleanBlocks.value == []
def setUp(self): self.dataShape = (1, 100, 100, 10, 1) self.data = (numpy.random.random(self.dataShape) * 100).astype(int) self.data = numpy.ma.masked_array(self.data, mask=numpy.ma.getmaskarray( self.data), fill_value=self.data.dtype.type(0), shrink=False) self.data[0, :1, :1, 0, 0] = numpy.ma.masked graph = Graph() opProvider = OpArrayPiperWithAccessCount(graph=graph) opProvider.Input.meta.axistags = vigra.defaultAxistags('txyzc') opProvider.Input.meta.has_mask = True opProvider.Input.setValue(self.data) self.opProvider = opProvider opCache = OpSlicedBlockedArrayCache(graph=graph) opCache.Input.connect(opProvider.Output) opCache.innerBlockShape.setValue( ((10, 1, 10, 10, 10), (10, 10, 1, 10, 10), (10, 10, 10, 1, 10))) opCache.outerBlockShape.setValue( ((20, 2, 20, 20, 20), (20, 20, 2, 20, 20), (20, 20, 20, 2, 20))) opCache.fixAtCurrent.setValue(False) self.opCache = opCache
def testBasic5d_masked(self): logger.info("Generating sample data...") sampleData = numpy.indices((3, 100, 200, 150, 2), dtype=numpy.float32).sum(0) sampleData = sampleData.view(numpy.ma.masked_array) sampleData.set_fill_value(numpy.float32(numpy.nan)) sampleData[0] = numpy.ma.masked graph = Graph() opData = OpArrayPiperWithAccessCount(graph=graph) opData.Input.meta.has_mask = True opData.Input.meta.axistags = vigra.defaultAxistags('txyzc') opData.Input.setValue(sampleData) op = OpCompressedCache(parent=None, graph=graph) #logger.debug("Setting block shape...") op.BlockShape.setValue([1, 100, 75, 50, 2]) op.Input.connect(opData.Output) assert op.Output.ready() slicing = numpy.s_[0:2, 0:100, 50:150, 75:150, 0:1] expectedData = sampleData[slicing] #logger.debug("Requesting data...") readData = op.Output[slicing].wait() #logger.debug("Checking data...") assert (readData == expectedData).all() and \ (readData.mask == expectedData.mask).all() and \ ((readData.fill_value == expectedData.fill_value) | (numpy.isnan(readData.fill_value) & numpy.isnan(expectedData.fill_value))).all(),\ "Incorrect output!" assert opData.accessCount == 2 * 1 * 2 * 2 * 1, str(opData.accessCount)
def testCacheHandling(self): n, k = 10, 5 vol = np.zeros((n, ) * 5, dtype=np.uint8) vol = vigra.taggedView(vol, axistags='txyzc') g = Graph() pipe = OpArrayPiperWithAccessCount(graph=g) cache = OpArrayCache(graph=g) mgr = CacheMemoryManager() # disallow cache memory Memory.setAvailableRamCaches(0) # set to frequent cleanup mgr.setRefreshInterval(.01) mgr.enable() cache.blockShape.setValue((k, ) * 5) cache.Input.connect(pipe.Output) pipe.Input.setValue(vol) a = pipe.accessCount cache.Output[...].wait() b = pipe.accessCount assert b > a, "did not cache" # let the manager clean up mgr.enable() time.sleep(.5) gc.collect() cache.Output[...].wait() c = pipe.accessCount assert c > b, "did not clean up"
def testBasic5d(self): logger.info("Generating sample data...") sampleData = numpy.indices((3, 100, 200, 150, 2), dtype=numpy.float32).sum(0) sampleData = sampleData.view(vigra.VigraArray) sampleData.axistags = vigra.defaultAxistags("txyzc") graph = Graph() opData = OpArrayPiperWithAccessCount(graph=graph) opData.Input.setValue(sampleData) op = OpCompressedCache(parent=None, graph=graph) # logger.debug("Setting block shape...") op.BlockShape.setValue([1, 100, 75, 50, 2]) op.Input.connect(opData.Output) assert op.Output.ready() slicing = numpy.s_[0:2, 0:100, 50:150, 75:150, 0:1] expectedData = sampleData[slicing].view(numpy.ndarray) # logger.debug("Requesting data...") readData = op.Output[slicing].wait() # logger.debug("Checking data...") assert (readData == expectedData).all(), "Incorrect output!" assert opData.accessCount == 2 * 1 * 2 * 2 * 1, str(opData.accessCount)
def setup_method(self, method): g = Graph() vol = np.random.random(size=(100, 110, 120)) self.vol = vigra.taggedView(vol, axistags='xyz') vol5d = np.random.random(size=(3, 100, 110, 120, 7)) self.vol5d = vigra.taggedView(vol5d, axistags='cxyzt') piper = OpArrayPiperWithAccessCount(graph=g) piper.Input.setValue(self.vol) self.g = g self.piper = piper
def setup_method(self, method): self.dataShape = (1, 100, 100, 10, 1) self.data = (numpy.random.random(self.dataShape) * 100).astype(numpy.uint32) self.data = self.data.view(vigra.VigraArray) self.data.axistags = vigra.defaultAxistags("txyzc") graph = Graph() opProvider = OpArrayPiperWithAccessCount(graph=graph) opProvider.Input.setValue(self.data) self.opProvider = opProvider opCache = OpSlicedBlockedArrayCache(graph=graph) opCache.Input.connect(opProvider.Output) opCache.BlockShape.setValue(((20, 2, 20, 20, 20), (20, 20, 2, 20, 20), (20, 20, 20, 2, 20))) opCache.fixAtCurrent.setValue(False) self.opCache = opCache
def testCacheApi(self): graph = Graph() opDataProvider = OpArrayPiperWithAccessCount(graph=graph) opCache = OpUnblockedArrayCache(graph=graph) data = np.random.random((100, 100, 100)).astype(np.float32) opDataProvider.Input.setValue(vigra.taggedView(data, 'zyx')) opCache.Input.connect(opDataProvider.Output) opCache.Output[10:20, 20:40, 50:100].wait() opCache.Output[11:21, 22:43, 53:90].wait() l = opCache.getBlockAccessTimes() assert len(l) == 2 for k, t in l: assert t > 0.0
def setUp(self): self.dataShape = (1, 100, 100, 10, 1) self.data = (numpy.random.random(self.dataShape) * 100).astype(int) self.data = self.data.view(vigra.VigraArray) self.data.axistags = vigra.defaultAxistags('txyzc') graph = Graph() opProvider = OpArrayPiperWithAccessCount(graph=graph) opProvider.Input.setValue(self.data) self.opProvider = opProvider opCache = OpArrayCache(graph=graph) opCache.Input.connect(opProvider.Output) opCache.blockShape.setValue((10, 10, 10, 10, 10)) opCache.fixAtCurrent.setValue(False) self.opCache = opCache
def setUp(self): self.dataShape = (1, 100, 100, 10, 1) self.data = numpy.random.randint(0, 256, size=self.dataShape) self.data = self.data.astype(numpy.uint32) self.data = self.data.view(vigra.VigraArray) self.data.axistags = vigra.defaultAxistags('txyzc') graph = Graph() opProvider = OpArrayPiperWithAccessCount(graph=graph) opProvider.Input.setValue(self.data) self.opProvider = opProvider opCache = OpBlockedArrayCache(graph=graph) opCache.Input.connect(opProvider.Output) opCache.innerBlockShape.setValue((10, 10, 10, 10, 10)) opCache.outerBlockShape.setValue((20, 20, 20, 20, 20)) opCache.fixAtCurrent.setValue(False) self.opCache = opCache
def setUp(self): g = Graph() vol = np.random.random(size=(100, 110, 120)) self.vol = vigra.taggedView(vol, axistags='xyz') vol5d = np.random.random(size=(3, 100, 110, 120, 7)) self.vol5d = vigra.taggedView(vol5d, axistags='cxyzt') piper = OpArrayPiperWithAccessCount(graph=g) op = OpCacheFixer(graph=g) op.Input.connect(piper.Output) call = OpCallWhenDirty(graph=g) call.Input.connect(op.Output) self.piper = piper self.op = op self.call = call
def testFree(self): sampleData = numpy.indices((100, 200, 150), dtype=numpy.float32).sum(0) sampleData = vigra.taggedView(sampleData, axistags='xyz') graph = Graph() opData = OpArrayPiperWithAccessCount(graph=graph) opData.Input.setValue(sampleData) op = OpCompressedCache(graph=graph) #logger.debug("Setting block shape...") op.BlockShape.setValue([100, 75, 50]) op.Input.connect(opData.Output) op.Output[...].wait() mem = op.usedMemory() keys = [x[0] for x in op.getBlockAccessTimes()] key = keys[0] op.freeBlock(key) assert op.usedMemory() < mem
def setup_method(self, method): self.dataShape = (1, 100, 100, 10, 1) self.data = (numpy.random.random(self.dataShape) * 100).astype(int) self.data = numpy.ma.masked_array( self.data, mask=numpy.ma.getmaskarray(self.data), fill_value=numpy.iinfo(int).max, shrink=False ) self.data[:, 0] = numpy.ma.masked graph = Graph() opProvider = OpArrayPiperWithAccessCount(graph=graph) opProvider.Input.meta.axistags = vigra.defaultAxistags("txyzc") opProvider.Input.meta.has_mask = True opProvider.Input.setValue(self.data) self.opProvider = opProvider opCache = OpBlockedArrayCache(graph=graph) opCache.Input.connect(opProvider.Output) opCache.BlockShape.setValue((20, 20, 20, 20, 20)) opCache.fixAtCurrent.setValue(False) self.opCache = opCache
class TestOpArrayPiperWithAccessCount2(object): def setup_method(self, method): self.graph = Graph() self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph) self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc") self.operator_identity.Input.meta.has_mask = True def test1(self): # Generate a random dataset and see if it we get the right masking from the operator. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False) # Provide input read all output. self.operator_identity.Input.setValue(data) output = self.operator_identity.Output[None].wait() assert self.operator_identity.accessCount == 1 assert (data == output).all() assert data.mask.shape == output.mask.shape assert (data.mask == output.mask).all() def test2(self): # Generate a dataset and grab chunks of it from the operator. The result should be the same as above. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False) # Create array to store results. Don't keep original data. output = data.copy() output[:] = 0 output[:] = numpy.ma.nomask # Provide input and grab chunks. self.operator_identity.Input.setValue(data) output[:2] = self.operator_identity.Output[:2].wait() output[2:] = self.operator_identity.Output[2:].wait() assert self.operator_identity.accessCount == 2 assert (data == output).all() assert data.mask.shape == output.mask.shape assert (data.mask == output.mask).all() def test3(self): # Generate a random dataset and see if it we get the right masking from the operator. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False) # Provide input read all output. self.operator_identity.Input.setValue(numpy.zeros_like(data)) output = self.operator_identity.Output[None].wait() assert self.operator_identity.accessCount == 1 assert (output == 0).all() assert data.mask.shape == output.mask.shape assert (output.mask == False).all() def teardown_method(self, method): # Take down operators self.operator_identity.Input.disconnect() self.operator_identity.Output.disconnect() self.operator_identity.cleanUp()
class TestOpArrayPiperWithAccessCount2(object): def setUp(self): self.graph = Graph() self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph) self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc") self.operator_identity.Input.meta.has_mask = True def test1(self): # Generate a random dataset and see if it we get the right masking from the operator. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array( data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False ) # Provide input read all output. self.operator_identity.Input.setValue(data) output = self.operator_identity.Output[None].wait() assert (self.operator_identity.accessCount == 1) assert((data == output).all()) assert(data.mask.shape == output.mask.shape) assert((data.mask == output.mask).all()) def test2(self): # Generate a dataset and grab chunks of it from the operator. The result should be the same as above. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array( data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False ) # Create array to store results. Don't keep original data. output = data.copy() output[:] = 0 output[:] = numpy.ma.nomask # Provide input and grab chunks. self.operator_identity.Input.setValue(data) output[:2] = self.operator_identity.Output[:2].wait() output[2:] = self.operator_identity.Output[2:].wait() assert (self.operator_identity.accessCount == 2) assert((data == output).all()) assert(data.mask.shape == output.mask.shape) assert((data.mask == output.mask).all()) def test3(self): # Generate a random dataset and see if it we get the right masking from the operator. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array( data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False ) # Provide input read all output. self.operator_identity.Input.setValue(numpy.zeros_like(data)) output = self.operator_identity.Output[None].wait() assert (self.operator_identity.accessCount == 1) assert((output == 0).all()) assert(data.mask.shape == output.mask.shape) assert((output.mask == False).all()) def tearDown(self): # Take down operators self.operator_identity.Input.disconnect() self.operator_identity.Output.disconnect() self.operator_identity.cleanUp()
class TestOpArrayPiperWithAccessCount4(object): def setup_method(self, method): self.graph = Graph() self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph) self.operator_identity.Input.allow_mask = False self.operator_identity.Output.allow_mask = False self.operator_identity.Input.meta.has_mask = False self.operator_identity.Output.meta.has_mask = False self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc") def test1(self): # Generate a random dataset and see if it we get the right masking from the operator. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False) # Provide input read all output. with pytest.raises(AllowMaskException): try: self.operator_identity.Input.setValue(data) except AssertionError as e: raise AllowMaskException(str(e)) def test2(self): # Generate a dataset and grab chunks of it from the operator. The result should be the same as above. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False) # Create array to store results. Don't keep original data. output = data.copy() output[:] = 0 output[:] = numpy.ma.nomask # Provide input and grab chunks. with pytest.raises(AllowMaskException): try: self.operator_identity.Input.setValue(data) except AssertionError as e: raise AllowMaskException(str(e)) def test3(self): # Generate a random dataset and see if it we get the right masking from the operator. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False) # Provide input read all output. with pytest.raises(AllowMaskException): try: self.operator_identity.Input.setValue(numpy.zeros_like(data)) except AssertionError as e: raise AllowMaskException(str(e)) def teardown_method(self, method): # Take down operators self.operator_identity.Input.disconnect() self.operator_identity.Output.disconnect() self.operator_identity.cleanUp()
def testBadMemoryConditions(self): """ TestCacheMemoryManager.testBadMemoryConditions This test is a proof of the proposition in https://github.com/ilastik/lazyflow/issue/185 which states that, given certain memory constraints, the cache cleanup strategy in use is inefficient. An advanced strategy should pass the test. """ mgr = _CacheMemoryManager() mgr.setRefreshInterval(0.01) mgr.enable() d = 2 tags = "xy" shape = (999,) * d blockshape = (333,) * d # restrict memory for computation to one block (including fudge # factor 2 of bigRequestStreamer) cacheMem = np.prod(shape) Memory.setAvailableRam(np.prod(blockshape) * 2 + cacheMem) # restrict cache memory to the whole volume Memory.setAvailableRamCaches(cacheMem) # to ease observation, do everything single threaded Request.reset_thread_pool(num_workers=1) x = np.zeros(shape, dtype=np.uint8) x = vigra.taggedView(x, axistags=tags) g = Graph() pipe = OpArrayPiperWithAccessCount(graph=g) pipe.Input.setValue(x) pipe.Output.meta.ideal_blockshape = blockshape # simulate BlockedArrayCache behaviour without caching # cache = OpSplitRequestsBlockwise(True, graph=g) # cache.BlockShape.setValue(blockshape) # cache.Input.connect(pipe.Output) cache = OpBlockedArrayCache(graph=g) cache.Input.connect(pipe.Output) cache.BlockShape.setValue(blockshape) op = OpEnlarge(graph=g) op.Input.connect(cache.Output) split = OpSplitRequestsBlockwise(True, graph=g) split.BlockShape.setValue(blockshape) split.Input.connect(op.Output) streamer = BigRequestStreamer(split.Output, [(0,) * len(shape), shape]) streamer.execute() # in the worst case, we have 4*4 + 4*6 + 9 = 49 requests to pipe # in the best case, we have 9 np.testing.assert_equal(pipe.accessCount, 9)