class TestOpArrayPiperWithAccessCount7(object):
def setUp(self):
self.graph = Graph()
self.operator_identity_1 = OpArrayPiperWithAccessCount(
graph=self.graph)
self.operator_identity_2 = OpArrayPiperWithAccessCount(
graph=self.graph)
self.operator_identity_1.Input.meta.axistags = vigra.AxisTags("txyzc")
self.operator_identity_2.Input.meta.axistags = vigra.AxisTags("txyzc")
def test1(self):
# Explicitly set has_mask for the input
self.operator_identity_1.Input.meta.has_mask = True
self.operator_identity_1.Output.meta.has_mask = True
# Generate a dataset and grab chunks of it from the operator. The result should be the same as above.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(data,
mask=numpy.zeros(data.shape, dtype=bool),
shrink=False)
# Try to connect the compatible operators.
self.operator_identity_2.Input.connect(self.operator_identity_1.Output)
self.operator_identity_1.Input.setValue(data)
output = self.operator_identity_2.Output[None].wait()
assert (self.operator_identity_1.accessCount == 1)
assert (self.operator_identity_2.accessCount == 1)
assert ((data == output).all())
assert (data.mask.shape == output.mask.shape)
assert ((data.mask == output.mask).all())
def test2(self):
# Generate a dataset and grab chunks of it from the operator. The result should be the same as above.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(data,
mask=numpy.zeros(data.shape, dtype=bool),
shrink=False)
# Try to connect the compatible operators.
self.operator_identity_1.Input.setValue(data)
self.operator_identity_2.Input.connect(self.operator_identity_1.Output)
output = self.operator_identity_2.Output[None].wait()
assert (self.operator_identity_1.accessCount == 1)
assert (self.operator_identity_2.accessCount == 1)
assert ((data == output).all())
assert (data.mask.shape == output.mask.shape)
assert ((data.mask == output.mask).all())
def tearDown(self):
# Take down operators
self.operator_identity_2.Input.disconnect()
self.operator_identity_2.Output.disconnect()
self.operator_identity_2.cleanUp()
self.operator_identity_1.Input.disconnect()
self.operator_identity_1.Output.disconnect()
self.operator_identity_1.cleanUp()
def setUp(self):
self.graph = Graph()
self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph)
self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc")
self.operator_identity.Input.meta.has_mask = True
def setup_method(self, method):
self.graph = Graph()
self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph)
self.operator_identity.Input.allow_mask = False
self.operator_identity.Output.allow_mask = False
self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc")
def setup_method(self, method):
self.graph = Graph()
self.operator_identity_1 = OpArrayPiperWithAccessCount(graph=self.graph)
self.operator_identity_2 = OpArrayPiperWithAccessCount(graph=self.graph)
self.operator_identity_1.Input.meta.axistags = vigra.AxisTags("txyzc")
self.operator_identity_2.Input.meta.axistags = vigra.AxisTags("txyzc")
class TestOpArrayPiperWithAccessCount4(object):
def setup_method(self, method):
self.graph = Graph()
self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph)
self.operator_identity.Input.allow_mask = False
self.operator_identity.Output.allow_mask = False
self.operator_identity.Input.meta.has_mask = False
self.operator_identity.Output.meta.has_mask = False
self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc")
@nose.tools.raises(AllowMaskException)
def test1(self):
# Generate a random dataset and see if it we get the right masking from the operator.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False)
# Provide input read all output.
try:
self.operator_identity.Input.setValue(data)
except AssertionError as e:
raise AllowMaskException(str(e))
@nose.tools.raises(AllowMaskException)
def test2(self):
# Generate a dataset and grab chunks of it from the operator. The result should be the same as above.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False)
# Create array to store results. Don't keep original data.
output = data.copy()
output[:] = 0
output[:] = numpy.ma.nomask
# Provide input and grab chunks.
try:
self.operator_identity.Input.setValue(data)
except AssertionError as e:
raise AllowMaskException(str(e))
@nose.tools.raises(AllowMaskException)
def test3(self):
# Generate a random dataset and see if it we get the right masking from the operator.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False)
# Provide input read all output.
try:
self.operator_identity.Input.setValue(numpy.zeros_like(data))
except AssertionError as e:
raise AllowMaskException(str(e))
def teardown_method(self, method):
# Take down operators
self.operator_identity.Input.disconnect()
self.operator_identity.Output.disconnect()
self.operator_identity.cleanUp()
class TestOpArrayPiperWithAccessCount5(object):
def setUp(self):
self.graph = Graph()
self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph)
self.operator_identity.Input.allow_mask = False
self.operator_identity.Output.allow_mask = False
self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc")
@nose.tools.raises(AllowMaskException)
def test1(self):
# Generate a random dataset and see if it we get the right masking from the operator.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(data,
mask=numpy.zeros(data.shape, dtype=bool),
shrink=False)
# Provide input read all output.
try:
self.operator_identity.Input.setValue(data)
except AssertionError as e:
raise AllowMaskException(str(e))
@nose.tools.raises(AllowMaskException)
def test2(self):
# Generate a dataset and grab chunks of it from the operator. The result should be the same as above.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(data,
mask=numpy.zeros(data.shape, dtype=bool),
shrink=False)
# Provide input and grab chunks.
try:
self.operator_identity.Input.setValue(data)
except AssertionError as e:
raise AllowMaskException(str(e))
@nose.tools.raises(AllowMaskException)
def test3(self):
# Generate a random dataset and see if it we get the right masking from the operator.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(data,
mask=numpy.zeros(data.shape, dtype=bool),
shrink=False)
# Provide input read all output.
try:
self.operator_identity.Input.setValue(numpy.zeros_like(data))
except AssertionError as e:
raise AllowMaskException(str(e))
def tearDown(self):
# Take down operators
self.operator_identity.Input.disconnect()
self.operator_identity.Output.disconnect()
self.operator_identity.cleanUp()
def setup_method(self, method):
self.graph = Graph()
self.operator_identity_1 = OpArrayPiperWithAccessCount(
graph=self.graph)
self.operator_identity_2 = OpArrayPiperWithAccessCount(
graph=self.graph)
self.operator_identity_1.Input.meta.axistags = vigra.AxisTags("txyzc")
self.operator_identity_2.Input.meta.axistags = vigra.AxisTags("txyzc")
def setUp(self):
self.graph = Graph()
self.operator_identity_1 = OpArrayPiperWithAccessCount(graph=self.graph)
self.operator_identity_2 = OpArrayPiperWithAccessCount(graph=self.graph)
self.operator_identity_2.Input.allow_mask = False
self.operator_identity_2.Output.allow_mask = False
self.operator_identity_1.Input.meta.axistags = vigra.AxisTags("txyzc")
self.operator_identity_2.Input.meta.axistags = vigra.AxisTags("txyzc")
class TestOpArrayPiperWithAccessCount7(object):
def setup_method(self, method):
self.graph = Graph()
self.operator_identity_1 = OpArrayPiperWithAccessCount(graph=self.graph)
self.operator_identity_2 = OpArrayPiperWithAccessCount(graph=self.graph)
self.operator_identity_1.Input.meta.axistags = vigra.AxisTags("txyzc")
self.operator_identity_2.Input.meta.axistags = vigra.AxisTags("txyzc")
def test1(self):
# Explicitly set has_mask for the input
self.operator_identity_1.Input.meta.has_mask = True
self.operator_identity_1.Output.meta.has_mask = True
# Generate a dataset and grab chunks of it from the operator. The result should be the same as above.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False)
# Try to connect the compatible operators.
self.operator_identity_2.Input.connect(self.operator_identity_1.Output)
self.operator_identity_1.Input.setValue(data)
output = self.operator_identity_2.Output[None].wait()
assert self.operator_identity_1.accessCount == 1
assert self.operator_identity_2.accessCount == 1
assert (data == output).all()
assert data.mask.shape == output.mask.shape
assert (data.mask == output.mask).all()
def test2(self):
# Generate a dataset and grab chunks of it from the operator. The result should be the same as above.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False)
# Try to connect the compatible operators.
self.operator_identity_1.Input.setValue(data)
self.operator_identity_2.Input.connect(self.operator_identity_1.Output)
output = self.operator_identity_2.Output[None].wait()
assert self.operator_identity_1.accessCount == 1
assert self.operator_identity_2.accessCount == 1
assert (data == output).all()
assert data.mask.shape == output.mask.shape
assert (data.mask == output.mask).all()
def teardown_method(self, method):
# Take down operators
self.operator_identity_2.Input.disconnect()
self.operator_identity_2.Output.disconnect()
self.operator_identity_2.cleanUp()
self.operator_identity_1.Input.disconnect()
self.operator_identity_1.Output.disconnect()
self.operator_identity_1.cleanUp()
class TestOpArrayPiperWithAccessCount6(object):
def setUp(self):
self.graph = Graph()
self.operator_identity_1 = OpArrayPiperWithAccessCount(
graph=self.graph)
self.operator_identity_2 = OpArrayPiperWithAccessCount(
graph=self.graph)
self.operator_identity_2.Input.allow_mask = False
self.operator_identity_2.Output.allow_mask = False
self.operator_identity_1.Input.meta.axistags = vigra.AxisTags("txyzc")
self.operator_identity_2.Input.meta.axistags = vigra.AxisTags("txyzc")
@nose.tools.raises(AllowMaskException)
def test1(self):
# Explicitly set has_mask for the input
self.operator_identity_1.Input.meta.has_mask = True
self.operator_identity_1.Output.meta.has_mask = True
# Try to connect the incompatible operators.
try:
self.operator_identity_2.Input.connect(
self.operator_identity_1.Output)
except AssertionError as e:
raise AllowMaskException(str(e))
@nose.tools.raises(AllowMaskException)
def test2(self):
# Generate a dataset and grab chunks of it from the operator. The result should be the same as above.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(data,
mask=numpy.zeros(data.shape, dtype=bool),
shrink=False)
# Implicitly set has_mask for the input by setting the value.
self.operator_identity_1.Input.setValue(data)
# Try to connect the incompatible operators.
try:
self.operator_identity_2.Input.connect(
self.operator_identity_1.Output)
except AssertionError as e:
raise AllowMaskException(str(e))
def tearDown(self):
# Take down operators
self.operator_identity_2.Input.disconnect()
self.operator_identity_2.Output.disconnect()
self.operator_identity_2.cleanUp()
self.operator_identity_1.Input.disconnect()
self.operator_identity_1.Output.disconnect()
self.operator_identity_1.cleanUp()
def testBasic(self):
graph = Graph()
opDataProvider = OpArrayPiperWithAccessCount(graph=graph)
opCache = OpUnblockedArrayCache(graph=graph)
data = np.random.random((100, 100, 100)).astype(np.float32)
opDataProvider.Input.setValue(vigra.taggedView(data, 'zyx'))
opCache.Input.connect(opDataProvider.Output)
assert opCache.CleanBlocks.value == []
roi = ((30, 30, 30), (50, 50, 50))
cache_data = opCache.Output(*roi).wait()
assert (cache_data == data[roiToSlice(*roi)]).all()
assert opDataProvider.accessCount == 1
assert opCache.CleanBlocks.value == [roiToSlice(*roi)]
# Request the same data a second time.
# Access count should not change.
cache_data = opCache.Output(*roi).wait()
assert (cache_data == data[roiToSlice(*roi)]).all()
assert opDataProvider.accessCount == 1
assert opCache.CleanBlocks.value == [roiToSlice(*roi)]
# Now invalidate a part of the data
# The cache will discard it, so the access count should increase.
opDataProvider.Input.setDirty((30, 30, 30), (31, 31, 31))
assert opCache.CleanBlocks.value == []
cache_data = opCache.Output(*roi).wait()
assert (cache_data == data[roiToSlice(*roi)]).all()
assert opDataProvider.accessCount == 2
# Repeat this next part just for safety
for _ in range(10):
# Make sure the cache is empty
opDataProvider.Input.setDirty((30, 30, 30), (31, 31, 31))
opDataProvider.accessCount = 0
# Create many requests for the same data.
# Upstream data should only be accessed ONCE.
pool = RequestPool()
for _ in range(10):
pool.add(opCache.Output(*roi))
pool.wait()
assert opDataProvider.accessCount == 1
# Also, make sure requests for INNER rois of stored blocks are also serviced from memory
opDataProvider.accessCount = 0
inner_roi = ((35, 35, 35), (45, 45, 45))
cache_data = opCache.Output(*inner_roi).wait()
assert (cache_data == data[roiToSlice(*inner_roi)]).all()
assert opDataProvider.accessCount == 0
assert opCache.CleanBlocks.value == [roiToSlice(*roi)]
def testBasic(self):
graph = Graph()
opDataProvider = OpArrayPiperWithAccessCount(graph=graph)
opCache = OpUnblockedArrayCache(graph=graph)
data = np.random.random((100, 100, 100)).astype(np.float32)
opDataProvider.Input.setValue(vigra.taggedView(data, "zyx"))
opCache.Input.connect(opDataProvider.Output)
assert opCache.CleanBlocks.value == []
roi = ((30, 30, 30), (50, 50, 50))
cache_data = opCache.Output(*roi).wait()
assert (cache_data == data[roiToSlice(*roi)]).all()
assert opDataProvider.accessCount == 1
assert opCache.CleanBlocks.value == [roiToSlice(*roi)]
# Request the same data a second time.
# Access count should not change.
cache_data = opCache.Output(*roi).wait()
assert (cache_data == data[roiToSlice(*roi)]).all()
assert opDataProvider.accessCount == 1
assert opCache.CleanBlocks.value == [roiToSlice(*roi)]
# Now invalidate a part of the data
# The cache will discard it, so the access count should increase.
opDataProvider.Input.setDirty((30, 30, 30), (31, 31, 31))
assert opCache.CleanBlocks.value == []
cache_data = opCache.Output(*roi).wait()
assert (cache_data == data[roiToSlice(*roi)]).all()
assert opDataProvider.accessCount == 2
# Repeat this next part just for safety
for _ in range(10):
# Make sure the cache is empty
opDataProvider.Input.setDirty((30, 30, 30), (31, 31, 31))
opDataProvider.accessCount = 0
# Create many requests for the same data.
# Upstream data should only be accessed ONCE.
pool = RequestPool()
for _ in range(10):
pool.add(opCache.Output(*roi))
pool.wait()
assert opDataProvider.accessCount == 1
# Also, make sure requests for INNER rois of stored blocks are also serviced from memory
opDataProvider.accessCount = 0
inner_roi = ((35, 35, 35), (45, 45, 45))
cache_data = opCache.Output(*inner_roi).wait()
assert (cache_data == data[roiToSlice(*inner_roi)]).all()
assert opDataProvider.accessCount == 0
assert opCache.CleanBlocks.value == [roiToSlice(*roi)]
def setUp(self):
self.graph = Graph()
self.operator_identity_1 = OpArrayPiperWithAccessCount(
graph=self.graph)
self.operator_identity_2 = OpArrayPiperWithAccessCount(
graph=self.graph)
self.operator_identity_2.Input.allow_mask = False
self.operator_identity_2.Output.allow_mask = False
self.operator_identity_1.Input.meta.axistags = vigra.AxisTags("txyzc")
self.operator_identity_2.Input.meta.axistags = vigra.AxisTags("txyzc")
class TestOpArrayPiperWithAccessCount(object):
def setup_method(self, method):
self.graph = Graph()
self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph)
self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc")
def test1(self):
# Generate a random dataset and see if it we get the right masking from the operator.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
# Provide input read all output.
self.operator_identity.Input.setValue(data)
output = self.operator_identity.Output[None].wait()
assert self.operator_identity.accessCount == 1
assert (data == output).all()
def test2(self):
# Generate a dataset and grab chunks of it from the operator. The result should be the same as above.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
# Create array to store results. Don't keep original data.
output = data.copy()
output[:] = 0
# Provide input and grab chunks.
self.operator_identity.Input.setValue(data)
output[:2] = self.operator_identity.Output[:2].wait()
output[2:] = self.operator_identity.Output[2:].wait()
assert self.operator_identity.accessCount == 2
assert (data == output).all()
def test3(self):
# Generate a random dataset and see if it we get the right masking from the operator.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
# Provide input read all output.
self.operator_identity.Input.setValue(numpy.zeros_like(data))
output = self.operator_identity.Output[None].wait()
assert self.operator_identity.accessCount == 1
assert (output == 0).all()
def teardown_method(self, method):
# Take down operators
self.operator_identity.Input.disconnect()
self.operator_identity.Output.disconnect()
self.operator_identity.cleanUp()
class TestOpArrayPiperWithAccessCount6(object):
def setUp(self):
self.graph = Graph()
self.operator_identity_1 = OpArrayPiperWithAccessCount(graph=self.graph)
self.operator_identity_2 = OpArrayPiperWithAccessCount(graph=self.graph)
self.operator_identity_2.Input.allow_mask = False
self.operator_identity_2.Output.allow_mask = False
self.operator_identity_1.Input.meta.axistags = vigra.AxisTags("txyzc")
self.operator_identity_2.Input.meta.axistags = vigra.AxisTags("txyzc")
@nose.tools.raises(AllowMaskException)
def test1(self):
# Explicitly set has_mask for the input
self.operator_identity_1.Input.meta.has_mask = True
self.operator_identity_1.Output.meta.has_mask = True
# Try to connect the incompatible operators.
try:
self.operator_identity_2.Input.connect(self.operator_identity_1.Output)
except AssertionError as e:
raise AllowMaskException(str(e))
@nose.tools.raises(AllowMaskException)
def test2(self):
# Generate a dataset and grab chunks of it from the operator. The result should be the same as above.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(
data,
mask=numpy.zeros(data.shape, dtype=bool),
shrink=False
)
# Implicitly set has_mask for the input by setting the value.
self.operator_identity_1.Input.setValue(data)
# Try to connect the incompatible operators.
try:
self.operator_identity_2.Input.connect(self.operator_identity_1.Output)
except AssertionError as e:
raise AllowMaskException(str(e))
def tearDown(self):
# Take down operators
self.operator_identity_2.Input.disconnect()
self.operator_identity_2.Output.disconnect()
self.operator_identity_2.cleanUp()
self.operator_identity_1.Input.disconnect()
self.operator_identity_1.Output.disconnect()
self.operator_identity_1.cleanUp()
class TestOpArrayPiperWithAccessCount(object):
def setUp(self):
self.graph = Graph()
self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph)
self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc")
def test1(self):
# Generate a random dataset and see if it we get the right masking from the operator.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
# Provide input read all output.
self.operator_identity.Input.setValue(data)
output = self.operator_identity.Output[None].wait()
assert (self.operator_identity.accessCount == 1)
assert ((data == output).all())
def test2(self):
# Generate a dataset and grab chunks of it from the operator. The result should be the same as above.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
# Create array to store results. Don't keep original data.
output = data.copy()
output[:] = 0
# Provide input and grab chunks.
self.operator_identity.Input.setValue(data)
output[:2] = self.operator_identity.Output[:2].wait()
output[2:] = self.operator_identity.Output[2:].wait()
assert (self.operator_identity.accessCount == 2)
assert ((data == output).all())
def test3(self):
# Generate a random dataset and see if it we get the right masking from the operator.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
# Provide input read all output.
self.operator_identity.Input.setValue(numpy.zeros_like(data))
output = self.operator_identity.Output[None].wait()
assert (self.operator_identity.accessCount == 1)
assert ((output == 0).all())
def tearDown(self):
# Take down operators
self.operator_identity.Input.disconnect()
self.operator_identity.Output.disconnect()
self.operator_identity.cleanUp()
def testBlockedCacheHandling(self, cacheMemoryManager):
n, k = 10, 5
vol = np.zeros((n,) * 5, dtype=np.uint8)
vol = vigra.taggedView(vol, axistags="txyzc")
g = Graph()
pipe = OpArrayPiperWithAccessCount(graph=g)
cache = OpBlockedArrayCache(graph=g)
# restrict cache memory to 0 Byte
Memory.setAvailableRamCaches(0)
# set to frequent cleanup
cacheMemoryManager.setRefreshInterval(0.01)
cacheMemoryManager.enable()
cache.BlockShape.setValue((k,) * 5)
cache.Input.connect(pipe.Output)
pipe.Input.setValue(vol)
a = pipe.accessCount
cache.Output[...].wait()
b = pipe.accessCount
assert b > a, "did not cache"
# let the manager clean up
cacheMemoryManager.enable()
time.sleep(0.5)
gc.collect()
cache.Output[...].wait()
c = pipe.accessCount
assert c > b, "did not clean up"
def testSetInSlot(self):
graph = Graph()
opDataProvider = OpArrayPiperWithAccessCount(graph=graph)
opCache = OpUnblockedArrayCache(graph=graph)
data = np.random.random((100, 100, 100)).astype(np.float32)
opDataProvider.Input.setValue(vigra.taggedView(data, 'zyx'))
opCache.Input.connect(opDataProvider.Output)
# Write two blocks
opCache.Input[10:20, 10:20, 10:20] = 2 * np.ones(
(10, 10, 10), dtype=np.float32)
opCache.Input[20:30, 20:30, 20:30] = 3 * np.ones(
(10, 10, 10), dtype=np.float32)
# They should be valid now
assert opCache.CleanBlocks.value == [
tuple(make_key[10:20, 10:20, 10:20]),
tuple(make_key[20:30, 20:30, 20:30])
]
assert (opCache.Output[10:20, 10:20, 10:20].wait() == 2).all()
assert (opCache.Output[20:30, 20:30, 20:30].wait() == 3).all()
# Mark a big dirty area
opDataProvider.Input.setDirty((10, 10, 10), (40, 40, 40))
# No blocks left in the cache
assert opCache.CleanBlocks.value == []
def setUp(self):
self.dataShape = (1, 100, 100, 10, 1)
self.data = (numpy.random.random(self.dataShape) * 100).astype(int)
self.data = numpy.ma.masked_array(self.data,
mask=numpy.ma.getmaskarray(
self.data),
fill_value=self.data.dtype.type(0),
shrink=False)
self.data[0, :1, :1, 0, 0] = numpy.ma.masked
graph = Graph()
opProvider = OpArrayPiperWithAccessCount(graph=graph)
opProvider.Input.meta.axistags = vigra.defaultAxistags('txyzc')
opProvider.Input.meta.has_mask = True
opProvider.Input.setValue(self.data)
self.opProvider = opProvider
opCache = OpSlicedBlockedArrayCache(graph=graph)
opCache.Input.connect(opProvider.Output)
opCache.innerBlockShape.setValue(
((10, 1, 10, 10, 10), (10, 10, 1, 10, 10), (10, 10, 10, 1, 10)))
opCache.outerBlockShape.setValue(
((20, 2, 20, 20, 20), (20, 20, 2, 20, 20), (20, 20, 20, 2, 20)))
opCache.fixAtCurrent.setValue(False)
self.opCache = opCache
def testBasic5d_masked(self):
logger.info("Generating sample data...")
sampleData = numpy.indices((3, 100, 200, 150, 2),
dtype=numpy.float32).sum(0)
sampleData = sampleData.view(numpy.ma.masked_array)
sampleData.set_fill_value(numpy.float32(numpy.nan))
sampleData[0] = numpy.ma.masked
graph = Graph()
opData = OpArrayPiperWithAccessCount(graph=graph)
opData.Input.meta.has_mask = True
opData.Input.meta.axistags = vigra.defaultAxistags('txyzc')
opData.Input.setValue(sampleData)
op = OpCompressedCache(parent=None, graph=graph)
#logger.debug("Setting block shape...")
op.BlockShape.setValue([1, 100, 75, 50, 2])
op.Input.connect(opData.Output)
assert op.Output.ready()
slicing = numpy.s_[0:2, 0:100, 50:150, 75:150, 0:1]
expectedData = sampleData[slicing]
#logger.debug("Requesting data...")
readData = op.Output[slicing].wait()
#logger.debug("Checking data...")
assert (readData == expectedData).all() and \
(readData.mask == expectedData.mask).all() and \
((readData.fill_value == expectedData.fill_value) |
(numpy.isnan(readData.fill_value) & numpy.isnan(expectedData.fill_value))).all(),\
"Incorrect output!"
assert opData.accessCount == 2 * 1 * 2 * 2 * 1, str(opData.accessCount)
def testCacheHandling(self):
n, k = 10, 5
vol = np.zeros((n, ) * 5, dtype=np.uint8)
vol = vigra.taggedView(vol, axistags='txyzc')
g = Graph()
pipe = OpArrayPiperWithAccessCount(graph=g)
cache = OpArrayCache(graph=g)
mgr = CacheMemoryManager()
# disallow cache memory
Memory.setAvailableRamCaches(0)
# set to frequent cleanup
mgr.setRefreshInterval(.01)
mgr.enable()
cache.blockShape.setValue((k, ) * 5)
cache.Input.connect(pipe.Output)
pipe.Input.setValue(vol)
a = pipe.accessCount
cache.Output[...].wait()
b = pipe.accessCount
assert b > a, "did not cache"
# let the manager clean up
mgr.enable()
time.sleep(.5)
gc.collect()
cache.Output[...].wait()
c = pipe.accessCount
assert c > b, "did not clean up"
def setUp(self):
self.graph = Graph()
self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph)
self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc")
self.operator_identity.Input.meta.has_mask = True
def testBasic5d(self):
logger.info("Generating sample data...")
sampleData = numpy.indices((3, 100, 200, 150, 2),
dtype=numpy.float32).sum(0)
sampleData = sampleData.view(vigra.VigraArray)
sampleData.axistags = vigra.defaultAxistags("txyzc")
graph = Graph()
opData = OpArrayPiperWithAccessCount(graph=graph)
opData.Input.setValue(sampleData)
op = OpCompressedCache(parent=None, graph=graph)
# logger.debug("Setting block shape...")
op.BlockShape.setValue([1, 100, 75, 50, 2])
op.Input.connect(opData.Output)
assert op.Output.ready()
slicing = numpy.s_[0:2, 0:100, 50:150, 75:150, 0:1]
expectedData = sampleData[slicing].view(numpy.ndarray)
# logger.debug("Requesting data...")
readData = op.Output[slicing].wait()
# logger.debug("Checking data...")
assert (readData == expectedData).all(), "Incorrect output!"
assert opData.accessCount == 2 * 1 * 2 * 2 * 1, str(opData.accessCount)
def setup_method(self, method):
self.graph = Graph()
self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph)
self.operator_identity.Input.allow_mask = False
self.operator_identity.Output.allow_mask = False
self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc")
def setup_method(self, method):
g = Graph()
vol = np.random.random(size=(100, 110, 120))
self.vol = vigra.taggedView(vol, axistags='xyz')
vol5d = np.random.random(size=(3, 100, 110, 120, 7))
self.vol5d = vigra.taggedView(vol5d, axistags='cxyzt')
piper = OpArrayPiperWithAccessCount(graph=g)
piper.Input.setValue(self.vol)
self.g = g
self.piper = piper
def setup_method(self, method):
self.dataShape = (1, 100, 100, 10, 1)
self.data = (numpy.random.random(self.dataShape) * 100).astype(numpy.uint32)
self.data = self.data.view(vigra.VigraArray)
self.data.axistags = vigra.defaultAxistags("txyzc")
graph = Graph()
opProvider = OpArrayPiperWithAccessCount(graph=graph)
opProvider.Input.setValue(self.data)
self.opProvider = opProvider
opCache = OpSlicedBlockedArrayCache(graph=graph)
opCache.Input.connect(opProvider.Output)
opCache.BlockShape.setValue(((20, 2, 20, 20, 20), (20, 20, 2, 20, 20), (20, 20, 20, 2, 20)))
opCache.fixAtCurrent.setValue(False)
self.opCache = opCache
def testCacheApi(self):
graph = Graph()
opDataProvider = OpArrayPiperWithAccessCount(graph=graph)
opCache = OpUnblockedArrayCache(graph=graph)
data = np.random.random((100, 100, 100)).astype(np.float32)
opDataProvider.Input.setValue(vigra.taggedView(data, 'zyx'))
opCache.Input.connect(opDataProvider.Output)
opCache.Output[10:20, 20:40, 50:100].wait()
opCache.Output[11:21, 22:43, 53:90].wait()
l = opCache.getBlockAccessTimes()
assert len(l) == 2
for k, t in l:
assert t > 0.0
def setUp(self):
self.dataShape = (1, 100, 100, 10, 1)
self.data = (numpy.random.random(self.dataShape) * 100).astype(int)
self.data = self.data.view(vigra.VigraArray)
self.data.axistags = vigra.defaultAxistags('txyzc')
graph = Graph()
opProvider = OpArrayPiperWithAccessCount(graph=graph)
opProvider.Input.setValue(self.data)
self.opProvider = opProvider
opCache = OpArrayCache(graph=graph)
opCache.Input.connect(opProvider.Output)
opCache.blockShape.setValue((10, 10, 10, 10, 10))
opCache.fixAtCurrent.setValue(False)
self.opCache = opCache
def setUp(self):
self.dataShape = (1, 100, 100, 10, 1)
self.data = numpy.random.randint(0, 256, size=self.dataShape)
self.data = self.data.astype(numpy.uint32)
self.data = self.data.view(vigra.VigraArray)
self.data.axistags = vigra.defaultAxistags('txyzc')
graph = Graph()
opProvider = OpArrayPiperWithAccessCount(graph=graph)
opProvider.Input.setValue(self.data)
self.opProvider = opProvider
opCache = OpBlockedArrayCache(graph=graph)
opCache.Input.connect(opProvider.Output)
opCache.innerBlockShape.setValue((10, 10, 10, 10, 10))
opCache.outerBlockShape.setValue((20, 20, 20, 20, 20))
opCache.fixAtCurrent.setValue(False)
self.opCache = opCache
def setUp(self):
g = Graph()
vol = np.random.random(size=(100, 110, 120))
self.vol = vigra.taggedView(vol, axistags='xyz')
vol5d = np.random.random(size=(3, 100, 110, 120, 7))
self.vol5d = vigra.taggedView(vol5d, axistags='cxyzt')
piper = OpArrayPiperWithAccessCount(graph=g)
op = OpCacheFixer(graph=g)
op.Input.connect(piper.Output)
call = OpCallWhenDirty(graph=g)
call.Input.connect(op.Output)
self.piper = piper
self.op = op
self.call = call
def testFree(self):
sampleData = numpy.indices((100, 200, 150), dtype=numpy.float32).sum(0)
sampleData = vigra.taggedView(sampleData, axistags='xyz')
graph = Graph()
opData = OpArrayPiperWithAccessCount(graph=graph)
opData.Input.setValue(sampleData)
op = OpCompressedCache(graph=graph)
#logger.debug("Setting block shape...")
op.BlockShape.setValue([100, 75, 50])
op.Input.connect(opData.Output)
op.Output[...].wait()
mem = op.usedMemory()
keys = [x[0] for x in op.getBlockAccessTimes()]
key = keys[0]
op.freeBlock(key)
assert op.usedMemory() < mem
def setup_method(self, method):
self.dataShape = (1, 100, 100, 10, 1)
self.data = (numpy.random.random(self.dataShape) * 100).astype(int)
self.data = numpy.ma.masked_array(
self.data, mask=numpy.ma.getmaskarray(self.data), fill_value=numpy.iinfo(int).max, shrink=False
)
self.data[:, 0] = numpy.ma.masked
graph = Graph()
opProvider = OpArrayPiperWithAccessCount(graph=graph)
opProvider.Input.meta.axistags = vigra.defaultAxistags("txyzc")
opProvider.Input.meta.has_mask = True
opProvider.Input.setValue(self.data)
self.opProvider = opProvider
opCache = OpBlockedArrayCache(graph=graph)
opCache.Input.connect(opProvider.Output)
opCache.BlockShape.setValue((20, 20, 20, 20, 20))
opCache.fixAtCurrent.setValue(False)
self.opCache = opCache
class TestOpArrayPiperWithAccessCount2(object):
def setup_method(self, method):
self.graph = Graph()
self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph)
self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc")
self.operator_identity.Input.meta.has_mask = True
def test1(self):
# Generate a random dataset and see if it we get the right masking from the operator.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(data,
mask=numpy.zeros(data.shape, dtype=bool),
shrink=False)
# Provide input read all output.
self.operator_identity.Input.setValue(data)
output = self.operator_identity.Output[None].wait()
assert self.operator_identity.accessCount == 1
assert (data == output).all()
assert data.mask.shape == output.mask.shape
assert (data.mask == output.mask).all()
def test2(self):
# Generate a dataset and grab chunks of it from the operator. The result should be the same as above.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(data,
mask=numpy.zeros(data.shape, dtype=bool),
shrink=False)
# Create array to store results. Don't keep original data.
output = data.copy()
output[:] = 0
output[:] = numpy.ma.nomask
# Provide input and grab chunks.
self.operator_identity.Input.setValue(data)
output[:2] = self.operator_identity.Output[:2].wait()
output[2:] = self.operator_identity.Output[2:].wait()
assert self.operator_identity.accessCount == 2
assert (data == output).all()
assert data.mask.shape == output.mask.shape
assert (data.mask == output.mask).all()
def test3(self):
# Generate a random dataset and see if it we get the right masking from the operator.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(data,
mask=numpy.zeros(data.shape, dtype=bool),
shrink=False)
# Provide input read all output.
self.operator_identity.Input.setValue(numpy.zeros_like(data))
output = self.operator_identity.Output[None].wait()
assert self.operator_identity.accessCount == 1
assert (output == 0).all()
assert data.mask.shape == output.mask.shape
assert (output.mask == False).all()
def teardown_method(self, method):
# Take down operators
self.operator_identity.Input.disconnect()
self.operator_identity.Output.disconnect()
self.operator_identity.cleanUp()
class TestOpArrayPiperWithAccessCount2(object):
def setUp(self):
self.graph = Graph()
self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph)
self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc")
self.operator_identity.Input.meta.has_mask = True
def test1(self):
# Generate a random dataset and see if it we get the right masking from the operator.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(
data,
mask=numpy.zeros(data.shape, dtype=bool),
shrink=False
)
# Provide input read all output.
self.operator_identity.Input.setValue(data)
output = self.operator_identity.Output[None].wait()
assert (self.operator_identity.accessCount == 1)
assert((data == output).all())
assert(data.mask.shape == output.mask.shape)
assert((data.mask == output.mask).all())
def test2(self):
# Generate a dataset and grab chunks of it from the operator. The result should be the same as above.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(
data,
mask=numpy.zeros(data.shape, dtype=bool),
shrink=False
)
# Create array to store results. Don't keep original data.
output = data.copy()
output[:] = 0
output[:] = numpy.ma.nomask
# Provide input and grab chunks.
self.operator_identity.Input.setValue(data)
output[:2] = self.operator_identity.Output[:2].wait()
output[2:] = self.operator_identity.Output[2:].wait()
assert (self.operator_identity.accessCount == 2)
assert((data == output).all())
assert(data.mask.shape == output.mask.shape)
assert((data.mask == output.mask).all())
def test3(self):
# Generate a random dataset and see if it we get the right masking from the operator.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(
data,
mask=numpy.zeros(data.shape, dtype=bool),
shrink=False
)
# Provide input read all output.
self.operator_identity.Input.setValue(numpy.zeros_like(data))
output = self.operator_identity.Output[None].wait()
assert (self.operator_identity.accessCount == 1)
assert((output == 0).all())
assert(data.mask.shape == output.mask.shape)
assert((output.mask == False).all())
def tearDown(self):
# Take down operators
self.operator_identity.Input.disconnect()
self.operator_identity.Output.disconnect()
self.operator_identity.cleanUp()
class TestOpArrayPiperWithAccessCount4(object):
def setup_method(self, method):
self.graph = Graph()
self.operator_identity = OpArrayPiperWithAccessCount(graph=self.graph)
self.operator_identity.Input.allow_mask = False
self.operator_identity.Output.allow_mask = False
self.operator_identity.Input.meta.has_mask = False
self.operator_identity.Output.meta.has_mask = False
self.operator_identity.Input.meta.axistags = vigra.AxisTags("txyzc")
def test1(self):
# Generate a random dataset and see if it we get the right masking from the operator.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(data,
mask=numpy.zeros(data.shape, dtype=bool),
shrink=False)
# Provide input read all output.
with pytest.raises(AllowMaskException):
try:
self.operator_identity.Input.setValue(data)
except AssertionError as e:
raise AllowMaskException(str(e))
def test2(self):
# Generate a dataset and grab chunks of it from the operator. The result should be the same as above.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(data,
mask=numpy.zeros(data.shape, dtype=bool),
shrink=False)
# Create array to store results. Don't keep original data.
output = data.copy()
output[:] = 0
output[:] = numpy.ma.nomask
# Provide input and grab chunks.
with pytest.raises(AllowMaskException):
try:
self.operator_identity.Input.setValue(data)
except AssertionError as e:
raise AllowMaskException(str(e))
def test3(self):
# Generate a random dataset and see if it we get the right masking from the operator.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(data,
mask=numpy.zeros(data.shape, dtype=bool),
shrink=False)
# Provide input read all output.
with pytest.raises(AllowMaskException):
try:
self.operator_identity.Input.setValue(numpy.zeros_like(data))
except AssertionError as e:
raise AllowMaskException(str(e))
def teardown_method(self, method):
# Take down operators
self.operator_identity.Input.disconnect()
self.operator_identity.Output.disconnect()
self.operator_identity.cleanUp()
def testBadMemoryConditions(self):
"""
TestCacheMemoryManager.testBadMemoryConditions
This test is a proof of the proposition in
https://github.com/ilastik/lazyflow/issue/185
which states that, given certain memory constraints, the cache
cleanup strategy in use is inefficient. An advanced strategy
should pass the test.
"""
mgr = _CacheMemoryManager()
mgr.setRefreshInterval(0.01)
mgr.enable()
d = 2
tags = "xy"
shape = (999,) * d
blockshape = (333,) * d
# restrict memory for computation to one block (including fudge
# factor 2 of bigRequestStreamer)
cacheMem = np.prod(shape)
Memory.setAvailableRam(np.prod(blockshape) * 2 + cacheMem)
# restrict cache memory to the whole volume
Memory.setAvailableRamCaches(cacheMem)
# to ease observation, do everything single threaded
Request.reset_thread_pool(num_workers=1)
x = np.zeros(shape, dtype=np.uint8)
x = vigra.taggedView(x, axistags=tags)
g = Graph()
pipe = OpArrayPiperWithAccessCount(graph=g)
pipe.Input.setValue(x)
pipe.Output.meta.ideal_blockshape = blockshape
# simulate BlockedArrayCache behaviour without caching
# cache = OpSplitRequestsBlockwise(True, graph=g)
# cache.BlockShape.setValue(blockshape)
# cache.Input.connect(pipe.Output)
cache = OpBlockedArrayCache(graph=g)
cache.Input.connect(pipe.Output)
cache.BlockShape.setValue(blockshape)
op = OpEnlarge(graph=g)
op.Input.connect(cache.Output)
split = OpSplitRequestsBlockwise(True, graph=g)
split.BlockShape.setValue(blockshape)
split.Input.connect(op.Output)
streamer = BigRequestStreamer(split.Output, [(0,) * len(shape), shape])
streamer.execute()
# in the worst case, we have 4*4 + 4*6 + 9 = 49 requests to pipe
# in the best case, we have 9
np.testing.assert_equal(pipe.accessCount, 9)
