def testChooseChunkShape(self):
a = (15, )
b = chooseChunkShape(a, 3)
assert_array_equal(b, (3, ))
a = (20, 50)
b = chooseChunkShape(a, 10)
assert_array_equal(b, (2, 5))
a = (20, 50)
b = chooseChunkShape(a, 3000)
assert_array_equal(b, (20, 50))
a = (17, 33) # roughly 1:2
b = chooseChunkShape(a, 3)
assert_array_equal(b, (1, 2))
def testChooseChunkShape(self):
a = (15,)
b = chooseChunkShape(a, 3)
assert_array_equal(b, (3,))
a = (20, 50)
b = chooseChunkShape(a, 10)
assert_array_equal(b, (2, 5))
a = (20, 50)
b = chooseChunkShape(a, 3000)
assert_array_equal(b, (20, 50))
a = (17, 33) # roughly 1:2
b = chooseChunkShape(a, 3)
assert_array_equal(b, (1, 2))
def _chooseChunkshape(self, blockshape):
"""
Choose an optimal chunkshape for our blockshape and Input shape.
We assume access patterns to vary more in space than in time or channel
and choose the inner chunk shape to be about 1MiB slices of t and c.
Furthermore, we use the function
lazyflow.utility.chunkHelpers.chooseChunkShape()
to preserve the aspect ratio of the input (at least approximately).
"""
if blockshape is None:
return None
def isConsistent(idealshape):
"""
check if ideal block shape and given block shape are consistent
shapes are consistent if, for each dimension,
* input is unready, or
* blockshape equals fullshape, or
* idealshape divides blockshape evenly
"""
if not self.Input.ready():
return True
fullshape = self.Input.meta.shape
z = zip(idealshape, blockshape, fullshape)
m = map(lambda (i, b, f): b == f or b % i == 0, z)
return all(m)
if not self._ignore_ideal_blockshape and self.Input.ready():
# take the ideal chunk shape, but check if sane
ideal = self.Input.meta.ideal_blockshape
if ideal is not None:
if len(ideal) == len(blockshape):
ideal = numpy.asarray(ideal, dtype=numpy.int)
for i, d in enumerate(ideal):
if d == 0:
ideal[i] = blockshape[i]
if not isConsistent(ideal):
logger.warn("{}: BlockShape and ideal_blockshape are "
"inconsistent {} vs {}".format(self.name, blockshape, ideal))
else:
return tuple(ideal)
else:
logger.warn("{}: Encountered meta.ideal_blockshape that does "
"not fit the data".format(self.name))
# we need to figure out an ideal chunk shape on our own
# Start with a copy of blockshape
axes = self.Output.meta.getTaggedShape().keys()
taggedBlockShape = collections.OrderedDict(zip(axes, self._blockshape))
dtypeBytes = self._getDtypeBytes(self.Output.meta.dtype)
desiredSpace = 1024**2 / float(dtypeBytes)
if numpy.prod(blockshape) <= desiredSpace:
return blockshape
# set t and c to 1
for key in 'tc':
if key in taggedBlockShape:
taggedBlockShape[key] = 1
logger.debug("desired space: {}".format(desiredSpace))
# extract only the spatial shape
spatialKeys = [k for k in taggedBlockShape.keys() if k in 'xyz']
spatialShape = [taggedBlockShape[k] for k in spatialKeys]
newSpatialShape = chooseChunkShape(spatialShape, desiredSpace)
for k, v in zip(spatialKeys, newSpatialShape):
taggedBlockShape[k] = v
chunkShape = tuple(taggedBlockShape.values())
logger.debug("Using chunk shape: {}".format(chunkShape))
return chunkShape
