def test_invalid_parameters(self):
with self.assertRaises(AssertionError):
getIntersectingBlocks((256, 256, 0, 2),
([0, 0, 0, 0], [256, 256, 256, 2]))
with self.assertRaises(AssertionError):
getIntersectingBlocks(numpy.array((256, 256, 0, 2)),
([0, 0, 0, 0], [256, 256, 256, 2]))
def _executeOutput(self, roi, destination):
assert len(roi.stop) == len(self.Input.meta.shape), "roi: {} has the wrong number of dimensions for Input shape: {}".format( roi, self.Input.meta.shape )
assert numpy.less_equal(roi.stop, self.Input.meta.shape).all(), "roi: {} is out-of-bounds for Input shape: {}".format( roi, self.Input.meta.shape )
block_starts = getIntersectingBlocks( self._blockshape, (roi.start, roi.stop) )
block_starts = map( tuple, block_starts )
# Ensure all block cache files are up-to-date
reqPool = RequestPool() # (Do the work in parallel.)
for block_start in block_starts:
entire_block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, block_start )
f = partial( self._ensureCached, entire_block_roi)
reqPool.add( Request(f) )
logger.debug( "Waiting for {} blocks...".format( len(block_starts) ) )
reqPool.wait()
# Copy data from each block
# (Parallelism not needed here: h5py will serialize these requests anyway)
logger.debug( "Copying data from {} blocks...".format( len(block_starts) ) )
for block_start in block_starts:
entire_block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, block_start )
# This block's portion of the roi
intersecting_roi = getIntersection( (roi.start, roi.stop), entire_block_roi )
# Compute slicing within destination array and slicing within this block
destination_relative_intersection = numpy.subtract(intersecting_roi, roi.start)
block_relative_intersection = numpy.subtract(intersecting_roi, block_start)
# Copy from block to destination
dataset = self._getBlockDataset( entire_block_roi )
destination[ roiToSlice(*destination_relative_intersection) ] = dataset[ roiToSlice( *block_relative_intersection ) ]
return destination
def _executePredictionImage(self, roi, destination):
# Determine intersecting blocks
block_shape = self._getFullShape(self.BlockShape3dDict.value)
block_starts = getIntersectingBlocks(block_shape,
(roi.start, roi.stop))
block_starts = map(tuple, block_starts)
# Ensure that block pipelines exist (create first if necessary)
for block_start in block_starts:
self._ensurePipelineExists(block_start)
# Retrieve result from each block, and write into the appropriate region of the destination
# TODO: Parallelize this loop
for block_start in block_starts:
opBlockPipeline = self._blockPipelines[block_start]
block_roi = opBlockPipeline.block_roi
block_intersection = getIntersection(block_roi,
(roi.start, roi.stop))
block_relative_intersection = numpy.subtract(
block_intersection, block_roi[0])
destination_relative_intersection = numpy.subtract(
block_intersection, roi.start)
destination_slice = roiToSlice(*destination_relative_intersection)
req = opBlockPipeline.PredictionImage(*block_relative_intersection)
req.writeInto(destination[destination_slice])
req.wait()
return destination
def testFailedProcessing(self):
op = OpArrayPiper( graph=Graph() )
inputData = numpy.indices( (100,100) ).sum(0)
op.Input.setValue( inputData )
roiList = []
block_starts = getIntersectingBlocks( [10,10], ([0,0], [100, 100]) )
for block_start in block_starts:
roiList.append( getBlockBounds( [100,100], [10,10], block_start ) )
class SpecialException(Exception): pass
def handleResult(roi, result):
raise SpecialException("Intentional Exception: raised while handling the result")
totalVolume = numpy.prod( inputData.shape )
batch = RoiRequestBatch(op.Output, roiList.__iter__(), totalVolume, batchSize=10, allowParallelResults=False)
batch.resultSignal.subscribe( handleResult )
# FIXME: There are multiple places where the RoiRequestBatch tool should be prepared to handle exceptions.
# This only tests one of them (in the notify_finished() handler)
try:
batch.execute()
except SpecialException:
pass
else:
assert False, "Expected exception to be propagated out of the RoiRequestBatch."
def find_nearest_connector(self, detection_coord ):
"""
Search the given buckets of connectors for the one that is nearest to the given coordinates.
Buckets farther than SEARCH_RADIUS are not searched, in which case a default ConnectorInfo object is returned.
Returns: nearest_connector, distance to the nearest connector
"""
# Find nearby blocks
detection_coord_int = detection_coord.astype(int)
search_roi = ( detection_coord_int - self.SEARCH_RADIUS,
detection_coord_int + self.SEARCH_RADIUS )
nearby_block_starts = getIntersectingBlocks(self._blockshape, search_roi)
nearby_block_starts = map(tuple, nearby_block_starts)
# Accumulate connectors found in nearby blocks
nearby_connectors = []
for block_start in nearby_block_starts:
if block_start in self._blocks:
nearby_connectors += self._blocks[block_start]
# Closure. Distance from current point to given connector.
def distance( conn ):
return scipy.spatial.distance.euclidean( (conn.x_nm, conn.y_nm, conn.z_nm), detection_coord )
# Find closest connector.
if nearby_connectors:
nearest_connector = min(nearby_connectors, key=distance)
min_distance = distance( nearest_connector )
else:
# No connectors nearby. Emit default values.
nearest_connector = ConnectorInfo(-1, -1, -1, -1, [], [])
min_distance = 9999999.0
return nearest_connector, min_distance
def ingestData(self, slot):
"""
Read the data from the given slot and copy it into this cache.
The rules about special pixel meanings apply here, just like setInSlot
Returns: the max label found in the slot.
"""
assert self._blockshape is not None
assert self.Input.meta.shape == slot.meta.shape
max_label = 0
# Get logical blocking.
block_starts = getIntersectingBlocks( self._blockshape, roiFromShape(self.Input.meta.shape) )
block_starts = map( tuple, block_starts )
# Write each block
for block_start in block_starts:
block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, block_start )
# Request the block data
block_data = slot(*block_roi).wait()
# Write into the array
subregion_roi = SubRegion(self.Input, *block_roi)
cleaned_block_data = self._setInSlotInput( self.Input, (), subregion_roi, block_data )
max_label = max( max_label, cleaned_block_data.max() )
return max_label
def propagateDirty(self, slot, subindex, roi):
if slot == self.NonZeroLabelBlocks:
# Label changes will be handled via labelimage dirtyness propagation
return
assert slot == self.FeatureImage or slot == self.LabelImage
# Our blocks are tracked by label roi (1 channel)
roi = roi.copy()
roi.start[-1] = 0
roi.stop[-1] = 1
# Bookkeeping: Track the dirty blocks
block_starts = getIntersectingBlocks(self._blockshape,
(roi.start, roi.stop))
block_starts = map(tuple, block_starts)
#
# If the features were dirty (not labels), we only really care about
# the blocks that are actually stored already
# For big dirty rois (e.g. the entire image),
# we avoid a lot of unecessary entries in self._dirty_blocks
if slot == self.FeatureImage:
block_starts = set(block_starts).intersection(
self._blockwise_feature_matrices.keys())
with self._lock:
self._dirty_blocks.update(block_starts)
# Output has no notion of roi. It's all dirty.
self.LabelAndFeatureMatrix.setDirty()
def getAllBlockRois(self):
entire_dataset_roi = ([0]*len(self.description.view_shape), self.description.view_shape)
block_starts = getIntersectingBlocks(self.description.block_shape, entire_dataset_roi)
rois = []
for block_start in block_starts:
rois.append(self.getEntireBlockRoi(block_start))
return rois
def testFailedProcessing(self):
op = OpArrayPiper(graph=Graph())
inputData = numpy.indices((100, 100)).sum(0)
op.Input.setValue(inputData)
roiList = []
block_starts = getIntersectingBlocks([10, 10], ([0, 0], [100, 100]))
for block_start in block_starts:
roiList.append(getBlockBounds([100, 100], [10, 10], block_start))
class SpecialException(Exception):
pass
def handleResult(roi, result):
raise SpecialException(
"Intentional Exception: raised while handling the result")
totalVolume = numpy.prod(inputData.shape)
batch = RoiRequestBatch(op.Output,
roiList.__iter__(),
totalVolume,
batchSize=10,
allowParallelResults=False)
batch.resultSignal.subscribe(handleResult)
# FIXME: There are multiple places where the RoiRequestBatch tool should be prepared to handle exceptions.
# This only tests one of them (in the notify_finished() handler)
try:
batch.execute()
except SpecialException:
pass
else:
assert False, "Expected exception to be propagated out of the RoiRequestBatch."
def _transferData(self, roi, array_data, read):
"""
Read or write data from/to the fileset.
:param roi: The region of interest.
:param array_data: If ``read`` is True, ``array_data`` is the destination array for the read data. If ``read`` is False, array_data contains the data to write to disk.
:param read: If True, read data from the fileset into ``array_data``. Otherwise, write data from ``array_data`` into the fileset on disk.
:type read: bool
"""
entire_dataset_roi = ([0] * len(self._description.view_shape), self._description.view_shape)
clipped_roi = getIntersection(roi, entire_dataset_roi)
assert (
numpy.array(clipped_roi) == numpy.array(roi)
).all(), "Roi {} does not fit within dataset bounds: {}".format(roi, self._description.view_shape)
block_starts = getIntersectingBlocks(self._description.block_shape, roi)
# TODO: Parallelize this loop?
for block_start in block_starts:
entire_block_roi = self.getEntireBlockRoi(block_start) # Roi of this whole block within the whole dataset
transfer_block_roi = getIntersection(
entire_block_roi, roi
) # Roi of data needed from this block within the whole dataset
block_relative_roi = (
transfer_block_roi[0] - block_start,
transfer_block_roi[1] - block_start,
) # Roi of needed data from this block, relative to the block itself
array_data_roi = (
transfer_block_roi[0] - roi[0],
transfer_block_roi[1] - roi[0],
) # Roi of data needed from this block within array_data
array_slicing = roiToSlice(*array_data_roi)
self._transferBlockData(entire_block_roi, block_relative_roi, array_data, array_slicing, read)
def _executePredictionImage(self, roi, destination):
# Determine intersecting blocks
block_shape = self._getFullShape( self.BlockShape3dDict.value )
block_starts = getIntersectingBlocks( block_shape, (roi.start, roi.stop) )
block_starts = map( tuple, block_starts )
# Ensure that block pipelines exist (create first if necessary)
for block_start in block_starts:
self._ensurePipelineExists(block_start)
# Retrieve result from each block, and write into the approprate region of the destination
# TODO: Parallelize this loop
for block_start in block_starts:
opBlockPipeline = self._blockPipelines[block_start]
block_roi = opBlockPipeline.block_roi
block_intersection = getIntersection( block_roi, (roi.start, roi.stop) )
block_relative_intersection = numpy.subtract(block_intersection, block_roi[0])
destination_relative_intersection = numpy.subtract(block_intersection, roi.start)
destination_slice = roiToSlice( *destination_relative_intersection )
req = opBlockPipeline.PredictionImage( *block_relative_intersection )
req.writeInto( destination[destination_slice] )
req.wait()
return destination
def _executeBlockwiseRegionFeatures(self, roi, destination):
"""
Provide data for the BlockwiseRegionFeatures slot.
Note: Each block produces a single element of this slot's output. Construct requested roi coordinates accordingly.
e.g. if block_shape is (1,10,10,10,1), the features for the block starting at
(1,20,30,40,5) should be requested via roi [(1,2,3,4,5),(2,3,4,5,6)]
Note: It is assumed that you will request these features for debug purposes, AFTER requesting the prediction image.
Therefore, it is considered an error to request features that are not already computed.
"""
axiskeys = self.RawImage.meta.getAxisKeys()
# Find the corresponding block start coordinates
block_shape = self._getFullShape( self.BlockShape3dDict.value )
pixel_roi = numpy.array(block_shape) * (roi.start, roi.stop)
block_starts = getIntersectingBlocks( block_shape, pixel_roi )
block_starts = map( tuple, block_starts )
for block_start in block_starts:
assert block_start in self._blockPipelines, "Not allowed to request region features for blocks that haven't yet been processed." # See note above
# Discard spatial axes to get (t,c) index for region slot roi
tagged_block_start = zip( axiskeys, block_start )
tagged_block_start_tc = filter( lambda (k,v): k in 'tc', tagged_block_start )
block_start_tc = map( lambda (k,v): v, tagged_block_start_tc )
block_roi_tc = ( block_start_tc, block_start_tc + numpy.array([1,1]) )
destination_start = numpy.array(block_start) / block_shape - roi.start
destination_stop = destination_start + numpy.array( [1]*len(axiskeys) )
opBlockPipeline = self._blockPipelines[block_start]
req = opBlockPipeline.BlockwiseRegionFeatures( *block_roi_tc )
req.writeInto( destination[ roiToSlice( destination_start, destination_stop ) ] )
req.wait()
return destination
def _setInSlotInput(self, slot, subindex, roi, value):
assert len(roi.stop) == len(self.Input.meta.shape), "roi: {} has the wrong number of dimensions for Input shape: {}".format( roi, self.Input.meta.shape )
assert numpy.less_equal(roi.stop, self.Input.meta.shape).all(), "roi: {} is out-of-bounds for Input shape: {}".format( roi, self.Input.meta.shape )
block_starts = getIntersectingBlocks( self._blockshape, (roi.start, roi.stop) )
block_starts = map( tuple, block_starts )
# Copy data to each block
logger.debug( "Copying data INTO {} blocks...".format( len(block_starts) ) )
for block_start in block_starts:
entire_block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, block_start )
# This block's portion of the roi
intersecting_roi = getIntersection( (roi.start, roi.stop), entire_block_roi )
# Compute slicing within source array and slicing within this block
source_relative_intersection = numpy.subtract(intersecting_roi, roi.start)
block_relative_intersection = numpy.subtract(intersecting_roi, block_start)
# Copy from source to block
dataset = self._getBlockDataset( entire_block_roi )
dataset[ roiToSlice( *block_relative_intersection ) ] = value[ roiToSlice(*source_relative_intersection) ]
# Here, we assume that if this function is used to update ANY PART of a
# block, he is responsible for updating the ENTIRE block.
# Therefore, this block is no longer 'dirty'
self._dirtyBlocks.discard( block_start )
def __init__(self, outputSlot, roi, minBlockShape, batchSize=None):
self._outputSlot = outputSlot
self._bigRoi = roi
self._minBlockShape = minBlockShape
if batchSize is None:
batchSize=2
# Align the blocking with the start of the roi
offsetRoi = ([0] * len(roi[0]), numpy.subtract(roi[1], roi[0]))
self._minBlockStarts = getIntersectingBlocks(minBlockShape, offsetRoi)
self._minBlockStarts += roi[0] # Un-offset
totalVolume = numpy.prod( numpy.subtract(roi[1], roi[0]) )
# For now, simply iterate over the min blocks
# TODO: Auto-dialate block sizes based on CPU/RAM usage.
def roiGen():
block_iter = self._minBlockStarts.__iter__()
while True:
block_start = block_iter.next()
# Use offset blocking
offset_block_start = block_start - self._bigRoi[0]
offset_data_shape = numpy.subtract(self._bigRoi[1], self._bigRoi[0])
offset_block_bounds = getBlockBounds( offset_data_shape, minBlockShape, offset_block_start )
# Un-offset
block_bounds = ( offset_block_bounds[0] + self._bigRoi[0],
offset_block_bounds[1] + self._bigRoi[0] )
logger.debug( "Requesting Roi: {}".format( block_bounds ) )
yield block_bounds
self._requestBatch = RoiRequestBatch( self._outputSlot, roiGen(), totalVolume, batchSize )
def propagateDirty(self, slot, subindex, roi):
if slot == self.Labels:
if not self.ModelSession.ready():
return
try:
model_session = self.ModelSession.value
image_slot = self.Images[subindex]
label_slot = self.Labels[subindex]
# todo: get block shape in a less hacky way
block_shape = self.BlockShape[subindex].value
block_starts = getIntersectingBlocks(block_shape,
(roi.start, roi.stop))
label_shape = label_slot.meta.shape
axis_keys = label_slot.meta.getAxisKeys()
block_slicings = [[
slice(None) if axis == "c" else slice(dmax -
dblock, dmax) if
dstart + dblock > dmax else slice(dstart, dstart + dblock)
for dstart, dblock, dmax, axis in zip(
block_start, block_shape, label_shape, axis_keys)
] for block_start in block_starts]
image_blocks, label_blocks, block_ids = self._collect_blocks(
image_slot, label_slot, block_slicings)
axistags = self.Images[0].meta.axistags
model_session.update(image_blocks, label_blocks, axistags,
block_ids)
except Exception as e:
logger.error(e, exc_info=True)
else:
self.UpdatedModelSession.setDirty()
def propagateDirty(self, slot, subindex, roi):
assert slot == self.FeatureImage or slot == self.LabelImage
# Our blocks are tracked by label roi (1 channel)
roi = roi.copy()
roi.start[-1] = 0
roi.stop[-1] = 1
# Bookkeeping: Track the dirty blocks
# If the features were dirty (not labels), we only really care about
# the blocks that are actually stored already
# For big dirty rois (e.g. the entire image),
# we avoid a lot of unnecessary entries in self._dirty_blocks
if slot == self.FeatureImage:
# We ignore the ROI and assume all blocks are dirty.
# Technically, this would be inefficient if it's possible for the features
# to become only partially dirty in a small ROI.
# But currently, there is no known use-case for that.
block_starts = list(self._blockwise_feature_matrices.keys())
else:
block_starts = getIntersectingBlocks(self._blockshape, (roi.start, roi.stop))
block_starts = list(map(tuple, block_starts))
with self._lock:
self._dirty_blocks.update(set(block_starts))
# Output has no notion of roi. It's all dirty.
self.LabelAndFeatureMatrix.setDirty()
def propagateDirty(self, slot, subindex, roi):
if slot == self.NonZeroLabelBlocks:
# Label changes will be handled via labelimage dirtyness propagation
return
assert slot == self.FeatureImage or slot == self.LabelImage
# Our blocks are tracked by label roi (1 channel)
roi = roi.copy()
roi.start[-1] = 0
roi.stop[-1] = 1
# Bookkeeping: Track the dirty blocks
block_starts = getIntersectingBlocks( self._blockshape, (roi.start, roi.stop) )
block_starts = map( tuple, block_starts )
#
# If the features were dirty (not labels), we only really care about
# the blocks that are actually stored already
# For big dirty rois (e.g. the entire image),
# we avoid a lot of unnecessary entries in self._dirty_blocks
if slot == self.FeatureImage:
block_starts = set( block_starts ).intersection( self._blockwise_feature_matrices.keys() )
with self._lock:
self._dirty_blocks.update( block_starts )
# Output has no notion of roi. It's all dirty.
self.LabelAndFeatureMatrix.setDirty()
def read(self, view_roi, result_out):
"""
roi: (start, stop) tuples, ordered according to description.output_axes
roi should be relative to the view
"""
output_axes = self.description.output_axes
roi_transposed = zip(*view_roi)
roi_dict = dict( zip(output_axes, roi_transposed) )
view_roi = zip( *(roi_dict['z'], roi_dict['y'], roi_dict['x']) )
# First, normalize roi and result to zyx order
result_out = vigra.taggedView(result_out, output_axes)
result_out = result_out.withAxes(*'zyx')
assert numpy.array(view_roi).shape == (2,3), "Invalid roi for 3D volume: {}".format( view_roi )
view_roi = numpy.array(view_roi)
assert (result_out.shape == (view_roi[1] - view_roi[0])).all()
# User gave roi according to the view output.
# Now offset it find global roi.
roi = view_roi + self.description.view_origin_zyx
tile_blockshape = (1,) + tuple(self.description.tile_shape_2d_yx)
tile_starts = getIntersectingBlocks( tile_blockshape, roi )
pool = RequestPool()
for tile_start in tile_starts:
tile_roi_in = getBlockBounds( self.description.bounds_zyx, tile_blockshape, tile_start )
tile_roi_in = numpy.array(tile_roi_in)
# This tile's portion of the roi
intersecting_roi = getIntersection( roi, tile_roi_in )
intersecting_roi = numpy.array( intersecting_roi )
# Compute slicing within destination array and slicing within this tile
destination_relative_intersection = numpy.subtract(intersecting_roi, roi[0])
tile_relative_intersection = intersecting_roi - tile_roi_in[0]
# Get a view to the output slice
result_region = result_out[roiToSlice(*destination_relative_intersection)]
rest_args = self._get_rest_args(tile_blockshape, tile_roi_in)
if self.description.tile_url_format.startswith('http'):
retrieval_fn = partial( self._retrieve_remote_tile, rest_args, tile_relative_intersection, result_region )
else:
retrieval_fn = partial( self._retrieve_local_tile, rest_args, tile_relative_intersection, result_region )
PARALLEL_REQ = True
if PARALLEL_REQ:
pool.add( Request( retrieval_fn ) )
else:
# execute serially (leave the pool empty)
retrieval_fn()
if PARALLEL_REQ:
with Timer() as timer:
pool.wait()
logger.info("Loading {} tiles took a total of {}".format( len(tile_starts), timer.seconds() ))
def read(self, view_roi, result_out):
"""
roi: (start, stop) tuples, ordered according to description.output_axes
roi should be relative to the view
"""
output_axes = self.description.output_axes
roi_transposed = list(zip(*view_roi))
roi_dict = dict(list(zip(output_axes, roi_transposed)))
view_roi = list(zip(*(roi_dict["z"], roi_dict["y"], roi_dict["x"])))
# First, normalize roi and result to zyx order
result_out = vigra.taggedView(result_out, output_axes)
result_out = result_out.withAxes(*"zyx")
assert numpy.array(view_roi).shape == (2, 3), "Invalid roi for 3D volume: {}".format(view_roi)
view_roi = numpy.array(view_roi)
assert (result_out.shape == (view_roi[1] - view_roi[0])).all()
# User gave roi according to the view output.
# Now offset it find global roi.
roi = view_roi + self.description.view_origin_zyx
tile_blockshape = (1,) + tuple(self.description.tile_shape_2d_yx)
tile_starts = getIntersectingBlocks(tile_blockshape, roi)
pool = RequestPool()
for tile_start in tile_starts:
tile_roi_in = getBlockBounds(self.description.bounds_zyx, tile_blockshape, tile_start)
tile_roi_in = numpy.array(tile_roi_in)
# This tile's portion of the roi
intersecting_roi = getIntersection(roi, tile_roi_in)
intersecting_roi = numpy.array(intersecting_roi)
# Compute slicing within destination array and slicing within this tile
destination_relative_intersection = numpy.subtract(intersecting_roi, roi[0])
tile_relative_intersection = intersecting_roi - tile_roi_in[0]
# Get a view to the output slice
result_region = result_out[roiToSlice(*destination_relative_intersection)]
rest_args = self._get_rest_args(tile_blockshape, tile_roi_in)
if self.description.tile_url_format.startswith("http"):
retrieval_fn = partial(self._retrieve_remote_tile, rest_args, tile_relative_intersection, result_region)
else:
retrieval_fn = partial(self._retrieve_local_tile, rest_args, tile_relative_intersection, result_region)
PARALLEL_REQ = True
if PARALLEL_REQ:
pool.add(Request(retrieval_fn))
else:
# execute serially (leave the pool empty)
retrieval_fn()
if PARALLEL_REQ:
with Timer() as timer:
pool.wait()
logger.info("Loading {} tiles took a total of {}".format(len(tile_starts), timer.seconds()))
def autoSeedBackground(cls, laneView, foreground_label):
# Seed the entire image with background labels, except for the individual label in question
# To save memory, we'll do this in blocks instead of all at once
volume_shape = laneView.RavelerLabels.meta.shape
volume_roi = roiFromShape(volume_shape)
block_shape = (OpSplitBodyCarving.BLOCK_SIZE, ) * len(volume_shape)
block_shape = numpy.minimum(block_shape, volume_shape)
block_starts = getIntersectingBlocks(block_shape, volume_roi)
logger.debug("Auto-seeding {} blocks for label".format(
len(block_starts), foreground_label))
for block_index, block_start in enumerate(block_starts):
block_roi = getBlockBounds(volume_shape, block_shape, block_start)
label_block = laneView.RavelerLabels(*block_roi).wait()
background_block = numpy.where(label_block == foreground_label, 0,
1)
background_block = numpy.asarray(
background_block,
numpy.float32) # Distance transform requires float
if (background_block == 0.0).any():
# We need to leave a small border between the background seeds and the object membranes
background_block_view = background_block.view(vigra.VigraArray)
background_block_view.axistags = copy.copy(
laneView.RavelerLabels.meta.axistags)
background_block_view_4d = background_block_view.bindAxis(
't', 0)
background_block_view_3d = background_block_view_4d.bindAxis(
'c', 0)
distance_transformed_block = vigra.filters.distanceTransform(
background_block_view_3d, background=False)
distance_transformed_block = distance_transformed_block.astype(
numpy.uint8)
# Create a 'hull' surrounding the foreground, but leave some space.
background_seed_block = (distance_transformed_block ==
OpSplitBodyCarving.SEED_MARGIN)
background_seed_block = background_seed_block.astype(
numpy.uint8) * 1 # (In carving, background is label 1)
# # Make the hull VERY sparse to avoid over-biasing graph cut toward the background class
# # FIXME: Don't regenerate this random block on every loop iteration
# rand_bytes = numpy.random.randint(0, 1000, background_seed_block.shape)
# background_seed_block = numpy.where( rand_bytes < 1, background_seed_block, 0 )
# background_seed_block = background_seed_block.view(vigra.VigraArray)
# background_seed_block.axistags = background_block_view_3d.axistags
axisorder = laneView.RavelerLabels.meta.getTaggedShape().keys()
logger.debug("Writing backgound seeds: {}/{}".format(
block_index, len(block_starts)))
laneView.WriteSeeds[roiToSlice(
*block_roi)] = background_seed_block.withAxes(*axisorder)
else:
logger.debug("Skipping all-background block: {}/{}".format(
block_index, len(block_starts)))
def purgeAllLocks(self):
view_shape = self.localDescription.view_shape
view_roi = ([0]*len(view_shape), view_shape)
block_starts = list( getIntersectingBlocks(self.localDescription.block_shape, view_roi) )
for block_start in block_starts:
entire_block_roi = self.getEntireBlockRoi(block_start) # Roi of this whole block within the whole dataset
blockFilePathComponents = self.getDatasetPathComponents( block_start )
fileLock = FileLock( blockFilePathComponents.externalPath )
fileLock.purge()
def _setInSlotInput(self,
slot,
subindex,
roi,
value,
store_zero_blocks=True):
"""
Write the data in the array 'value' into the cache.
If the optional store_zero_blocks param is False, then don't bother
creating cache blocks for blocks that are totally zero.
"""
assert len(roi.stop) == len(self.Input.meta.shape), \
"roi: {} has the wrong number of dimensions for Input shape: {}"\
"".format( roi, self.Input.meta.shape )
assert numpy.less_equal(roi.stop, self.Input.meta.shape).all(), \
"roi: {} is out-of-bounds for Input shape: {}"\
"".format( roi, self.Input.meta.shape )
block_starts = getIntersectingBlocks(self._blockshape,
(roi.start, roi.stop))
block_starts = map(tuple, block_starts)
# Copy data to each block
logger.debug("Copying data INTO {} blocks...".format(
len(block_starts)))
for block_start in block_starts:
entire_block_roi = getBlockBounds(self.Output.meta.shape,
self._blockshape, block_start)
# This block's portion of the roi
intersecting_roi = getIntersection((roi.start, roi.stop),
entire_block_roi)
# Compute slicing within source array and slicing within this block
source_relative_intersection = numpy.subtract(
intersecting_roi, roi.start)
block_relative_intersection = numpy.subtract(
intersecting_roi, block_start)
new_block_data = value[roiToSlice(*source_relative_intersection)]
if not store_zero_blocks and new_block_data.sum(
) == 0 and block_start not in self._cacheFiles:
# Special fast-path: If this block doesn't exist yet,
# don't bother creating if we're just going to fill it with zeros
# (Used by the OpCompressedUserLabelArray)
pass
else:
# Copy from source to block
dataset = self._getBlockDataset(entire_block_roi)
dataset[roiToSlice(
*block_relative_intersection)] = new_block_data
# Here, we assume that if this function is used to update ANY PART of a
# block, he is responsible for updating the ENTIRE block.
# Therefore, this block is no longer 'dirty'
self._dirtyBlocks.discard(block_start)
def getAllBlockRois(self):
"""
Return the list of rois for all VIEWED blocks in the dataset.
"""
entire_dataset_roi = ([0] * len(self._description.view_shape), self._description.view_shape)
block_starts = getIntersectingBlocks(self._description.block_shape, entire_dataset_roi)
rois = []
for block_start in block_starts:
rois.append(self.getEntireBlockRoi(block_start))
return rois
def _executeOutput(self, roi, destination):
assert len(roi.stop) == len(self.Input.meta.shape), \
"roi: {} has the wrong number of dimensions for Input shape: {}"\
"".format( roi, self.Input.meta.shape )
assert numpy.less_equal(roi.stop, self.Input.meta.shape).all(), \
"roi: {} is out-of-bounds for Input shape: {}"\
"".format( roi, self.Input.meta.shape )
block_starts = getIntersectingBlocks( self._blockshape, (roi.start, roi.stop) )
self._copyData(roi, destination, block_starts)
return destination
def downloadAllBlocks(self, max_parallel):
"""
Download all blocks in the local view.
"""
view_shape = self.localDescription.view_shape
view_roi = ([0]*len(view_shape), view_shape)
block_starts = list( getIntersectingBlocks(self.localDescription.block_shape, view_roi) )
while len(block_starts) > 0:
batch = []
for _ in range( max_parallel ):
batch.append( block_starts.pop() )
logger.debug( "Next batch: {}".format(batch) )
self._waitForBlocks( batch )
def test_1_BasicWrite(self):
"""
This 'test' writes all blocks of the dataset.
SIDE EFFECT: This test is run first because the other tests use the dataset that it produces.
"""
logger.debug( "Writing test data..." )
totalRoi = ([0,0,0,0,0], self.dataShape)
self.bfs.writeData( totalRoi, self.data )
# All blocks are now available.
allBlockStarts = getIntersectingBlocks( self.bfs.description.block_shape, totalRoi )
for blockStart in allBlockStarts:
self.bfs.setBlockStatus(blockStart, BlockwiseFileset.BLOCK_AVAILABLE)
def readData(self, roi, out_array=None):
assert (numpy.array(roi[1]) <= numpy.array(self.localDescription.view_shape)).all(), "Requested roi '{}' is out of dataset bounds '{}'".format(roi, self.localDescription.view_shape)
# Before reading the data, check each of the needed blocks and download them first
block_starts = getIntersectingBlocks(self.localDescription.block_shape, roi)
missing_blocks = []
for block_start in block_starts:
# Offset to get the global (non-view) coordinates of the block.
if self.getBlockStatus(block_start) == BlockwiseFileset.BLOCK_NOT_AVAILABLE:
missing_blocks.append( block_start )
self._waitForBlocks( missing_blocks )
return super( RESTfulBlockwiseFileset, self ).readData( roi, out_array )
def setUp(self):
"""
Create a blockwise fileset to test with.
"""
if platform.system() == 'Windows':
# On windows, there are errors, and we make no attempt to solve them (at the moment).
raise nose.SkipTest
try:
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
raise nose.SkipTest
testConfig = \
"""
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "synapse_small",
"format" : "hdf5",
"axes" : "txyzc",
"shape" : [1,400,400,100,1],
"dtype" : "numpy.uint8",
"block_shape" : [1, 50, 50, 50, 100],
"block_file_name_format" : "cube{roiString}.h5/volume/data"
}
"""
self.tempDir = tempfile.mkdtemp()
self.configpath = os.path.join(self.tempDir, "config.json")
logger.debug("Loading config file...")
with open(self.configpath, 'w') as f:
f.write(testConfig)
logger.debug("Creating random test data...")
bfs = BlockwiseFileset(self.configpath, 'a')
dataShape = tuple(bfs.description.shape)
self.data = numpy.random.randint(255,
size=dataShape).astype(numpy.uint8)
logger.debug("Writing test data...")
datasetRoi = ([0, 0, 0, 0, 0], dataShape)
bfs.writeData(datasetRoi, self.data)
block_starts = getIntersectingBlocks(bfs.description.block_shape,
datasetRoi)
for block_start in block_starts:
bfs.setBlockStatus(block_start, BlockwiseFileset.BLOCK_AVAILABLE)
bfs.close()
def testBasic(self):
op = OpArrayPiper(graph=Graph())
inputData = numpy.indices((100, 100)).sum(0)
op.Input.setValue(inputData)
roiList = []
block_starts = getIntersectingBlocks([10, 10], ([0, 0], [100, 100]))
for block_start in block_starts:
roiList.append(getBlockBounds([100, 100], [10, 10], block_start))
results = numpy.zeros((100, 100), dtype=numpy.int32)
resultslock = threading.Lock()
resultsCount = [0]
def handleResult(roi, result):
acquired = resultslock.acquire(False)
assert acquired, "resultslock is contested! Access to callback is supposed to be automatically serialized."
results[roiToSlice(*roi)] = result
logger.debug("Got result for {}".format(roi))
resultslock.release()
resultsCount[0] += 1
progressList = []
def handleProgress(progress):
progressList.append(progress)
logger.debug("Progress update: {}".format(progress))
totalVolume = numpy.prod(inputData.shape)
batch = RoiRequestBatch(op.Output,
roiList.__iter__(),
totalVolume,
batchSize=10,
allowParallelResults=False)
batch.resultSignal.subscribe(handleResult)
batch.progressSignal.subscribe(handleProgress)
batch.execute()
logger.debug("Got {} results".format(resultsCount[0]))
assert (results == inputData).all()
# Progress reporting MUST start with 0 and end with 100
assert progressList[0] == 0, "Invalid progress reporting."
assert progressList[-1] == 100, "Invalid progress reporting."
# There should be some intermediate progress reporting, but exactly how much is unspecified.
assert len(progressList) >= 10
logger.debug("FINISHED")
def __init__(self, outputSlot, roi, minBlockShape, batchSize=None):
"""
Constructor.
:param outputSlot: The slot to request data from.
:param roi: The roi `(start, stop)` of interest. Will be broken up and requested via smaller requests.
:param minBlockShape: The minimum amount of data to request in each request.
Note: The current implementation breaks the big request into smaller
requests of exactly ``minBlockShape`` size. Future implementations could
concatenate smaller requests if it appears the system is not being overloaded by the smaller requests.
:param batchSize: The maximum number of requests to launch in parallel.
"""
self._outputSlot = outputSlot
self._bigRoi = roi
self._minBlockShape = minBlockShape
if batchSize is None:
batchSize = 2
# Align the blocking with the start of the roi
offsetRoi = ([0] * len(roi[0]), numpy.subtract(roi[1], roi[0]))
self._minBlockStarts = getIntersectingBlocks(minBlockShape, offsetRoi)
self._minBlockStarts += roi[0] # Un-offset
totalVolume = numpy.prod(numpy.subtract(roi[1], roi[0]))
# For now, simply iterate over the min blocks
# TODO: Auto-dialate block sizes based on CPU/RAM usage.
def roiGen():
block_iter = self._minBlockStarts.__iter__()
while True:
block_start = block_iter.next()
# Use offset blocking
offset_block_start = block_start - self._bigRoi[0]
offset_data_shape = numpy.subtract(self._bigRoi[1],
self._bigRoi[0])
offset_block_bounds = getBlockBounds(offset_data_shape,
minBlockShape,
offset_block_start)
# Un-offset
block_bounds = (offset_block_bounds[0] + self._bigRoi[0],
offset_block_bounds[1] + self._bigRoi[0])
logger.debug("Requesting Roi: {}".format(block_bounds))
yield block_bounds
self._requestBatch = RoiRequestBatch(self._outputSlot, roiGen(),
totalVolume, batchSize)
def downloadAllBlocks(self, max_parallel, skip_preparation=False):
"""
Download all blocks in the local view.
This is used in utility scripts for downloading an entire volume at once.
This function is NOT intended to be used by multiple threads in parallel
(i.e. it doesn't protect against downloading the same block twice.)
"""
view_shape = self.localDescription.view_shape
view_roi = ([0] * len(view_shape), view_shape)
block_starts = list(
getIntersectingBlocks(self.localDescription.block_shape, view_roi))
if not skip_preparation:
self._ensureDirectoriesExist(block_starts)
# Only wait for those that are missing.
blockQueue = Queue.Queue()
for block_start in block_starts:
if self.getBlockStatus(
block_start) == BlockwiseFileset.BLOCK_NOT_AVAILABLE:
blockQueue.put(block_start)
num_blocks = blockQueue.qsize()
logger.debug("Preparing to download {} blocks".format(num_blocks))
failedBlockQueue = Queue.Queue()
threads = []
for _ in range(max_parallel):
th = threading.Thread(
target=functools.partial(self._downloadFromQueue, num_blocks,
blockQueue, failedBlockQueue))
threads.append(th)
th.start()
for th in threads:
th.join()
errors = not failedBlockQueue.empty()
while not failedBlockQueue.empty():
logger.error(
"Failed to download block {}. Does it have a leftover lockfile?"
.format(failedBlockQueue.get()))
logger.debug("FINISHED DOWNLOADING.")
if errors:
logger.error(
"There were errors during the download process. Check error log output!"
)
def test_1_BasicWrite(self):
"""
This 'test' writes all blocks of the dataset.
SIDE EFFECT: This test is run first because the other tests use the dataset that it produces.
"""
logger.debug("Writing test data...")
totalRoi = ([0, 0, 0, 0, 0], self.dataShape)
self.bfs.writeData(totalRoi, self.data)
# All blocks are now available.
allBlockStarts = getIntersectingBlocks(
self.bfs.description.block_shape, totalRoi)
for blockStart in allBlockStarts:
self.bfs.setBlockStatus(blockStart,
BlockwiseFileset.BLOCK_AVAILABLE)
def readData(self, roi, out_array=None):
"""
Read data from the fileset. If any of the requested data is not yet available locally, download it first.
:param roi: The region of interest to read from the dataset. Must be a tuple of iterables: (start, stop).
:param out_array: The location to store the read data. Must be the correct size for the given roi. If not provided, an array is created for you.
:returns: The requested data. If out_array was provided, returns out_array.
"""
assert (numpy.array(roi[1]) <= numpy.array(self.localDescription.view_shape)).all(), "Requested roi '{}' is out of dataset bounds '{}'".format(roi, self.localDescription.view_shape)
# Before reading the data, make sure all the blocks we'll need to access are available on disk.
block_starts = getIntersectingBlocks(self.localDescription.block_shape, roi)
self._waitForBlocks( block_starts )
return super( RESTfulBlockwiseFileset, self ).readData( roi, out_array )
def _executeOutput(self, roi, destination):
assert len(roi.stop) == len(
self.Input.meta.shape
), "roi: {} has the wrong number of dimensions for Input shape: {}" "".format(roi, self.Input.meta.shape)
assert numpy.less_equal(
roi.stop, self.Input.meta.shape
).all(), "roi: {} is out-of-bounds for Input shape: {}" "".format(roi, self.Input.meta.shape)
block_starts = getIntersectingBlocks(self._blockshape, (roi.start, roi.stop))
block_starts = list(map(tuple, block_starts))
# Ensure all block cache files are up-to-date
self._waitForBlocks(block_starts)
self._copyData(roi, destination, block_starts)
return destination
def setUp(self):
"""
Create a blockwise fileset to test with.
"""
if platform.system() == 'Windows':
# On windows, there are errors, and we make no attempt to solve them (at the moment).
raise nose.SkipTest
try:
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
raise nose.SkipTest
testConfig = \
"""
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "synapse_small",
"format" : "hdf5",
"axes" : "txyzc",
"shape" : [1,400,400,100,1],
"dtype" : "numpy.uint8",
"block_shape" : [1, 50, 50, 50, 100],
"block_file_name_format" : "cube{roiString}.h5/volume/data"
}
"""
self.tempDir = tempfile.mkdtemp()
self.configpath = os.path.join(self.tempDir, "config.json")
logger.debug( "Loading config file..." )
with open(self.configpath, 'w') as f:
f.write(testConfig)
logger.debug( "Creating random test data..." )
bfs = BlockwiseFileset( self.configpath, 'a' )
dataShape = tuple(bfs.description.shape)
self.data = numpy.random.randint( 255, size=dataShape ).astype(numpy.uint8)
logger.debug( "Writing test data..." )
datasetRoi = ([0,0,0,0,0], dataShape)
bfs.writeData( datasetRoi, self.data )
block_starts = getIntersectingBlocks(bfs.description.block_shape, datasetRoi)
for block_start in block_starts:
bfs.setBlockStatus(block_start, BlockwiseFileset.BLOCK_AVAILABLE)
bfs.close()
def _executeOutput(self, roi, destination):
assert len(roi.stop) == len(self.Input.meta.shape), \
"roi: {} has the wrong number of dimensions for Input shape: {}"\
"".format( roi, self.Input.meta.shape )
assert numpy.less_equal(roi.stop, self.Input.meta.shape).all(), \
"roi: {} is out-of-bounds for Input shape: {}"\
"".format( roi, self.Input.meta.shape )
block_starts = getIntersectingBlocks(self._blockshape,
(roi.start, roi.stop))
block_starts = list(map(tuple, block_starts))
# Ensure all block cache files are up-to-date
self._waitForBlocks(block_starts)
self._copyData(roi, destination, block_starts)
return destination
def _generate_connector_buckets(cls, connectors, blockshape_xyz ):
"""
Store the list of ConnectorInfos into buckets (a dictionary of lists), grouped by block location.
The dict keys are simply the start coordinate of each block (as a tuple).
The block boundaries are determined by blockshape_zyx.
"""
blocks = {}
for conn in connectors:
coord = numpy.array( (conn.x_nm, conn.y_nm, conn.z_nm) ).astype(int)
block_start = getIntersectingBlocks( blockshape_xyz, (coord, coord+1) )[0]
block_start = tuple(block_start)
try:
blocks[block_start].append( conn )
except KeyError:
blocks[block_start] = [conn]
return blocks
def propagateDirty(self, slot, subindex, roi):
if slot == self.Input:
# Keep track of dirty blocks
with self._lock:
block_starts = getIntersectingBlocks( self._blockshape, (roi.start, roi.stop) )
block_starts = map( tuple, block_starts )
for block_start in block_starts:
self._dirtyBlocks.add( block_start )
# Forward to downstream connections
self.Output.setDirty( roi )
elif slot == self.BlockShape:
# Everything is dirty
self.Output.setDirty( slice(None) )
else:
assert False, "Unknown output slot"
def _executeBlockwiseRegionFeatures(self, roi, destination):
"""
Provide data for the BlockwiseRegionFeatures slot.
Note: Each block produces a single element of this slot's output. Construct requested roi coordinates accordingly.
e.g. if block_shape is (1,10,10,10,1), the features for the block starting at
(1,20,30,40,5) should be requested via roi [(1,2,3,4,5),(2,3,4,5,6)]
Note: It is assumed that you will request these features for debug purposes, AFTER requesting the prediction image.
Therefore, it is considered an error to request features that are not already computed.
"""
axiskeys = self.RawImage.meta.getAxisKeys()
# Find the corresponding block start coordinates
block_shape = self._getFullShape(self.BlockShape3dDict.value)
pixel_roi = numpy.array(block_shape) * (roi.start, roi.stop)
block_starts = getIntersectingBlocks(block_shape, pixel_roi)
block_starts = map(tuple, block_starts)
# TODO: Parallelize this?
for block_start in block_starts:
assert block_start in self._blockPipelines, "Not allowed to request region features for blocks that haven't yet been processed." # See note above
# Discard spatial axes to get (t,c) index for region slot roi
tagged_block_start = zip(axiskeys, block_start)
tagged_block_start_tc = filter(lambda (k, v): k in 'tc',
tagged_block_start)
block_start_tc = map(lambda (k, v): v, tagged_block_start_tc)
block_roi_tc = (block_start_tc,
block_start_tc + numpy.array([1, 1]))
block_roi_t = (block_roi_tc[0][:-1], block_roi_tc[1][:-1])
assert sys.version_info.major == 2, "Alert! This loop has not been tested "\
"under python 3. Please remove this assetion and be wary of any strnage behavior you encounter"
destination_start = numpy.array(
block_start) // block_shape - roi.start
destination_stop = destination_start + numpy.array(
[1] * len(axiskeys))
opBlockPipeline = self._blockPipelines[block_start]
req = opBlockPipeline.BlockwiseRegionFeatures(*block_roi_t)
destination_without_channel = destination[roiToSlice(
destination_start, destination_stop)]
destination_with_channel = destination_without_channel[
..., block_roi_tc[0][-1]:block_roi_tc[1][-1]]
req.writeInto(destination_with_channel)
req.wait()
return destination
def propagateDirty(self, slot, subindex, roi):
if slot == self.Input:
# Keep track of dirty blocks
if self._blockshape is not None:
with self._lock:
block_starts = getIntersectingBlocks( self._blockshape, (roi.start, roi.stop) )
block_starts = map( tuple, block_starts )
for block_start in block_starts:
self._dirtyBlocks.add( block_start )
# Forward to downstream connections
self.Output.setDirty( roi )
elif slot == self.BlockShape:
# Everything is dirty
self.Output.setDirty( slice(None) )
else:
assert False, "Unknown output slot"
def downloadAllBlocks(self, max_parallel, skip_preparation=False):
"""
Download all blocks in the local view.
This is used in utility scripts for downloading an entire volume at once.
This function is NOT intended to be used by multiple threads in parallel
(i.e. it doesn't protect against downloading the same block twice.)
"""
view_shape = self.localDescription.view_shape
view_roi = ([0] * len(view_shape), view_shape)
block_starts = list(getIntersectingBlocks(self.localDescription.block_shape, view_roi))
if not skip_preparation:
self._ensureDirectoriesExist(block_starts)
# Only wait for those that are missing.
blockQueue = queue.Queue()
for block_start in block_starts:
if self.getBlockStatus(block_start) == BlockwiseFileset.BLOCK_NOT_AVAILABLE:
blockQueue.put(block_start)
num_blocks = blockQueue.qsize()
logger.debug("Preparing to download {} blocks".format(num_blocks))
failedBlockQueue = queue.Queue()
threads = []
for _ in range(max_parallel):
th = threading.Thread(
target=functools.partial(self._downloadFromQueue, num_blocks, blockQueue, failedBlockQueue)
)
threads.append(th)
th.start()
for th in threads:
th.join()
errors = not failedBlockQueue.empty()
while not failedBlockQueue.empty():
logger.error(
"Failed to download block {}. Does it have a leftover lockfile?".format(failedBlockQueue.get())
)
logger.debug("FINISHED DOWNLOADING.")
if errors:
logger.error("There were errors during the download process. Check error log output!")
def __init__(self, outputSlot, roi, minBlockShape, batchSize=None):
"""
Constructor.
:param outputSlot: The slot to request data from.
:param roi: The roi `(start, stop)` of interest. Will be broken up and requested via smaller requests.
:param minBlockShape: The minimum amount of data to request in each request.
Note: The current implementation breaks the big request into smaller
requests of exactly ``minBlockShape`` size. Future implementations could
concatenate smaller requests if it appears the system is not being overloaded by the smaller requests.
:param batchSize: The maximum number of requests to launch in parallel.
"""
self._outputSlot = outputSlot
self._bigRoi = roi
self._minBlockShape = minBlockShape
if batchSize is None:
batchSize=2
# Align the blocking with the start of the roi
offsetRoi = ([0] * len(roi[0]), numpy.subtract(roi[1], roi[0]))
self._minBlockStarts = getIntersectingBlocks(minBlockShape, offsetRoi)
self._minBlockStarts += roi[0] # Un-offset
totalVolume = numpy.prod( numpy.subtract(roi[1], roi[0]) )
# For now, simply iterate over the min blocks
# TODO: Auto-dialate block sizes based on CPU/RAM usage.
def roiGen():
block_iter = self._minBlockStarts.__iter__()
while True:
block_start = block_iter.next()
# Use offset blocking
offset_block_start = block_start - self._bigRoi[0]
offset_data_shape = numpy.subtract(self._bigRoi[1], self._bigRoi[0])
offset_block_bounds = getBlockBounds( offset_data_shape, minBlockShape, offset_block_start )
# Un-offset
block_bounds = ( offset_block_bounds[0] + self._bigRoi[0],
offset_block_bounds[1] + self._bigRoi[0] )
logger.debug( "Requesting Roi: {}".format( block_bounds ) )
yield block_bounds
self._requestBatch = RoiRequestBatch( self._outputSlot, roiGen(), totalVolume, batchSize )
def autoSeedBackground(cls, laneView, foreground_label):
# Seed the entire image with background labels, except for the individual label in question
# To save memory, we'll do this in blocks instead of all at once
volume_shape = laneView.RavelerLabels.meta.shape
volume_roi = roiFromShape( volume_shape )
block_shape = (OpSplitBodyCarving.BLOCK_SIZE,) * len( volume_shape )
block_shape = numpy.minimum( block_shape, volume_shape )
block_starts = getIntersectingBlocks( block_shape, volume_roi )
logger.debug("Auto-seeding {} blocks for label".format( len(block_starts), foreground_label ))
for block_index, block_start in enumerate(block_starts):
block_roi = getBlockBounds( volume_shape, block_shape, block_start )
label_block = laneView.RavelerLabels(*block_roi).wait()
background_block = numpy.where( label_block == foreground_label, 0, 1 )
background_block = numpy.asarray( background_block, numpy.float32 ) # Distance transform requires float
if (background_block == 0.0).any():
# We need to leave a small border between the background seeds and the object membranes
background_block_view = background_block.view( vigra.VigraArray )
background_block_view.axistags = copy.copy( laneView.RavelerLabels.meta.axistags )
background_block_view_4d = background_block_view.bindAxis('t', 0)
background_block_view_3d = background_block_view_4d.bindAxis('c', 0)
distance_transformed_block = vigra.filters.distanceTransform3D(background_block_view_3d, background=False)
distance_transformed_block = distance_transformed_block.astype( numpy.uint8 )
# Create a 'hull' surrounding the foreground, but leave some space.
background_seed_block = (distance_transformed_block == OpSplitBodyCarving.SEED_MARGIN)
background_seed_block = background_seed_block.astype(numpy.uint8) * 1 # (In carving, background is label 1)
# # Make the hull VERY sparse to avoid over-biasing graph cut toward the background class
# # FIXME: Don't regenerate this random block on every loop iteration
# rand_bytes = numpy.random.randint(0, 1000, background_seed_block.shape)
# background_seed_block = numpy.where( rand_bytes < 1, background_seed_block, 0 )
# background_seed_block = background_seed_block.view(vigra.VigraArray)
# background_seed_block.axistags = background_block_view_3d.axistags
axisorder = laneView.RavelerLabels.meta.getTaggedShape().keys()
logger.debug("Writing backgound seeds: {}/{}".format( block_index, len(block_starts) ))
laneView.WriteSeeds[ roiToSlice( *block_roi ) ] = background_seed_block.withAxes(*axisorder)
else:
logger.debug("Skipping all-background block: {}/{}".format( block_index, len(block_starts) ))
def export_to_tiles(volume, tile_size, output_dir, print_progress=True):
"""
volume: The volume to export (either hdf5 dataset or numpy array). Must be 3D.
tile_size: The width of the tiles to generate
output_dir: The directory to dump the tiles to.
"""
assert len(volume.shape) == 3
tile_blockshape = (1, tile_size, tile_size)
tile_starts = getIntersectingBlocks(tile_blockshape,
roiFromShape(volume.shape))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
logger.info("Writing {} tiles ...".format(len(tile_starts)))
for tile_start in tile_starts:
tile_roi = getBlockBounds(volume.shape, tile_blockshape, tile_start)
if print_progress:
sys.stdout.write("Tile: {} ".format(tile_roi))
sys.stdout.flush()
tile_data = volume[roiToSlice(*tile_roi)]
tile_data = vigra.taggedView(tile_data, 'zyx')
if print_progress:
sys.stdout.write('reading... ')
sys.stdout.flush()
tile_name = 'tile_z{:05}_y{:05}_x{:05}.png'.format(*tile_start)
output_path = os.path.join(output_dir, tile_name)
if print_progress:
sys.stdout.write('writing... ')
sys.stdout.flush()
vigra.impex.writeImage(tile_data[0], output_path, dtype='NATIVE')
if print_progress:
sys.stdout.write('done.\n')
sys.stdout.flush()
logger.info("TILES COMPLETE.")
def testBasic(self):
op = OpArrayPiper( graph=Graph() )
inputData = numpy.indices( (100,100) ).sum(0)
op.Input.setValue( inputData )
roiList = []
block_starts = getIntersectingBlocks( [10,10], ([0,0], [100, 100]) )
for block_start in block_starts:
roiList.append( getBlockBounds( [100,100], [10,10], block_start ) )
results = numpy.zeros( (100,100), dtype=numpy.int32 )
resultslock = threading.Lock()
resultsCount = [0]
def handleResult(roi, result):
acquired = resultslock.acquire(False)
assert acquired, "resultslock is contested! Access to callback is supposed to be automatically serialized."
results[ roiToSlice( *roi ) ] = result
logger.debug( "Got result for {}".format(roi) )
resultslock.release()
resultsCount[0] += 1
progressList = []
def handleProgress( progress ):
progressList.append( progress )
logger.debug( "Progress update: {}".format(progress) )
totalVolume = numpy.prod( inputData.shape )
batch = RoiRequestBatch(op.Output, roiList.__iter__(), totalVolume, batchSize=10)
batch.resultSignal.subscribe( handleResult )
batch.progressSignal.subscribe( handleProgress )
batch.execute()
logger.debug( "Got {} results".format( resultsCount[0] ) )
assert (results == inputData).all()
# Progress reporting MUST start with 0 and end with 100
assert progressList[0] == 0, "Invalid progress reporting."
assert progressList[-1] == 100, "Invalid progress reporting."
# There should be some intermediate progress reporting, but exactly how much is unspecified.
assert len(progressList) >= 10
logger.debug( "FINISHED" )
def export_to_tiles(volume, tile_size, output_dir, print_progress=True):
"""
volume: The volume to export (either hdf5 dataset or numpy array). Must be 3D.
tile_size: The width of the tiles to generate
output_dir: The directory to dump the tiles to.
"""
assert len(volume.shape) == 3
tile_blockshape = (1, tile_size, tile_size)
tile_starts = getIntersectingBlocks( tile_blockshape, roiFromShape( volume.shape ) )
if not os.path.exists(output_dir):
os.makedirs(output_dir)
logger.info("Writing {} tiles ...".format( len(tile_starts) ) )
for tile_start in tile_starts:
tile_roi = getBlockBounds( volume.shape, tile_blockshape, tile_start )
if print_progress:
sys.stdout.write("Tile: {} ".format( tile_roi ))
sys.stdout.flush()
tile_data = volume[ roiToSlice(*tile_roi) ]
tile_data = vigra.taggedView(tile_data, 'zyx')
if print_progress:
sys.stdout.write('reading... ')
sys.stdout.flush()
tile_name = 'tile_z{:05}_y{:05}_x{:05}.png'.format( *tile_start )
output_path = os.path.join( output_dir, tile_name )
if print_progress:
sys.stdout.write('writing... ')
sys.stdout.flush()
vigra.impex.writeImage( tile_data[0], output_path, dtype='NATIVE' )
if print_progress:
sys.stdout.write('done.\n')
sys.stdout.flush()
logger.info( "TILES COMPLETE." )
def readData(self, roi, out_array=None):
"""
Read data from the fileset. If any of the requested data is not yet available locally, download it first.
:param roi: The region of interest to read from the dataset. Must be a tuple of iterables: (start, stop).
:param out_array: The location to store the read data. Must be the correct size for the given roi. If not provided, an array is created for you.
:returns: The requested data. If out_array was provided, returns out_array.
"""
assert (numpy.array(roi[1]) <= numpy.array(
self.localDescription.view_shape)).all(
), "Requested roi '{}' is out of dataset bounds '{}'".format(
roi, self.localDescription.view_shape)
# Before reading the data, make sure all the blocks we'll need to access are available on disk.
block_starts = getIntersectingBlocks(self.localDescription.block_shape,
roi)
self._waitForBlocks(block_starts)
return super(RESTfulBlockwiseFileset, self).readData(roi, out_array)
def _setInSlotInput(self, slot, subindex, roi, value, store_zero_blocks=True):
"""
Write the data in the array 'value' into the cache.
If the optional store_zero_blocks param is False, then don't bother
creating cache blocks for blocks that are totally zero.
"""
assert len(roi.stop) == len(self.Input.meta.shape), \
"roi: {} has the wrong number of dimensions for Input shape: {}"\
"".format( roi, self.Input.meta.shape )
assert numpy.less_equal(roi.stop, self.Input.meta.shape).all(), \
"roi: {} is out-of-bounds for Input shape: {}"\
"".format( roi, self.Input.meta.shape )
block_starts = getIntersectingBlocks( self._blockshape, (roi.start, roi.stop) )
block_starts = map( tuple, block_starts )
# Copy data to each block
logger.debug( "Copying data INTO {} blocks...".format( len(block_starts) ) )
for block_start in block_starts:
entire_block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, block_start )
# This block's portion of the roi
intersecting_roi = getIntersection( (roi.start, roi.stop), entire_block_roi )
# Compute slicing within source array and slicing within this block
source_relative_intersection = numpy.subtract(intersecting_roi, roi.start)
block_relative_intersection = numpy.subtract(intersecting_roi, block_start)
new_block_data = value[ roiToSlice(*source_relative_intersection) ]
if not store_zero_blocks and new_block_data.sum() == 0 and block_start not in self._cacheFiles:
# Special fast-path: If this block doesn't exist yet,
# don't bother creating if we're just going to fill it with zeros
# (Used by the OpCompressedUserLabelArray)
pass
else:
# Copy from source to block
dataset = self._getBlockDataset( entire_block_roi )
dataset[ roiToSlice( *block_relative_intersection ) ] = new_block_data
# Here, we assume that if this function is used to update ANY PART of a
# block, he is responsible for updating the ENTIRE block.
# Therefore, this block is no longer 'dirty'
self._dirtyBlocks.discard( block_start )
def _executePredictionImage(self, slot, roi, destination):
roi_one_channel = numpy.array((roi.start, roi.stop))
roi_one_channel[..., -1] = (0, 1)
# Determine intersecting blocks
block_shape = self._getFullShape(self.BlockShape3dDict.value)
block_starts = getIntersectingBlocks(block_shape, roi_one_channel)
block_starts = map(tuple, block_starts)
# Ensure that block pipelines exist (create first if necessary)
for block_start in block_starts:
self._ensurePipelineExists(block_start)
# Retrieve result from each block, and write into the appropriate region of the destination
pool = RequestPool()
for block_start in block_starts:
opBlockPipeline = self._blockPipelines[block_start]
block_roi = opBlockPipeline.block_roi
block_intersection = getIntersection(block_roi, roi_one_channel)
block_relative_intersection = numpy.subtract(
block_intersection, block_roi[0])
destination_relative_intersection = numpy.subtract(
block_intersection, roi_one_channel[0])
block_slot = opBlockPipeline.PredictionImage
if slot == self.ProbabilityChannelImage:
block_slot = opBlockPipeline.ProbabilityChannelImage
# Add channels back to roi
# request all channels
block_relative_intersection[..., -1] = (
0, opBlockPipeline.ProbabilityChannelImage.meta.shape[-1])
# But only write the ones that were specified in the original roi
destination_relative_intersection[..., -1] = (roi.start[-1],
roi.stop[-1])
# Request the data
destination_slice = roiToSlice(*destination_relative_intersection)
req = block_slot(*block_relative_intersection)
req.writeInto(destination[destination_slice])
pool.add(req)
pool.wait()
return destination
def purgeAllLocks(self):
"""
Clears all .lock files from the local blockwise fileset.
This may be necessary if previous processes crashed or were killed while some blocks were downloading.
You must ensure that this is NOT called while more than one process (or thread) has access to the fileset.
For example, in a master/worker situation, call this only from the master, before the workers have been started.
"""
found_lock = False
view_shape = self.description.view_shape
view_roi = ([0] * len(view_shape), view_shape)
block_starts = list(getIntersectingBlocks(self.description.block_shape, view_roi))
for block_start in block_starts:
blockFilePathComponents = self.getDatasetPathComponents(block_start)
fileLock = FileLock(blockFilePathComponents.externalPath)
found_lock |= fileLock.purge()
if found_lock:
logger.warning("Purged lock for block: {}".format(tuple(block_start)))
return found_lock
def _setInSlotInput(self,
slot,
subindex,
roi,
value,
store_zero_blocks=True):
"""
Write the data in the array 'value' into the cache.
If the optional store_zero_blocks param is False, then don't bother
creating cache blocks for blocks that are totally zero.
"""
assert len(roi.stop) == len(
self.Input.meta.shape
), "roi: {} has the wrong number of dimensions for Input shape: {}".format(
roi, self.Input.meta.shape)
assert numpy.less_equal(roi.stop, self.Input.meta.shape).all(
), "roi: {} is out-of-bounds for Input shape: {}".format(
roi, self.Input.meta.shape)
block_starts = getIntersectingBlocks(self._blockshape,
(roi.start, roi.stop))
block_starts = list(map(tuple, block_starts))
# Copy data to each block
logger.debug("Copying data INTO {} blocks...".format(
len(block_starts)))
for block_start in block_starts:
entire_block_roi = getBlockBounds(self.Output.meta.shape,
self._blockshape, block_start)
# This block's portion of the roi
intersecting_roi = getIntersection((roi.start, roi.stop),
entire_block_roi)
# Compute slicing within source array and slicing within this block
source_relative_intersection = numpy.subtract(
intersecting_roi, roi.start)
block_relative_intersection = numpy.subtract(
intersecting_roi, block_start)
source_relative_intersection_slicing = roiToSlice(
*source_relative_intersection)
block_relative_intersection_slicing = roiToSlice(
*block_relative_intersection)
new_block_data = value[source_relative_intersection_slicing]
new_block_sum = new_block_data.sum()
if not store_zero_blocks and new_block_sum == 0 and block_start not in self._cacheFiles:
# Special fast-path: If this block doesn't exist yet,
# don't bother creating if we're just going to fill it with zeros.
# (This feature is used by the OpCompressedUserLabelArray)
pass
else:
# Copy from source to block
dataset = self._getBlockDataset(entire_block_roi)
if self.Output.meta.has_mask:
dataset["data"][
block_relative_intersection_slicing] = new_block_data.data
dataset["mask"][
block_relative_intersection_slicing] = new_block_data.mask
dataset["fill_value"][()] = new_block_data.fill_value
# Untested. Write a test to use this.
# # If we can, remove this block entirely.
# if not store_zero_blocks and new_block_sum == 0 and (dataset["data"][:] == 0).all() and (dataset["mask"]).any() and (dataset["fill_value"] == 0).all():
# with self._lock:
# with self._blockLocks[block_start]:
# self._cacheFiles[block_start].close()
# del self._cacheFiles[block_start]
# del self._blockLocks[block_start]
else:
dataset[
block_relative_intersection_slicing] = new_block_data
# If we can, remove this block entirely.
if not store_zero_blocks and new_block_sum == 0 and (
dataset[:] == 0).all():
with self._lock:
with self._blockLocks[block_start]:
self._cacheFiles[block_start].close()
del self._cacheFiles[block_start]
del self._blockLocks[block_start]
# Here, we assume that if this function is used to update ANY PART of a
# block, he is responsible for updating the ENTIRE block.
# Therefore, this block is no longer 'dirty'
self._dirtyBlocks.discard(block_start)
def exportSubset(self, roi, exportDirectory, use_view_coordinates=True):
"""
Create a new blockwise fileset by copying a subset of this blockwise fileset.
:param roi: The portion to export. Must be along block boundaries, in ABSOLUTE coordinates.
:param exportDirectory: The directory to copy the new blockwise fileset to.
"""
# For now, this implementation assumes it can simply copy EVERYTHING in the block directories,
# including lock files. Therefore, we require that the fileset be opened in read-only mode.
# If that's a problem, change this function to ignore lock files when copying (or purge them afterwards).
roi = list(map(TinyVector, roi))
if not use_view_coordinates:
abs_roi = roi
assert (
abs_roi[0] >= self.description.view_origin
), "Roi {} is out-of-bounds: must not span lower than the view origin: ".format(
roi, self.description.origin
)
else:
abs_roi = roi + self.description.view_origin
assert self.mode == "r", "Can't export from a fileset that is open in read/write mode."
block_shape = self._description.block_shape
abs_shape = self._description.shape
view_origin = self._description.view_origin
assert (abs_roi[0] % block_shape == 0).all(), "exportSubset() requires roi to start on a block boundary"
assert (
(abs_roi[1] % block_shape == 0) | (abs_roi[1] == abs_shape)
).all(), "exported subset must end on block or dataset boundary."
if not os.path.exists(exportDirectory):
os.makedirs(exportDirectory)
source_desc_path = self._descriptionFilePath
source_desc_dir, source_desc_filename = os.path.split(source_desc_path)
source_root_dir = self.description.dataset_root_dir
# Copy/update description file
dest_desc_path = os.path.join(exportDirectory, source_desc_filename)
if os.path.exists(dest_desc_path):
dest_description = BlockwiseFileset.readDescription(dest_desc_path)
else:
dest_description = copy.copy(self._description)
dest_description.view_shape = abs_roi[1] - view_origin
dest_description.hash_id = None
BlockwiseFileset.writeDescription(dest_desc_path, dest_description)
# Determine destination root block dir
if os.path.isabs(source_root_dir):
source_root_dir = os.path.normpath(source_root_dir)
source_root_dir_name = os.path.split(source_root_dir)[1]
dest_root_dir = os.path.join(exportDirectory, source_root_dir_name)
else:
dest_root_dir = os.path.join(exportDirectory, source_root_dir)
source_root_dir, _ = getPathVariants(source_root_dir, source_desc_dir)
view_roi = abs_roi - view_origin
block_starts = getIntersectingBlocks(block_shape, view_roi)
for block_start in block_starts:
source_block_dir = self.getDatasetDirectory(block_start)
rel_block_dir = os.path.relpath(source_block_dir, source_root_dir)
dest_block_dir = os.path.join(dest_root_dir, rel_block_dir)
if os.path.exists(dest_block_dir):
logger.info("Skipping existing block directory: {}".format(dest_block_dir))
elif not os.path.exists(source_block_dir):
logger.info("Skipping missing block directory: {}".format(source_block_dir))
else:
# Copy the entire block directory
assert dest_block_dir[-1] != "/"
dest_block_dir_parent = os.path.split(dest_block_dir)[0]
if not os.path.exists(dest_block_dir_parent):
os.makedirs(dest_block_dir_parent)
shutil.copytree(source_block_dir, dest_block_dir)
return dest_desc_path
def __init__(self,
outputSlot,
roi,
blockshape=None,
batchSize=None,
blockAlignment='absolute',
allowParallelResults=False):
"""
Constructor.
:param outputSlot: The slot to request data from.
:param roi: The roi `(start, stop)` of interest. Will be broken up and requested via smaller requests.
:param blockshape: The amount of data to request in each request. If omitted, a default blockshape is chosen by inspecting the metadata of the given slot.
:param batchSize: The maximum number of requests to launch in parallel. This should not be necessary if the blockshape is small enough that you won't run out of RAM.
:param blockAlignment: Determines how block the requests. Choices are 'absolute' or 'relative'.
:param allowParallelResults: If False, The resultSignal will not be called in parallel.
In that case, your handler function has no need for locks.
"""
self._outputSlot = outputSlot
self._bigRoi = roi
totalVolume = numpy.prod(numpy.subtract(roi[1], roi[0]))
if batchSize is None:
batchSize = 1000
if blockshape is None:
blockshape = self._determine_blockshape(outputSlot)
assert blockAlignment in ['relative', 'absolute']
if blockAlignment == 'relative':
# Align the blocking with the start of the roi
offsetRoi = ([0] * len(roi[0]), numpy.subtract(roi[1], roi[0]))
block_starts = getIntersectingBlocks(blockshape, offsetRoi)
block_starts += roi[0] # Un-offset
# For now, simply iterate over the min blocks
# TODO: Auto-dialate block sizes based on CPU/RAM usage.
def roiGen():
block_iter = block_starts.__iter__()
while True:
block_start = block_iter.next()
# Use offset blocking
offset_block_start = block_start - self._bigRoi[0]
offset_data_shape = numpy.subtract(self._bigRoi[1],
self._bigRoi[0])
offset_block_bounds = getBlockBounds(
offset_data_shape, blockshape, offset_block_start)
# Un-offset
block_bounds = (offset_block_bounds[0] + self._bigRoi[0],
offset_block_bounds[1] + self._bigRoi[0])
logger.debug("Requesting Roi: {}".format(block_bounds))
yield block_bounds
else:
# Absolute blocking.
# Blocks are simply relative to (0,0,0,...)
# But we still clip the requests to the overall roi bounds.
block_starts = getIntersectingBlocks(blockshape, roi)
def roiGen():
block_iter = block_starts.__iter__()
while True:
block_start = block_iter.next()
block_bounds = getBlockBounds(outputSlot.meta.shape,
blockshape, block_start)
block_intersecting_portion = getIntersection(
block_bounds, roi)
logger.debug("Requesting Roi: {}".format(block_bounds))
yield block_intersecting_portion
self._requestBatch = RoiRequestBatch(self._outputSlot, roiGen(),
totalVolume, batchSize,
allowParallelResults)
def setBlockStatusesForRoi(self, roi, status):
block_starts = getIntersectingBlocks(self._description.block_shape, roi)
for block_start in block_starts:
self.setBlockStatus(block_start, status)
def _executeProjection2D(self, roi, destination):
assert sum(TinyVector(destination.shape) > 1) <= 2, "Projection result must be exactly 2D"
# First, we have to determine which axis we are projecting along.
# We infer this from the shape of the roi.
# For example, if the roi is of shape
# zyx = (1,256,256), then we know we're projecting along Z
# If more than one axis has a width of 1, then we choose an
# axis according to the following priority order: zyxt
tagged_input_shape = self.Input.meta.getTaggedShape()
tagged_result_shape = collections.OrderedDict( zip( tagged_input_shape.keys(),
destination.shape ) )
nonprojection_axes = []
for key in tagged_input_shape.keys():
if (key == 'c' or tagged_input_shape[key] == 1 or tagged_result_shape[key] > 1):
nonprojection_axes.append( key )
possible_projection_axes = set(tagged_input_shape) - set(nonprojection_axes)
if len(possible_projection_axes) == 0:
# If the image is 2D to begin with,
# then the projection is simply the same as the normal output,
# EXCEPT it is made binary
self.Output(roi.start, roi.stop).writeInto(destination).wait()
# make binary
numpy.greater(destination, 0, out=destination)
return
for k in 'zyxt':
if k in possible_projection_axes:
projection_axis_key = k
break
# Now we know which axis we're projecting along.
# Proceed with the projection, working blockwise to avoid unecessary work in unlabeled blocks
projection_axis_index = self.Input.meta.getAxisKeys().index(projection_axis_key)
projection_length = tagged_input_shape[projection_axis_key]
input_roi = roi.copy()
input_roi.start[projection_axis_index] = 0
input_roi.stop[projection_axis_index] = projection_length
destination[:] = 0.0
# Get the logical blocking.
block_starts = getIntersectingBlocks( self._blockshape, (input_roi.start, input_roi.stop) )
# (Parallelism wouldn't help here: h5py will serialize these requests anyway)
block_starts = map( tuple, block_starts )
for block_start in block_starts:
if block_start not in self._cacheFiles:
# No label data in this block. Move on.
continue
entire_block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, block_start )
# This block's portion of the roi
intersecting_roi = getIntersection( (input_roi.start, input_roi.stop), entire_block_roi )
# Compute slicing within the deep array and slicing within this block
deep_relative_intersection = numpy.subtract(intersecting_roi, input_roi.start)
block_relative_intersection = numpy.subtract(intersecting_roi, block_start)
deep_data = self._getBlockDataset( entire_block_roi )[roiToSlice(*block_relative_intersection)]
# make binary and convert to float
deep_data_float = numpy.where( deep_data, numpy.float32(1.0), numpy.float32(0.0) )
# multiply by slice-index
deep_data_view = numpy.rollaxis(deep_data_float, projection_axis_index, 0)
min_deep_slice_index = deep_relative_intersection[0][projection_axis_index]
max_deep_slice_index = deep_relative_intersection[1][projection_axis_index]
def calc_color_value(slice_index):
# Note 1: We assume that the colortable has at least 256 entries in it,
# so, we try to ensure that all colors are above 1/256
# (we don't want colors in low slices to be rounded to 0)
# Note 2: Ideally, we'd use a min projection in the code below, so that
# labels in the "back" slices would appear occluded. But the
# min projection would favor 0.0. Instead, we invert the
# relationship between color and slice index, do a max projection,
# and then re-invert the colors after everything is done.
# Hence, this function starts with (1.0 - ...)
return (1.0 - (float(slice_index) / projection_length)) * (1.0 - 1.0/255) + 1.0/255.0
min_color_value = calc_color_value(min_deep_slice_index)
max_color_value = calc_color_value(max_deep_slice_index)
num_slices = max_deep_slice_index - min_deep_slice_index
deep_data_view *= numpy.linspace( min_color_value, max_color_value, num=num_slices )\
[ (slice(None),) + (None,)*(deep_data_view.ndim-1) ]
# Take the max projection of this block's data.
block_max_projection = numpy.amax(deep_data_float, axis=projection_axis_index, keepdims=True)
# Merge this block's projection into the overall projection.
destination_relative_intersection = numpy.array(deep_relative_intersection)
destination_relative_intersection[:, projection_axis_index] = (0,1)
destination_subview = destination[roiToSlice(*destination_relative_intersection)]
numpy.maximum(block_max_projection, destination_subview, out=destination_subview)
# Invert the nonzero pixels so increasing colors correspond to increasing slices.
# See comment in calc_color_value(), above.
destination_subview[:] = numpy.where(destination_subview,
numpy.float32(1.0) - destination_subview,
numpy.float32(0.0))
return
