def _executeOutput(self, roi, destination):
assert len(roi.stop) == len(self.Input.meta.shape), "roi: {} has the wrong number of dimensions for Input shape: {}".format( roi, self.Input.meta.shape )
assert numpy.less_equal(roi.stop, self.Input.meta.shape).all(), "roi: {} is out-of-bounds for Input shape: {}".format( roi, self.Input.meta.shape )
block_starts = getIntersectingBlocks( self._blockshape, (roi.start, roi.stop) )
block_starts = map( tuple, block_starts )
# Ensure all block cache files are up-to-date
reqPool = RequestPool() # (Do the work in parallel.)
for block_start in block_starts:
entire_block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, block_start )
f = partial( self._ensureCached, entire_block_roi)
reqPool.add( Request(f) )
logger.debug( "Waiting for {} blocks...".format( len(block_starts) ) )
reqPool.wait()
# Copy data from each block
# (Parallelism not needed here: h5py will serialize these requests anyway)
logger.debug( "Copying data from {} blocks...".format( len(block_starts) ) )
for block_start in block_starts:
entire_block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, block_start )
# This block's portion of the roi
intersecting_roi = getIntersection( (roi.start, roi.stop), entire_block_roi )
# Compute slicing within destination array and slicing within this block
destination_relative_intersection = numpy.subtract(intersecting_roi, roi.start)
block_relative_intersection = numpy.subtract(intersecting_roi, block_start)
# Copy from block to destination
dataset = self._getBlockDataset( entire_block_roi )
destination[ roiToSlice(*destination_relative_intersection) ] = dataset[ roiToSlice( *block_relative_intersection ) ]
return destination
def _copyData(self, roi, destination, block_starts):
# Copy data from each block
# (Parallelism not needed here: h5py will serialize these requests anyway)
logger.debug( "Copying data from {} blocks...".format( len(block_starts) ) )
for block_start in block_starts:
entire_block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, block_start )
# This block's portion of the roi
intersecting_roi = getIntersection( (roi.start, roi.stop), entire_block_roi )
# Compute slicing within destination array and slicing within this block
destination_relative_intersection = numpy.subtract(intersecting_roi, roi.start)
block_relative_intersection = numpy.subtract(intersecting_roi, block_start)
destination_relative_intersection_slicing = roiToSlice(*destination_relative_intersection)
block_relative_intersection_slicing = roiToSlice( *block_relative_intersection )
# Copy from block to destination
dataset = self._getBlockDataset( entire_block_roi )
if self.Output.meta.has_mask:
destination.data[ destination_relative_intersection_slicing ] = dataset["data"][ block_relative_intersection_slicing ]
destination.mask[ destination_relative_intersection_slicing ] = dataset["mask"][ block_relative_intersection_slicing ]
destination.fill_value = dataset["fill_value"][()]
else:
destination[ destination_relative_intersection_slicing ] = dataset[ block_relative_intersection_slicing ]
self._last_access_times[block_start] = time.time()
def ingestData(self, slot):
"""
Read the data from the given slot and copy it into this cache.
The rules about special pixel meanings apply here, just like setInSlot
Returns: the max label found in the slot.
"""
assert self._blockshape is not None
assert self.Input.meta.shape == slot.meta.shape
max_label = 0
# Get logical blocking.
block_starts = getIntersectingBlocks( self._blockshape, roiFromShape(self.Input.meta.shape) )
block_starts = map( tuple, block_starts )
# Write each block
for block_start in block_starts:
block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, block_start )
# Request the block data
block_data = slot(*block_roi).wait()
# Write into the array
subregion_roi = SubRegion(self.Input, *block_roi)
cleaned_block_data = self._setInSlotInput( self.Input, (), subregion_roi, block_data )
max_label = max( max_label, cleaned_block_data.max() )
return max_label
def _setInSlotInput(self, slot, subindex, roi, value):
assert len(roi.stop) == len(self.Input.meta.shape), "roi: {} has the wrong number of dimensions for Input shape: {}".format( roi, self.Input.meta.shape )
assert numpy.less_equal(roi.stop, self.Input.meta.shape).all(), "roi: {} is out-of-bounds for Input shape: {}".format( roi, self.Input.meta.shape )
block_starts = getIntersectingBlocks( self._blockshape, (roi.start, roi.stop) )
block_starts = map( tuple, block_starts )
# Copy data to each block
logger.debug( "Copying data INTO {} blocks...".format( len(block_starts) ) )
for block_start in block_starts:
entire_block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, block_start )
# This block's portion of the roi
intersecting_roi = getIntersection( (roi.start, roi.stop), entire_block_roi )
# Compute slicing within source array and slicing within this block
source_relative_intersection = numpy.subtract(intersecting_roi, roi.start)
block_relative_intersection = numpy.subtract(intersecting_roi, block_start)
# Copy from source to block
dataset = self._getBlockDataset( entire_block_roi )
dataset[ roiToSlice( *block_relative_intersection ) ] = value[ roiToSlice(*source_relative_intersection) ]
# Here, we assume that if this function is used to update ANY PART of a
# block, he is responsible for updating the ENTIRE block.
# Therefore, this block is no longer 'dirty'
self._dirtyBlocks.discard( block_start )
def _copyData(self, roi, destination, block_starts):
"""
Copy data from each block into the destination array.
For blocks that aren't currently stored, just write zeros.
"""
# (Parallelism not needed here: h5py will serialize these requests anyway)
block_starts = map( tuple, block_starts )
for block_start in block_starts:
entire_block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, block_start )
# This block's portion of the roi
intersecting_roi = getIntersection( (roi.start, roi.stop), entire_block_roi )
# Compute slicing within destination array and slicing within this block
destination_relative_intersection = numpy.subtract(intersecting_roi, roi.start)
block_relative_intersection = numpy.subtract(intersecting_roi, block_start)
destination_relative_intersection_slicing = roiToSlice(*destination_relative_intersection)
block_relative_intersection_slicing = roiToSlice(*block_relative_intersection)
if block_start in self._cacheFiles:
# Copy from block to destination
dataset = self._getBlockDataset( entire_block_roi )
if self.Output.meta.has_mask:
destination[ destination_relative_intersection_slicing ] = dataset["data"][ block_relative_intersection_slicing ]
destination.mask[ destination_relative_intersection_slicing ] = dataset["mask"][ block_relative_intersection_slicing ]
destination.fill_value = dataset["fill_value"][()]
else:
destination[ destination_relative_intersection_slicing ] = dataset[ block_relative_intersection_slicing ]
else:
# Not stored yet. Overwrite with zeros.
destination[ destination_relative_intersection_slicing ] = 0
def _ensurePipelineExists(self, block_start):
if block_start in self._blockPipelines:
return
with self._lock:
if block_start in self._blockPipelines:
return
logger.debug( "Creating pipeline for block: {}".format( block_start ) )
block_shape = self._getFullShape( self._block_shape_dict )
halo_padding = self._getFullShape( self._halo_padding_dict )
input_shape = self.RawImage.meta.shape
block_stop = getBlockBounds( input_shape, block_shape, block_start )[1]
block_roi = (block_start, block_stop)
# Instantiate pipeline
opBlockPipeline = OpSingleBlockObjectPrediction( block_roi, halo_padding, parent=self )
opBlockPipeline.RawImage.connect( self.RawImage )
opBlockPipeline.BinaryImage.connect( self.BinaryImage )
opBlockPipeline.Classifier.connect( self.Classifier )
opBlockPipeline.LabelsCount.connect( self.LabelsCount )
# Forward dirtyness
opBlockPipeline.PredictionImage.notifyDirty( bind(self._handleDirtyBlock, block_start ) )
self._blockPipelines[block_start] = opBlockPipeline
def testFailedProcessing(self):
op = OpArrayPiper( graph=Graph() )
inputData = numpy.indices( (100,100) ).sum(0)
op.Input.setValue( inputData )
roiList = []
block_starts = getIntersectingBlocks( [10,10], ([0,0], [100, 100]) )
for block_start in block_starts:
roiList.append( getBlockBounds( [100,100], [10,10], block_start ) )
class SpecialException(Exception): pass
def handleResult(roi, result):
raise SpecialException("Intentional Exception: raised while handling the result")
totalVolume = numpy.prod( inputData.shape )
batch = RoiRequestBatch(op.Output, roiList.__iter__(), totalVolume, batchSize=10, allowParallelResults=False)
batch.resultSignal.subscribe( handleResult )
# FIXME: There are multiple places where the RoiRequestBatch tool should be prepared to handle exceptions.
# This only tests one of them (in the notify_finished() handler)
try:
batch.execute()
except SpecialException:
pass
else:
assert False, "Expected exception to be propagated out of the RoiRequestBatch."
def _getBlockFileName(self, block_start):
# Translate to find disk block start
block_start = numpy.add( self.description.view_origin, block_start )
# Get true (disk) block bounds (i.e. use on-disk shape, not view_shape)
entire_block_roi = getBlockBounds( self.description.shape, self.description.block_shape, block_start )
roiString = "{}".format( (list(entire_block_roi[0]), list(entire_block_roi[1]) ) )
datasetFilename = self.description.block_file_name_format.format( roiString=roiString )
return datasetFilename
def read(self, view_roi, result_out):
"""
roi: (start, stop) tuples, ordered according to description.output_axes
roi should be relative to the view
"""
output_axes = self.description.output_axes
roi_transposed = zip(*view_roi)
roi_dict = dict( zip(output_axes, roi_transposed) )
view_roi = zip( *(roi_dict['z'], roi_dict['y'], roi_dict['x']) )
# First, normalize roi and result to zyx order
result_out = vigra.taggedView(result_out, output_axes)
result_out = result_out.withAxes(*'zyx')
assert numpy.array(view_roi).shape == (2,3), "Invalid roi for 3D volume: {}".format( view_roi )
view_roi = numpy.array(view_roi)
assert (result_out.shape == (view_roi[1] - view_roi[0])).all()
# User gave roi according to the view output.
# Now offset it find global roi.
roi = view_roi + self.description.view_origin_zyx
tile_blockshape = (1,) + tuple(self.description.tile_shape_2d_yx)
tile_starts = getIntersectingBlocks( tile_blockshape, roi )
pool = RequestPool()
for tile_start in tile_starts:
tile_roi_in = getBlockBounds( self.description.bounds_zyx, tile_blockshape, tile_start )
tile_roi_in = numpy.array(tile_roi_in)
# This tile's portion of the roi
intersecting_roi = getIntersection( roi, tile_roi_in )
intersecting_roi = numpy.array( intersecting_roi )
# Compute slicing within destination array and slicing within this tile
destination_relative_intersection = numpy.subtract(intersecting_roi, roi[0])
tile_relative_intersection = intersecting_roi - tile_roi_in[0]
# Get a view to the output slice
result_region = result_out[roiToSlice(*destination_relative_intersection)]
rest_args = self._get_rest_args(tile_blockshape, tile_roi_in)
if self.description.tile_url_format.startswith('http'):
retrieval_fn = partial( self._retrieve_remote_tile, rest_args, tile_relative_intersection, result_region )
else:
retrieval_fn = partial( self._retrieve_local_tile, rest_args, tile_relative_intersection, result_region )
PARALLEL_REQ = True
if PARALLEL_REQ:
pool.add( Request( retrieval_fn ) )
else:
# execute serially (leave the pool empty)
retrieval_fn()
if PARALLEL_REQ:
with Timer() as timer:
pool.wait()
logger.info("Loading {} tiles took a total of {}".format( len(tile_starts), timer.seconds() ))
def roiGen():
block_iter = block_starts.__iter__()
while True:
block_start = block_iter.next()
block_bounds = getBlockBounds( outputSlot.meta.shape, blockshape, block_start )
block_intersecting_portion = getIntersection( block_bounds, roi )
logger.debug( "Requesting Roi: {}".format( block_bounds ) )
yield block_intersecting_portion
def _setInSlotInputHdf5(self, slot, subindex, roi, value):
logger.debug("Setting block {} from hdf5".format(roi))
if self.Output.meta.has_mask:
assert isinstance(
value, h5py.Group
), "InputHdf5 slot requires an hdf5 Group to copy from (not a numpy masked array)."
else:
assert isinstance(
value, h5py.Dataset
), "InputHdf5 slot requires an hdf5 Dataset to copy from (not a numpy array)."
block_roi = getBlockBounds(self.Output.meta.shape, self._blockshape, roi.start)
roi_is_exactly_one_block = True
roi_is_exactly_one_block &= ((roi.start % self._blockshape) == 0).all()
roi_is_exactly_one_block &= (block_roi == numpy.array((roi.start, roi.stop))).all()
if roi_is_exactly_one_block:
cachefile = self._getCacheFile(block_roi)
logger.debug("Copying HDF5 data directly into block {}".format(block_roi))
if self.Output.meta.has_mask:
assert len(value) == 3
for each in ["data", "mask", "fill_value"]:
assert each in value
assert cachefile[each].dtype == value[each].dtype
assert cachefile[each].shape == value[each].shape
for each in ["data", "mask", "fill_value"]:
del cachefile[each]
cachefile.copy(value[each], each)
else:
assert cachefile["data"].dtype == value.dtype
assert cachefile["data"].shape == value.shape
del cachefile["data"]
cachefile.copy(value, "data")
block_start = tuple(roi.start)
self._dirtyBlocks.discard(block_start)
else:
# This hdf5 data does not correspond to exactly one block.
# We must uncompress it and write it the "normal" way (the slow way)
# FIXME: This would use less memory if we uncompressed the data block-by-block
data = None
if self.Output.meta.has_mask:
data = numpy.ma.masked_array(
value["data"][()], mask=value["mask"][()], fill_value=value["fill_value"][()], shrink=False
)
else:
data = value[()]
self.Input[roiToSlice(roi.start, roi.stop)] = data
def _waitForBlocks(self, block_starts):
"""
Make sure that all blocks in the given list of blocks are present in the cache before returning.
(Blocks that are not yet present will be requested from our Input slot.)
"""
reqPool = RequestPool() # (Do the work in parallel.)
for block_start in block_starts:
entire_block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, block_start )
f = partial( self._ensureCached, entire_block_roi)
reqPool.add( Request(f) )
logger.debug( "Waiting for {} blocks...".format( len(block_starts) ) )
reqPool.wait()
def _executeOutputHdf5(self, roi, destination):
logger.debug("Servicing request for hdf5 block {}".format( roi ))
assert isinstance( destination, h5py.Group ), "OutputHdf5 slot requires an hdf5 GROUP to copy into (not a numpy array)."
assert ((roi.start % self._blockshape) == 0).all(), "OutputHdf5 slot requires roi to be exactly one block."
block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, roi.start )
assert (block_roi == numpy.array((roi.start, roi.stop))).all(), "OutputHdf5 slot requires roi to be exactly one block."
block_roi = [roi.start, roi.stop]
self._ensureCached( block_roi )
dataset = self._getBlockDataset( block_roi )
assert str(block_roi) not in destination, "destination hdf5 group already has a dataset with this block's name"
destination.copy( dataset, str(block_roi) )
return destination
def roiGen():
block_iter = block_starts.__iter__()
while True:
try:
block_start = next(block_iter)
except StopIteration:
# As of Python 3.7, not allowed to let StopIteration exceptions escape a generator
# https://www.python.org/dev/peps/pep-0479
break
else:
block_bounds = getBlockBounds(outputSlot.meta.shape, blockshape, block_start)
block_intersecting_portion = getIntersection(block_bounds, roi)
logger.debug("Requesting Roi: {}".format(block_bounds))
yield block_intersecting_portion
def roiGen():
block_iter = self._minBlockStarts.__iter__()
while True:
block_start = block_iter.next()
# Use offset blocking
offset_block_start = block_start - self._bigRoi[0]
offset_data_shape = numpy.subtract(self._bigRoi[1], self._bigRoi[0])
offset_block_bounds = getBlockBounds( offset_data_shape, minBlockShape, offset_block_start )
# Un-offset
block_bounds = ( offset_block_bounds[0] + self._bigRoi[0],
offset_block_bounds[1] + self._bigRoi[0] )
logger.debug( "Requesting Roi: {}".format( block_bounds ) )
yield block_bounds
def _get_features_for_block(self, block_start):
"""
Computes the feature matrix for the given block IFF the block is dirty.
Otherwise, returns None.
"""
# Caller must ensure that the lock for this block already exists!
with self._block_locks[block_start]:
if block_start not in self._dirty_blocks:
# Nothing to do if this block isn't actually dirty
# (For parallel requests, its theoretically possible.)
return None
block_roi = getBlockBounds( self.LabelImage.meta.shape, self._blockshape, block_start )
# TODO: Shrink the requested roi using the nonzero blocks slot...
# ...or just get rid of the nonzero blocks slot...
labels_and_features_matrix = self._extract_feature_matrix(block_roi)
return labels_and_features_matrix
def _setInSlotInputHdf5(self, slot, subindex, roi, value):
logger.debug("Setting block {} from hdf5".format( roi ))
assert isinstance( value, h5py.Dataset ), "InputHdf5 slot requires an hdf5 Dataset to copy from (not a numpy array)."
assert ((roi.start % self._blockshape) == 0).all(), "InputHdf5 slot requires roi to be exactly one block."
block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, roi.start )
assert (block_roi == numpy.array((roi.start, roi.stop))).all(), "InputHdf5 slot requires roi to be exactly one block."
cachefile = self._getCacheFile( block_roi )
logger.debug( "Copying HDF5 data directly into block {}".format( block_roi ) )
assert cachefile['data'].dtype == value.dtype
assert cachefile['data'].shape == value.shape
del cachefile['data']
cachefile.copy( value, 'data' )
block_start = tuple(roi.start)
self._dirtyBlocks.discard( block_start )
def _copyData(self, roi, destination, block_starts):
# Copy data from each block
# (Parallelism not needed here: h5py will serialize these requests anyway)
logger.debug( "Copying data from {} blocks...".format( len(block_starts) ) )
for block_start in block_starts:
entire_block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, block_start )
# This block's portion of the roi
intersecting_roi = getIntersection( (roi.start, roi.stop), entire_block_roi )
# Compute slicing within destination array and slicing within this block
destination_relative_intersection = numpy.subtract(intersecting_roi, roi.start)
block_relative_intersection = numpy.subtract(intersecting_roi, block_start)
# Copy from block to destination
dataset = self._getBlockDataset( entire_block_roi )
destination[ roiToSlice(*destination_relative_intersection) ] = dataset[ roiToSlice( *block_relative_intersection ) ]
def _get_features_for_block(self, block_start):
"""
Computes the feature matrix for the given block IFF the block is dirty.
Otherwise, returns None.
"""
# Caller must ensure that the lock for this block already exists!
with self._block_locks[block_start]:
if block_start not in self._dirty_blocks:
# Nothing to do if this block isn't actually dirty
# (For parallel requests, its theoretically possible.)
return None
block_roi = getBlockBounds(self.LabelImage.meta.shape,
self._blockshape, block_start)
# TODO: Shrink the requested roi using the nonzero blocks slot...
# ...or just get rid of the nonzero blocks slot...
labels_and_features_matrix = self._extract_feature_matrix(
block_roi)
return labels_and_features_matrix
def export_to_tiles(volume, tile_size, output_dir, print_progress=True):
"""
volume: The volume to export (either hdf5 dataset or numpy array). Must be 3D.
tile_size: The width of the tiles to generate
output_dir: The directory to dump the tiles to.
"""
assert len(volume.shape) == 3
tile_blockshape = (1, tile_size, tile_size)
tile_starts = getIntersectingBlocks(tile_blockshape,
roiFromShape(volume.shape))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
logger.info("Writing {} tiles ...".format(len(tile_starts)))
for tile_start in tile_starts:
tile_roi = getBlockBounds(volume.shape, tile_blockshape, tile_start)
if print_progress:
sys.stdout.write("Tile: {} ".format(tile_roi))
sys.stdout.flush()
tile_data = volume[roiToSlice(*tile_roi)]
tile_data = vigra.taggedView(tile_data, 'zyx')
if print_progress:
sys.stdout.write('reading... ')
sys.stdout.flush()
tile_name = 'tile_z{:05}_y{:05}_x{:05}.png'.format(*tile_start)
output_path = os.path.join(output_dir, tile_name)
if print_progress:
sys.stdout.write('writing... ')
sys.stdout.flush()
vigra.impex.writeImage(tile_data[0], output_path, dtype='NATIVE')
if print_progress:
sys.stdout.write('done.\n')
sys.stdout.flush()
logger.info("TILES COMPLETE.")
def testBasic(self):
op = OpArrayPiper( graph=Graph() )
inputData = numpy.indices( (100,100) ).sum(0)
op.Input.setValue( inputData )
roiList = []
block_starts = getIntersectingBlocks( [10,10], ([0,0], [100, 100]) )
for block_start in block_starts:
roiList.append( getBlockBounds( [100,100], [10,10], block_start ) )
results = numpy.zeros( (100,100), dtype=numpy.int32 )
resultslock = threading.Lock()
resultsCount = [0]
def handleResult(roi, result):
acquired = resultslock.acquire(False)
assert acquired, "resultslock is contested! Access to callback is supposed to be automatically serialized."
results[ roiToSlice( *roi ) ] = result
logger.debug( "Got result for {}".format(roi) )
resultslock.release()
resultsCount[0] += 1
progressList = []
def handleProgress( progress ):
progressList.append( progress )
logger.debug( "Progress update: {}".format(progress) )
totalVolume = numpy.prod( inputData.shape )
batch = RoiRequestBatch(op.Output, roiList.__iter__(), totalVolume, batchSize=10)
batch.resultSignal.subscribe( handleResult )
batch.progressSignal.subscribe( handleProgress )
batch.execute()
logger.debug( "Got {} results".format( resultsCount[0] ) )
assert (results == inputData).all()
# Progress reporting MUST start with 0 and end with 100
assert progressList[0] == 0, "Invalid progress reporting."
assert progressList[-1] == 100, "Invalid progress reporting."
# There should be some intermediate progress reporting, but exactly how much is unspecified.
assert len(progressList) >= 10
logger.debug( "FINISHED" )
def autoSeedBackground(cls, laneView, foreground_label):
# Seed the entire image with background labels, except for the individual label in question
# To save memory, we'll do this in blocks instead of all at once
volume_shape = laneView.RavelerLabels.meta.shape
volume_roi = roiFromShape( volume_shape )
block_shape = (OpSplitBodyCarving.BLOCK_SIZE,) * len( volume_shape )
block_shape = numpy.minimum( block_shape, volume_shape )
block_starts = getIntersectingBlocks( block_shape, volume_roi )
logger.debug("Auto-seeding {} blocks for label".format( len(block_starts), foreground_label ))
for block_index, block_start in enumerate(block_starts):
block_roi = getBlockBounds( volume_shape, block_shape, block_start )
label_block = laneView.RavelerLabels(*block_roi).wait()
background_block = numpy.where( label_block == foreground_label, 0, 1 )
background_block = numpy.asarray( background_block, numpy.float32 ) # Distance transform requires float
if (background_block == 0.0).any():
# We need to leave a small border between the background seeds and the object membranes
background_block_view = background_block.view( vigra.VigraArray )
background_block_view.axistags = copy.copy( laneView.RavelerLabels.meta.axistags )
background_block_view_4d = background_block_view.bindAxis('t', 0)
background_block_view_3d = background_block_view_4d.bindAxis('c', 0)
distance_transformed_block = vigra.filters.distanceTransform3D(background_block_view_3d, background=False)
distance_transformed_block = distance_transformed_block.astype( numpy.uint8 )
# Create a 'hull' surrounding the foreground, but leave some space.
background_seed_block = (distance_transformed_block == OpSplitBodyCarving.SEED_MARGIN)
background_seed_block = background_seed_block.astype(numpy.uint8) * 1 # (In carving, background is label 1)
# # Make the hull VERY sparse to avoid over-biasing graph cut toward the background class
# # FIXME: Don't regenerate this random block on every loop iteration
# rand_bytes = numpy.random.randint(0, 1000, background_seed_block.shape)
# background_seed_block = numpy.where( rand_bytes < 1, background_seed_block, 0 )
# background_seed_block = background_seed_block.view(vigra.VigraArray)
# background_seed_block.axistags = background_block_view_3d.axistags
axisorder = laneView.RavelerLabels.meta.getTaggedShape().keys()
logger.debug("Writing backgound seeds: {}/{}".format( block_index, len(block_starts) ))
laneView.WriteSeeds[ roiToSlice( *block_roi ) ] = background_seed_block.withAxes(*axisorder)
else:
logger.debug("Skipping all-background block: {}/{}".format( block_index, len(block_starts) ))
def _copyData(self, roi, destination, block_starts):
"""
Copy data from each block into the destination array.
For blocks that aren't currently stored, just write zeros.
"""
# (Parallelism not needed here: h5py will serialize these requests anyway)
block_starts = map(tuple, block_starts)
for block_start in block_starts:
entire_block_roi = getBlockBounds(self.Output.meta.shape,
self._blockshape, block_start)
# This block's portion of the roi
intersecting_roi = getIntersection((roi.start, roi.stop),
entire_block_roi)
# Compute slicing within destination array and slicing within this block
destination_relative_intersection = numpy.subtract(
intersecting_roi, roi.start)
block_relative_intersection = numpy.subtract(
intersecting_roi, block_start)
destination_relative_intersection_slicing = roiToSlice(
*destination_relative_intersection)
block_relative_intersection_slicing = roiToSlice(
*block_relative_intersection)
if block_start in self._cacheFiles:
# Copy from block to destination
dataset = self._getBlockDataset(entire_block_roi)
if self.Output.meta.has_mask:
destination[
destination_relative_intersection_slicing] = dataset[
"data"][block_relative_intersection_slicing]
destination.mask[
destination_relative_intersection_slicing] = dataset[
"mask"][block_relative_intersection_slicing]
destination.fill_value = dataset["fill_value"][()]
else:
destination[
destination_relative_intersection_slicing] = dataset[
block_relative_intersection_slicing]
else:
# Not stored yet. Overwrite with zeros.
destination[destination_relative_intersection_slicing] = 0
def export_to_tiles(volume, tile_size, output_dir, print_progress=True):
"""
volume: The volume to export (either hdf5 dataset or numpy array). Must be 3D.
tile_size: The width of the tiles to generate
output_dir: The directory to dump the tiles to.
"""
assert len(volume.shape) == 3
tile_blockshape = (1, tile_size, tile_size)
tile_starts = getIntersectingBlocks( tile_blockshape, roiFromShape( volume.shape ) )
if not os.path.exists(output_dir):
os.makedirs(output_dir)
logger.info("Writing {} tiles ...".format( len(tile_starts) ) )
for tile_start in tile_starts:
tile_roi = getBlockBounds( volume.shape, tile_blockshape, tile_start )
if print_progress:
sys.stdout.write("Tile: {} ".format( tile_roi ))
sys.stdout.flush()
tile_data = volume[ roiToSlice(*tile_roi) ]
tile_data = vigra.taggedView(tile_data, 'zyx')
if print_progress:
sys.stdout.write('reading... ')
sys.stdout.flush()
tile_name = 'tile_z{:05}_y{:05}_x{:05}.png'.format( *tile_start )
output_path = os.path.join( output_dir, tile_name )
if print_progress:
sys.stdout.write('writing... ')
sys.stdout.flush()
vigra.impex.writeImage( tile_data[0], output_path, dtype='NATIVE' )
if print_progress:
sys.stdout.write('done.\n')
sys.stdout.flush()
logger.info( "TILES COMPLETE." )
def _setInSlotInput(self, slot, subindex, roi, value, store_zero_blocks=True):
"""
Write the data in the array 'value' into the cache.
If the optional store_zero_blocks param is False, then don't bother
creating cache blocks for blocks that are totally zero.
"""
assert len(roi.stop) == len(self.Input.meta.shape), \
"roi: {} has the wrong number of dimensions for Input shape: {}"\
"".format( roi, self.Input.meta.shape )
assert numpy.less_equal(roi.stop, self.Input.meta.shape).all(), \
"roi: {} is out-of-bounds for Input shape: {}"\
"".format( roi, self.Input.meta.shape )
block_starts = getIntersectingBlocks( self._blockshape, (roi.start, roi.stop) )
block_starts = map( tuple, block_starts )
# Copy data to each block
logger.debug( "Copying data INTO {} blocks...".format( len(block_starts) ) )
for block_start in block_starts:
entire_block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, block_start )
# This block's portion of the roi
intersecting_roi = getIntersection( (roi.start, roi.stop), entire_block_roi )
# Compute slicing within source array and slicing within this block
source_relative_intersection = numpy.subtract(intersecting_roi, roi.start)
block_relative_intersection = numpy.subtract(intersecting_roi, block_start)
new_block_data = value[ roiToSlice(*source_relative_intersection) ]
if not store_zero_blocks and new_block_data.sum() == 0 and block_start not in self._cacheFiles:
# Special fast-path: If this block doesn't exist yet,
# don't bother creating if we're just going to fill it with zeros
# (Used by the OpCompressedUserLabelArray)
pass
else:
# Copy from source to block
dataset = self._getBlockDataset( entire_block_roi )
dataset[ roiToSlice( *block_relative_intersection ) ] = new_block_data
# Here, we assume that if this function is used to update ANY PART of a
# block, he is responsible for updating the ENTIRE block.
# Therefore, this block is no longer 'dirty'
self._dirtyBlocks.discard( block_start )
def _executeOutputHdf5(self, roi, destination):
logger.debug("Servicing request for hdf5 block {}".format(roi))
assert isinstance(
destination, h5py.Group
), "OutputHdf5 slot requires an hdf5 GROUP to copy into (not a numpy array)."
assert ((roi.start % self._blockshape) == 0).all(
), "OutputHdf5 slot requires roi to be exactly one block."
block_roi = getBlockBounds(self.Output.meta.shape, self._blockshape,
roi.start)
assert (block_roi == numpy.array(
(roi.start, roi.stop
))).all(), "OutputHdf5 slot requires roi to be exactly one block."
block_roi = [roi.start, roi.stop]
self._ensureCached(block_roi)
dataset = self._getBlockDataset(block_roi)
assert str(
block_roi
) not in destination, "destination hdf5 group already has a dataset with this block's name"
destination.copy(dataset, str(block_roi))
return destination
def roiGen():
block_iter = block_starts.__iter__()
while True:
try:
block_start = next(block_iter)
except StopIteration:
# As of Python 3.7, not allowed to let StopIteration exceptions escape a generator
# https://www.python.org/dev/peps/pep-0479
break
else:
# Use offset blocking
offset_block_start = block_start - self._bigRoi[0]
offset_data_shape = numpy.subtract(self._bigRoi[1], self._bigRoi[0])
offset_block_bounds = getBlockBounds(offset_data_shape, blockshape, offset_block_start)
# Un-offset
block_bounds = (
offset_block_bounds[0] + self._bigRoi[0],
offset_block_bounds[1] + self._bigRoi[0],
)
logger.debug("Requesting Roi: {}".format(block_bounds))
yield block_bounds
def _copyData(self, roi, destination, block_starts):
# Copy data from each block
# (Parallelism not needed here: h5py will serialize these requests anyway)
logger.debug("Copying data from {} blocks...".format(
len(block_starts)))
for block_start in block_starts:
entire_block_roi = getBlockBounds(self.Output.meta.shape,
self._blockshape, block_start)
# This block's portion of the roi
intersecting_roi = getIntersection((roi.start, roi.stop),
entire_block_roi)
# Compute slicing within destination array and slicing within this block
destination_relative_intersection = numpy.subtract(
intersecting_roi, roi.start)
block_relative_intersection = numpy.subtract(
intersecting_roi, block_start)
destination_relative_intersection_slicing = roiToSlice(
*destination_relative_intersection)
block_relative_intersection_slicing = roiToSlice(
*block_relative_intersection)
# Copy from block to destination
dataset = self._getBlockDataset(entire_block_roi)
if self.Output.meta.has_mask:
destination.data[
destination_relative_intersection_slicing] = dataset[
"data"][block_relative_intersection_slicing]
destination.mask[
destination_relative_intersection_slicing] = dataset[
"mask"][block_relative_intersection_slicing]
destination.fill_value = dataset["fill_value"][()]
else:
destination[
destination_relative_intersection_slicing] = dataset[
block_relative_intersection_slicing]
self._last_access_times[block_start] = time.time()
def ingestData(self, slot):
"""
Read the data from the given slot and copy it into this cache.
The rules about special pixel meanings apply here, just like setInSlot
Returns: the max label found in the slot.
"""
assert self._blockshape is not None
assert self.Output.meta.shape[:
-1] == slot.meta.shape[:
-1], "{} != {}".format(
self.Output.
meta.shape,
slot.meta.
shape)
max_label = 0
# Get logical blocking.
block_starts = getIntersectingBlocks(
self._blockshape, roiFromShape(self.Output.meta.shape))
block_starts = list(map(tuple, block_starts))
# Write each block
for block_start in block_starts:
block_roi = getBlockBounds(self.Output.meta.shape,
self._blockshape, block_start)
# Request the block data
block_data = slot(*block_roi).wait()
# Write into the array
subregion_roi = SubRegion(self.Output, *block_roi)
cleaned_block_max = self._setInSlotInput(self.Input, (),
subregion_roi, block_data)
max_label = max(max_label, cleaned_block_max)
return max_label
def _update_block(self, block_start):
if block_start not in self._block_locks:
with self._lock:
if block_start not in self._block_locks:
self._block_locks[block_start] = RequestLock()
with self._block_locks[block_start]:
if block_start not in self._dirty_blocks:
# Nothing to do if this block isn't actually dirty
# (For parallel requests, its theoretically possible.)
return
block_roi = getBlockBounds( self.LabelImage.meta.shape, self._blockshape, block_start )
# TODO: Shrink the requested roi using the nonzero blocks slot...
# ...or just get rid of the nonzero blocks slot...
labels_and_features_matrix = self._extract_feature_matrix(block_roi)
with self._lock:
self._dirty_blocks.remove(block_start)
if labels_and_features_matrix.shape[0] > 0:
self._blockwise_feature_matrices[block_start] = labels_and_features_matrix
else:
try:
del self._blockwise_feature_matrices[block_start]
except KeyError:
pass
def testFailedProcessing(self):
op = OpArrayPiper(graph=Graph())
inputData = numpy.indices((100, 100)).sum(0)
op.Input.setValue(inputData)
roiList = []
block_starts = getIntersectingBlocks([10, 10], ([0, 0], [100, 100]))
for block_start in block_starts:
roiList.append(getBlockBounds([100, 100], [10, 10], block_start))
class SpecialException(Exception):
pass
def handleResult(roi, result):
raise SpecialException("Intentional Exception: raised while handling the result")
totalVolume = numpy.prod(inputData.shape)
batch = RoiRequestBatch(op.Output, roiList.__iter__(), totalVolume, batchSize=10, allowParallelResults=False)
batch.resultSignal.subscribe(handleResult)
# FIXME: There are multiple places where the RoiRequestBatch tool should be prepared to handle exceptions.
# This only tests one of them (in the notify_finished() handler)
with pytest.raises(SpecialException):
batch.execute()
def roiGen():
block_iter = block_starts.__iter__()
while True:
try:
block_start = next(block_iter)
except StopIteration:
# As of Python 3.7, not allowed to let StopIteration exceptions escape a generator
# https://www.python.org/dev/peps/pep-0479
break
else:
# Use offset blocking
offset_block_start = block_start - self._bigRoi[0]
offset_data_shape = numpy.subtract(
self._bigRoi[1], self._bigRoi[0])
offset_block_bounds = getBlockBounds(
offset_data_shape, blockshape, offset_block_start)
# Un-offset
block_bounds = (
offset_block_bounds[0] + self._bigRoi[0],
offset_block_bounds[1] + self._bigRoi[0],
)
logger.debug("Requesting Roi: {}".format(block_bounds))
yield block_bounds
def _setInSlotInputHdf5(self, slot, subindex, roi, value):
logger.debug("Setting block {} from hdf5".format( roi ))
assert isinstance( value, h5py.Dataset ), "InputHdf5 slot requires an hdf5 Dataset to copy from (not a numpy array)."
block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, roi.start )
roi_is_exactly_one_block = True
roi_is_exactly_one_block &= ((roi.start % self._blockshape) == 0).all()
roi_is_exactly_one_block &= (block_roi == numpy.array((roi.start, roi.stop))).all()
if roi_is_exactly_one_block:
cachefile = self._getCacheFile( block_roi )
logger.debug( "Copying HDF5 data directly into block {}".format( block_roi ) )
assert cachefile['data'].dtype == value.dtype
assert cachefile['data'].shape == value.shape
del cachefile['data']
cachefile.copy( value, 'data' )
block_start = tuple(roi.start)
self._dirtyBlocks.discard( block_start )
else:
# This hdf5 data does not correspond to exactly one block.
# We must uncompress it and write it the "normal" way (the slow way)
# FIXME: This would use less memory if we uncompressed the data block-by-block
self.Input[roiToSlice(roi.start, roi.stop)] = value[:]
def _copyData(self, roi, destination, block_starts):
# Copy data from each block
# (Parallelism not needed here: h5py will serialize these requests anyway)
logger.debug("Copying data from {} blocks...".format(
len(block_starts)))
for block_start in block_starts:
entire_block_roi = getBlockBounds(self.Output.meta.shape,
self._blockshape, block_start)
# This block's portion of the roi
intersecting_roi = getIntersection((roi.start, roi.stop),
entire_block_roi)
# Compute slicing within destination array and slicing within this block
destination_relative_intersection = numpy.subtract(
intersecting_roi, roi.start)
block_relative_intersection = numpy.subtract(
intersecting_roi, block_start)
# Copy from block to destination
dataset = self._getBlockDataset(entire_block_roi)
destination[roiToSlice(
*destination_relative_intersection)] = dataset[roiToSlice(
*block_relative_intersection)]
def get_block_roi(self, block_start):
block_shape = self._getFullShape(self._block_shape_dict)
input_shape = self.RawImage.meta.shape
block_stop = getBlockBounds(input_shape, block_shape, block_start)[1]
block_roi = (block_start, block_stop)
return block_roi
def _setInSlotInput(self, slot, subindex, roi, value, store_zero_blocks=True):
"""
Write the data in the array 'value' into the cache.
If the optional store_zero_blocks param is False, then don't bother
creating cache blocks for blocks that are totally zero.
"""
assert len(roi.stop) == len(self.Input.meta.shape), \
"roi: {} has the wrong number of dimensions for Input shape: {}"\
"".format( roi, self.Input.meta.shape )
assert numpy.less_equal(roi.stop, self.Input.meta.shape).all(), \
"roi: {} is out-of-bounds for Input shape: {}"\
"".format( roi, self.Input.meta.shape )
block_starts = getIntersectingBlocks( self._blockshape, (roi.start, roi.stop) )
block_starts = map( tuple, block_starts )
# Copy data to each block
logger.debug( "Copying data INTO {} blocks...".format( len(block_starts) ) )
for block_start in block_starts:
entire_block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, block_start )
# This block's portion of the roi
intersecting_roi = getIntersection( (roi.start, roi.stop), entire_block_roi )
# Compute slicing within source array and slicing within this block
source_relative_intersection = numpy.subtract(intersecting_roi, roi.start)
block_relative_intersection = numpy.subtract(intersecting_roi, block_start)
source_relative_intersection_slicing = roiToSlice( *source_relative_intersection )
block_relative_intersection_slicing = roiToSlice( *block_relative_intersection )
new_block_data = value[ source_relative_intersection_slicing ]
new_block_sum = new_block_data.sum()
if not store_zero_blocks and new_block_sum == 0 and block_start not in self._cacheFiles:
# Special fast-path: If this block doesn't exist yet,
# don't bother creating if we're just going to fill it with zeros.
# (This feature is used by the OpCompressedUserLabelArray)
pass
else:
# Copy from source to block
dataset = self._getBlockDataset( entire_block_roi )
if self.Output.meta.has_mask:
dataset["data"][ block_relative_intersection_slicing ] = new_block_data.data
dataset["mask"][ block_relative_intersection_slicing ] = new_block_data.mask
dataset["fill_value"][()] = new_block_data.fill_value
# Untested. Write a test to use this.
# # If we can, remove this block entirely.
# if not store_zero_blocks and new_block_sum == 0 and (dataset["data"][:] == 0).all() and (dataset["mask"]).any() and (dataset["fill_value"] == 0).all():
# with self._lock:
# with self._blockLocks[block_start]:
# self._cacheFiles[block_start].close()
# del self._cacheFiles[block_start]
# del self._blockLocks[block_start]
else:
dataset[ block_relative_intersection_slicing ] = new_block_data
# If we can, remove this block entirely.
if not store_zero_blocks and new_block_sum == 0 and (dataset[:] == 0).all():
with self._lock:
with self._blockLocks[block_start]:
self._cacheFiles[block_start].close()
del self._cacheFiles[block_start]
del self._blockLocks[block_start]
# Here, we assume that if this function is used to update ANY PART of a
# block, he is responsible for updating the ENTIRE block.
# Therefore, this block is no longer 'dirty'
self._dirtyBlocks.discard( block_start )
def getEntireBlockRoi(self, block_start):
"""
Return the roi for the entire block that starts at the given coordinate.
"""
return getBlockBounds(self._description.view_shape, self._description.block_shape, block_start)
def read(self, view_roi, result_out):
"""
roi: (start, stop) tuples, ordered according to description.output_axes
roi should be relative to the view
"""
output_axes = self.description.output_axes
roi_transposed = zip(*view_roi)
roi_dict = dict(zip(output_axes, roi_transposed))
view_roi = zip(*(roi_dict['z'], roi_dict['y'], roi_dict['x']))
# First, normalize roi and result to zyx order
result_out = vigra.taggedView(result_out, output_axes)
result_out = result_out.withAxes(*'zyx')
assert numpy.array(view_roi).shape == (
2, 3), "Invalid roi for 3D volume: {}".format(view_roi)
view_roi = numpy.array(view_roi)
assert (result_out.shape == (view_roi[1] - view_roi[0])).all()
# User gave roi according to the view output.
# Now offset it find global roi.
roi = view_roi + self.description.view_origin_zyx
tile_blockshape = (1, ) + tuple(self.description.tile_shape_2d_yx)
tile_starts = getIntersectingBlocks(tile_blockshape, roi)
pool = RequestPool()
for tile_start in tile_starts:
tile_roi_in = getBlockBounds(self.description.bounds_zyx,
tile_blockshape, tile_start)
tile_roi_in = numpy.array(tile_roi_in)
# This tile's portion of the roi
intersecting_roi = getIntersection(roi, tile_roi_in)
intersecting_roi = numpy.array(intersecting_roi)
# Compute slicing within destination array and slicing within this tile
destination_relative_intersection = numpy.subtract(
intersecting_roi, roi[0])
tile_relative_intersection = intersecting_roi - tile_roi_in[0]
# Get a view to the output slice
result_region = result_out[roiToSlice(
*destination_relative_intersection)]
rest_args = self._get_rest_args(tile_blockshape, tile_roi_in)
if self.description.tile_url_format.startswith('http'):
retrieval_fn = partial(self._retrieve_remote_tile, rest_args,
tile_relative_intersection,
result_region)
else:
retrieval_fn = partial(self._retrieve_local_tile, rest_args,
tile_relative_intersection,
result_region)
PARALLEL_REQ = True
if PARALLEL_REQ:
pool.add(Request(retrieval_fn))
else:
# execute serially (leave the pool empty)
retrieval_fn()
if PARALLEL_REQ:
with Timer() as timer:
pool.wait()
logger.info("Loading {} tiles took a total of {}".format(
len(tile_starts), timer.seconds()))
def get_block_roi(self, block_start):
block_shape = self._getFullShape( self._block_shape_dict )
input_shape = self.RawImage.meta.shape
block_stop = getBlockBounds( input_shape, block_shape, block_start )[1]
block_roi = (block_start, block_stop)
return block_roi
def read(self, roi, result_out):
"""
roi: (start, stop) tuples, ordered according to description.output_axes
"""
output_axes = self.description.output_axes
roi_transposed = zip(*roi)
roi_dict = dict(zip(output_axes, roi_transposed))
roi = zip(*(roi_dict['z'], roi_dict['y'], roi_dict['x']))
# First, normalize roi and result to zyx order
result_out = vigra.taggedView(result_out, output_axes)
result_out = result_out.withAxes(*'zyx')
assert numpy.array(roi).shape == (
2, 3), "Invalid roi for 3D volume: {}".format(roi)
roi = numpy.array(roi)
assert (result_out.shape == (roi[1] - roi[0])).all()
tile_blockshape = (1, ) + tuple(self.description.tile_shape_2d_yx)
tile_starts = getIntersectingBlocks(tile_blockshape, roi)
# We use a fresh tmp dir for each read to avoid conflicts between parallel reads
tmpdir = tempfile.mkdtemp()
pool = RequestPool()
for tile_start in tile_starts:
tile_roi_in = getBlockBounds(self.description.shape_zyx,
tile_blockshape, tile_start)
tile_roi_in = numpy.array(tile_roi_in)
# This tile's portion of the roi
intersecting_roi = getIntersection(roi, tile_roi_in)
intersecting_roi = numpy.array(intersecting_roi)
# Compute slicing within destination array and slicing within this tile
destination_relative_intersection = numpy.subtract(
intersecting_roi, roi[0])
tile_relative_intersection = intersecting_roi - tile_roi_in[0]
# Get a view to the output slice
result_region = result_out[roiToSlice(
*destination_relative_intersection)]
# Special feature:
# Some slices are missing, in which case we provide fake data from a different slice.
# Overwrite the rest args to pull data from an alternate source tile.
z_start = tile_roi_in[0][0]
if z_start in self._slice_remapping:
new_source_slice = self._slice_remapping[z_start]
tile_roi_in[0][0] = new_source_slice
tile_roi_in[1][0] = new_source_slice + 1
tile_index = numpy.array(tile_roi_in[0]) / tile_blockshape
rest_args = {
'z_start': tile_roi_in[0][0],
'z_stop': tile_roi_in[1][0],
'y_start': tile_roi_in[0][1],
'y_stop': tile_roi_in[1][1],
'x_start': tile_roi_in[0][2],
'x_stop': tile_roi_in[1][2],
'z_index': tile_index[0],
'y_index': tile_index[1],
'x_index': tile_index[2]
}
# Quick sanity check
assert rest_args['z_index'] == rest_args['z_start']
retrieval_fn = partial(self._retrieve_tile, tmpdir, rest_args,
tile_relative_intersection, result_region)
PARALLEL_REQ = True
if PARALLEL_REQ:
pool.add(Request(retrieval_fn))
else:
# execute serially (leave the pool empty)
retrieval_fn()
pool.wait()
# Clean up our temp files.
shutil.rmtree(tmpdir)
def _executeProjection2D(self, roi, destination):
assert sum(TinyVector(destination.shape) > 1) <= 2, "Projection result must be exactly 2D"
# First, we have to determine which axis we are projecting along.
# We infer this from the shape of the roi.
# For example, if the roi is of shape
# zyx = (1,256,256), then we know we're projecting along Z
# If more than one axis has a width of 1, then we choose an
# axis according to the following priority order: zyxt
tagged_input_shape = self.Input.meta.getTaggedShape()
tagged_result_shape = collections.OrderedDict( zip( tagged_input_shape.keys(),
destination.shape ) )
nonprojection_axes = []
for key in tagged_input_shape.keys():
if (key == 'c' or tagged_input_shape[key] == 1 or tagged_result_shape[key] > 1):
nonprojection_axes.append( key )
possible_projection_axes = set(tagged_input_shape) - set(nonprojection_axes)
if len(possible_projection_axes) == 0:
# If the image is 2D to begin with,
# then the projection is simply the same as the normal output,
# EXCEPT it is made binary
self.Output(roi.start, roi.stop).writeInto(destination).wait()
# make binary
numpy.greater(destination, 0, out=destination)
return
for k in 'zyxt':
if k in possible_projection_axes:
projection_axis_key = k
break
# Now we know which axis we're projecting along.
# Proceed with the projection, working blockwise to avoid unecessary work in unlabeled blocks
projection_axis_index = self.Input.meta.getAxisKeys().index(projection_axis_key)
projection_length = tagged_input_shape[projection_axis_key]
input_roi = roi.copy()
input_roi.start[projection_axis_index] = 0
input_roi.stop[projection_axis_index] = projection_length
destination[:] = 0.0
# Get the logical blocking.
block_starts = getIntersectingBlocks( self._blockshape, (input_roi.start, input_roi.stop) )
# (Parallelism wouldn't help here: h5py will serialize these requests anyway)
block_starts = map( tuple, block_starts )
for block_start in block_starts:
if block_start not in self._cacheFiles:
# No label data in this block. Move on.
continue
entire_block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, block_start )
# This block's portion of the roi
intersecting_roi = getIntersection( (input_roi.start, input_roi.stop), entire_block_roi )
# Compute slicing within the deep array and slicing within this block
deep_relative_intersection = numpy.subtract(intersecting_roi, input_roi.start)
block_relative_intersection = numpy.subtract(intersecting_roi, block_start)
deep_data = self._getBlockDataset( entire_block_roi )[roiToSlice(*block_relative_intersection)]
# make binary and convert to float
deep_data_float = numpy.where( deep_data, numpy.float32(1.0), numpy.float32(0.0) )
# multiply by slice-index
deep_data_view = numpy.rollaxis(deep_data_float, projection_axis_index, 0)
min_deep_slice_index = deep_relative_intersection[0][projection_axis_index]
max_deep_slice_index = deep_relative_intersection[1][projection_axis_index]
def calc_color_value(slice_index):
# Note 1: We assume that the colortable has at least 256 entries in it,
# so, we try to ensure that all colors are above 1/256
# (we don't want colors in low slices to be rounded to 0)
# Note 2: Ideally, we'd use a min projection in the code below, so that
# labels in the "back" slices would appear occluded. But the
# min projection would favor 0.0. Instead, we invert the
# relationship between color and slice index, do a max projection,
# and then re-invert the colors after everything is done.
# Hence, this function starts with (1.0 - ...)
return (1.0 - (float(slice_index) / projection_length)) * (1.0 - 1.0/255) + 1.0/255.0
min_color_value = calc_color_value(min_deep_slice_index)
max_color_value = calc_color_value(max_deep_slice_index)
num_slices = max_deep_slice_index - min_deep_slice_index
deep_data_view *= numpy.linspace( min_color_value, max_color_value, num=num_slices )\
[ (slice(None),) + (None,)*(deep_data_view.ndim-1) ]
# Take the max projection of this block's data.
block_max_projection = numpy.amax(deep_data_float, axis=projection_axis_index, keepdims=True)
# Merge this block's projection into the overall projection.
destination_relative_intersection = numpy.array(deep_relative_intersection)
destination_relative_intersection[:, projection_axis_index] = (0,1)
destination_subview = destination[roiToSlice(*destination_relative_intersection)]
numpy.maximum(block_max_projection, destination_subview, out=destination_subview)
# Invert the nonzero pixels so increasing colors correspond to increasing slices.
# See comment in calc_color_value(), above.
destination_subview[:] = numpy.where(destination_subview,
numpy.float32(1.0) - destination_subview,
numpy.float32(0.0))
return
def read(self, roi, result_out):
"""
roi: (start, stop) tuples, ordered according to description.output_axes
"""
output_axes = self.description.output_axes
roi_transposed = zip(*roi)
roi_dict = dict( zip(output_axes, roi_transposed) )
roi = zip( *(roi_dict['z'], roi_dict['y'], roi_dict['x']) )
# First, normalize roi and result to zyx order
result_out = vigra.taggedView(result_out, output_axes)
result_out = result_out.withAxes(*'zyx')
assert numpy.array(roi).shape == (2,3), "Invalid roi for 3D volume: {}".format( roi )
roi = numpy.array(roi)
assert (result_out.shape == (roi[1] - roi[0])).all()
tile_blockshape = (1,) + tuple(self.description.tile_shape_2d_yx)
tile_starts = getIntersectingBlocks( tile_blockshape, roi )
# We use a fresh tmp dir for each read to avoid conflicts between parallel reads
tmpdir = tempfile.mkdtemp()
pool = RequestPool()
for tile_start in tile_starts:
tile_roi_in = getBlockBounds( self.description.shape_zyx, tile_blockshape, tile_start )
tile_roi_in = numpy.array(tile_roi_in)
# This tile's portion of the roi
intersecting_roi = getIntersection( roi, tile_roi_in )
intersecting_roi = numpy.array( intersecting_roi )
# Compute slicing within destination array and slicing within this tile
destination_relative_intersection = numpy.subtract(intersecting_roi, roi[0])
tile_relative_intersection = intersecting_roi - tile_roi_in[0]
# Get a view to the output slice
result_region = result_out[roiToSlice(*destination_relative_intersection)]
# Special feature:
# Some slices are missing, in which case we provide fake data from a different slice.
# Overwrite the rest args to pull data from an alternate source tile.
z_start = tile_roi_in[0][0]
if z_start in self._slice_remapping:
new_source_slice = self._slice_remapping[z_start]
tile_roi_in[0][0] = new_source_slice
tile_roi_in[1][0] = new_source_slice+1
tile_index = numpy.array(tile_roi_in[0]) / tile_blockshape
rest_args = { 'z_start' : tile_roi_in[0][0],
'z_stop' : tile_roi_in[1][0],
'y_start' : tile_roi_in[0][1],
'y_stop' : tile_roi_in[1][1],
'x_start' : tile_roi_in[0][2],
'x_stop' : tile_roi_in[1][2],
'z_index' : tile_index[0],
'y_index' : tile_index[1],
'x_index' : tile_index[2] }
# Quick sanity check
assert rest_args['z_index'] == rest_args['z_start']
retrieval_fn = partial( self._retrieve_tile, tmpdir, rest_args, tile_relative_intersection, result_region )
PARALLEL_REQ = True
if PARALLEL_REQ:
pool.add( Request( retrieval_fn ) )
else:
# execute serially (leave the pool empty)
retrieval_fn()
pool.wait()
# Clean up our temp files.
shutil.rmtree(tmpdir)
def read(self, view_roi, result_out):
"""
roi: (start, stop) tuples, ordered according to description.output_axes
roi should be relative to the view
"""
output_axes = self.description.output_axes
roi_transposed = zip(*view_roi)
roi_dict = dict( zip(output_axes, roi_transposed) )
view_roi = zip( *(roi_dict['z'], roi_dict['y'], roi_dict['x']) )
# First, normalize roi and result to zyx order
result_out = vigra.taggedView(result_out, output_axes)
result_out = result_out.withAxes(*'zyx')
assert numpy.array(view_roi).shape == (2,3), "Invalid roi for 3D volume: {}".format( view_roi )
view_roi = numpy.array(view_roi)
assert (result_out.shape == (view_roi[1] - view_roi[0])).all()
# User gave roi according to the view output.
# Now offset it find global roi.
roi = view_roi + self.description.view_origin_zyx
tile_blockshape = (1,) + tuple(self.description.tile_shape_2d_yx)
tile_starts = getIntersectingBlocks( tile_blockshape, roi )
pool = RequestPool()
for tile_start in tile_starts:
tile_roi_in = getBlockBounds( self.description.bounds_zyx, tile_blockshape, tile_start )
tile_roi_in = numpy.array(tile_roi_in)
# This tile's portion of the roi
intersecting_roi = getIntersection( roi, tile_roi_in )
intersecting_roi = numpy.array( intersecting_roi )
# Compute slicing within destination array and slicing within this tile
destination_relative_intersection = numpy.subtract(intersecting_roi, roi[0])
tile_relative_intersection = intersecting_roi - tile_roi_in[0]
# Get a view to the output slice
result_region = result_out[roiToSlice(*destination_relative_intersection)]
# Special feature:
# Some slices are missing, in which case we provide fake data from a different slice.
# Overwrite the rest args to pull data from an alternate source tile.
z_start = tile_roi_in[0][0]
if z_start in self._slice_remapping:
new_source_slice = self._slice_remapping[z_start]
tile_roi_in[0][0] = new_source_slice
tile_roi_in[1][0] = new_source_slice+1
tile_index = numpy.array(tile_roi_in[0]) / tile_blockshape
rest_args = { 'z_start' : tile_roi_in[0][0],
'z_stop' : tile_roi_in[1][0],
'y_start' : tile_roi_in[0][1],
'y_stop' : tile_roi_in[1][1],
'x_start' : tile_roi_in[0][2],
'x_stop' : tile_roi_in[1][2],
'z_index' : tile_index[0],
'y_index' : tile_index[1],
'x_index' : tile_index[2] }
# Apply special z_translation_function
if self.description.z_translation_function is not None:
z_update_func = eval(self.description.z_translation_function)
rest_args['z_index'] = rest_args['z_start'] = z_update_func(rest_args['z_index'])
rest_args['z_stop'] = 1 + rest_args['z_start']
# Quick sanity check
assert rest_args['z_index'] == rest_args['z_start']
if self.description.tile_url_format.startswith('http'):
retrieval_fn = partial( self._retrieve_remote_tile, rest_args, tile_relative_intersection, result_region )
else:
retrieval_fn = partial( self._retrieve_local_tile, rest_args, tile_relative_intersection, result_region )
PARALLEL_REQ = True
if PARALLEL_REQ:
pool.add( Request( retrieval_fn ) )
else:
# execute serially (leave the pool empty)
retrieval_fn()
if PARALLEL_REQ:
with Timer() as timer:
pool.wait()
logger.info("Loading {} tiles took a total of {}".format( len(tile_starts), timer.seconds() ))
