def _executeOutput(self, roi, destination): assert len(roi.stop) == len(self.Input.meta.shape), "roi: {} has the wrong number of dimensions for Input shape: {}".format( roi, self.Input.meta.shape ) assert numpy.less_equal(roi.stop, self.Input.meta.shape).all(), "roi: {} is out-of-bounds for Input shape: {}".format( roi, self.Input.meta.shape ) block_starts = getIntersectingBlocks( self._blockshape, (roi.start, roi.stop) ) block_starts = map( tuple, block_starts ) # Ensure all block cache files are up-to-date reqPool = RequestPool() # (Do the work in parallel.) for block_start in block_starts: entire_block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, block_start ) f = partial( self._ensureCached, entire_block_roi) reqPool.add( Request(f) ) logger.debug( "Waiting for {} blocks...".format( len(block_starts) ) ) reqPool.wait() # Copy data from each block # (Parallelism not needed here: h5py will serialize these requests anyway) logger.debug( "Copying data from {} blocks...".format( len(block_starts) ) ) for block_start in block_starts: entire_block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, block_start ) # This block's portion of the roi intersecting_roi = getIntersection( (roi.start, roi.stop), entire_block_roi ) # Compute slicing within destination array and slicing within this block destination_relative_intersection = numpy.subtract(intersecting_roi, roi.start) block_relative_intersection = numpy.subtract(intersecting_roi, block_start) # Copy from block to destination dataset = self._getBlockDataset( entire_block_roi ) destination[ roiToSlice(*destination_relative_intersection) ] = dataset[ roiToSlice( *block_relative_intersection ) ] return destination
def _copyData(self, roi, destination, block_starts): # Copy data from each block # (Parallelism not needed here: h5py will serialize these requests anyway) logger.debug( "Copying data from {} blocks...".format( len(block_starts) ) ) for block_start in block_starts: entire_block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, block_start ) # This block's portion of the roi intersecting_roi = getIntersection( (roi.start, roi.stop), entire_block_roi ) # Compute slicing within destination array and slicing within this block destination_relative_intersection = numpy.subtract(intersecting_roi, roi.start) block_relative_intersection = numpy.subtract(intersecting_roi, block_start) destination_relative_intersection_slicing = roiToSlice(*destination_relative_intersection) block_relative_intersection_slicing = roiToSlice( *block_relative_intersection ) # Copy from block to destination dataset = self._getBlockDataset( entire_block_roi ) if self.Output.meta.has_mask: destination.data[ destination_relative_intersection_slicing ] = dataset["data"][ block_relative_intersection_slicing ] destination.mask[ destination_relative_intersection_slicing ] = dataset["mask"][ block_relative_intersection_slicing ] destination.fill_value = dataset["fill_value"][()] else: destination[ destination_relative_intersection_slicing ] = dataset[ block_relative_intersection_slicing ] self._last_access_times[block_start] = time.time()
def ingestData(self, slot): """ Read the data from the given slot and copy it into this cache. The rules about special pixel meanings apply here, just like setInSlot Returns: the max label found in the slot. """ assert self._blockshape is not None assert self.Input.meta.shape == slot.meta.shape max_label = 0 # Get logical blocking. block_starts = getIntersectingBlocks( self._blockshape, roiFromShape(self.Input.meta.shape) ) block_starts = map( tuple, block_starts ) # Write each block for block_start in block_starts: block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, block_start ) # Request the block data block_data = slot(*block_roi).wait() # Write into the array subregion_roi = SubRegion(self.Input, *block_roi) cleaned_block_data = self._setInSlotInput( self.Input, (), subregion_roi, block_data ) max_label = max( max_label, cleaned_block_data.max() ) return max_label
def _setInSlotInput(self, slot, subindex, roi, value): assert len(roi.stop) == len(self.Input.meta.shape), "roi: {} has the wrong number of dimensions for Input shape: {}".format( roi, self.Input.meta.shape ) assert numpy.less_equal(roi.stop, self.Input.meta.shape).all(), "roi: {} is out-of-bounds for Input shape: {}".format( roi, self.Input.meta.shape ) block_starts = getIntersectingBlocks( self._blockshape, (roi.start, roi.stop) ) block_starts = map( tuple, block_starts ) # Copy data to each block logger.debug( "Copying data INTO {} blocks...".format( len(block_starts) ) ) for block_start in block_starts: entire_block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, block_start ) # This block's portion of the roi intersecting_roi = getIntersection( (roi.start, roi.stop), entire_block_roi ) # Compute slicing within source array and slicing within this block source_relative_intersection = numpy.subtract(intersecting_roi, roi.start) block_relative_intersection = numpy.subtract(intersecting_roi, block_start) # Copy from source to block dataset = self._getBlockDataset( entire_block_roi ) dataset[ roiToSlice( *block_relative_intersection ) ] = value[ roiToSlice(*source_relative_intersection) ] # Here, we assume that if this function is used to update ANY PART of a # block, he is responsible for updating the ENTIRE block. # Therefore, this block is no longer 'dirty' self._dirtyBlocks.discard( block_start )
def _copyData(self, roi, destination, block_starts): """ Copy data from each block into the destination array. For blocks that aren't currently stored, just write zeros. """ # (Parallelism not needed here: h5py will serialize these requests anyway) block_starts = map( tuple, block_starts ) for block_start in block_starts: entire_block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, block_start ) # This block's portion of the roi intersecting_roi = getIntersection( (roi.start, roi.stop), entire_block_roi ) # Compute slicing within destination array and slicing within this block destination_relative_intersection = numpy.subtract(intersecting_roi, roi.start) block_relative_intersection = numpy.subtract(intersecting_roi, block_start) destination_relative_intersection_slicing = roiToSlice(*destination_relative_intersection) block_relative_intersection_slicing = roiToSlice(*block_relative_intersection) if block_start in self._cacheFiles: # Copy from block to destination dataset = self._getBlockDataset( entire_block_roi ) if self.Output.meta.has_mask: destination[ destination_relative_intersection_slicing ] = dataset["data"][ block_relative_intersection_slicing ] destination.mask[ destination_relative_intersection_slicing ] = dataset["mask"][ block_relative_intersection_slicing ] destination.fill_value = dataset["fill_value"][()] else: destination[ destination_relative_intersection_slicing ] = dataset[ block_relative_intersection_slicing ] else: # Not stored yet. Overwrite with zeros. destination[ destination_relative_intersection_slicing ] = 0
def _ensurePipelineExists(self, block_start): if block_start in self._blockPipelines: return with self._lock: if block_start in self._blockPipelines: return logger.debug( "Creating pipeline for block: {}".format( block_start ) ) block_shape = self._getFullShape( self._block_shape_dict ) halo_padding = self._getFullShape( self._halo_padding_dict ) input_shape = self.RawImage.meta.shape block_stop = getBlockBounds( input_shape, block_shape, block_start )[1] block_roi = (block_start, block_stop) # Instantiate pipeline opBlockPipeline = OpSingleBlockObjectPrediction( block_roi, halo_padding, parent=self ) opBlockPipeline.RawImage.connect( self.RawImage ) opBlockPipeline.BinaryImage.connect( self.BinaryImage ) opBlockPipeline.Classifier.connect( self.Classifier ) opBlockPipeline.LabelsCount.connect( self.LabelsCount ) # Forward dirtyness opBlockPipeline.PredictionImage.notifyDirty( bind(self._handleDirtyBlock, block_start ) ) self._blockPipelines[block_start] = opBlockPipeline
def testFailedProcessing(self): op = OpArrayPiper( graph=Graph() ) inputData = numpy.indices( (100,100) ).sum(0) op.Input.setValue( inputData ) roiList = [] block_starts = getIntersectingBlocks( [10,10], ([0,0], [100, 100]) ) for block_start in block_starts: roiList.append( getBlockBounds( [100,100], [10,10], block_start ) ) class SpecialException(Exception): pass def handleResult(roi, result): raise SpecialException("Intentional Exception: raised while handling the result") totalVolume = numpy.prod( inputData.shape ) batch = RoiRequestBatch(op.Output, roiList.__iter__(), totalVolume, batchSize=10, allowParallelResults=False) batch.resultSignal.subscribe( handleResult ) # FIXME: There are multiple places where the RoiRequestBatch tool should be prepared to handle exceptions. # This only tests one of them (in the notify_finished() handler) try: batch.execute() except SpecialException: pass else: assert False, "Expected exception to be propagated out of the RoiRequestBatch."
def _getBlockFileName(self, block_start): # Translate to find disk block start block_start = numpy.add( self.description.view_origin, block_start ) # Get true (disk) block bounds (i.e. use on-disk shape, not view_shape) entire_block_roi = getBlockBounds( self.description.shape, self.description.block_shape, block_start ) roiString = "{}".format( (list(entire_block_roi[0]), list(entire_block_roi[1]) ) ) datasetFilename = self.description.block_file_name_format.format( roiString=roiString ) return datasetFilename
def read(self, view_roi, result_out): """ roi: (start, stop) tuples, ordered according to description.output_axes roi should be relative to the view """ output_axes = self.description.output_axes roi_transposed = zip(*view_roi) roi_dict = dict( zip(output_axes, roi_transposed) ) view_roi = zip( *(roi_dict['z'], roi_dict['y'], roi_dict['x']) ) # First, normalize roi and result to zyx order result_out = vigra.taggedView(result_out, output_axes) result_out = result_out.withAxes(*'zyx') assert numpy.array(view_roi).shape == (2,3), "Invalid roi for 3D volume: {}".format( view_roi ) view_roi = numpy.array(view_roi) assert (result_out.shape == (view_roi[1] - view_roi[0])).all() # User gave roi according to the view output. # Now offset it find global roi. roi = view_roi + self.description.view_origin_zyx tile_blockshape = (1,) + tuple(self.description.tile_shape_2d_yx) tile_starts = getIntersectingBlocks( tile_blockshape, roi ) pool = RequestPool() for tile_start in tile_starts: tile_roi_in = getBlockBounds( self.description.bounds_zyx, tile_blockshape, tile_start ) tile_roi_in = numpy.array(tile_roi_in) # This tile's portion of the roi intersecting_roi = getIntersection( roi, tile_roi_in ) intersecting_roi = numpy.array( intersecting_roi ) # Compute slicing within destination array and slicing within this tile destination_relative_intersection = numpy.subtract(intersecting_roi, roi[0]) tile_relative_intersection = intersecting_roi - tile_roi_in[0] # Get a view to the output slice result_region = result_out[roiToSlice(*destination_relative_intersection)] rest_args = self._get_rest_args(tile_blockshape, tile_roi_in) if self.description.tile_url_format.startswith('http'): retrieval_fn = partial( self._retrieve_remote_tile, rest_args, tile_relative_intersection, result_region ) else: retrieval_fn = partial( self._retrieve_local_tile, rest_args, tile_relative_intersection, result_region ) PARALLEL_REQ = True if PARALLEL_REQ: pool.add( Request( retrieval_fn ) ) else: # execute serially (leave the pool empty) retrieval_fn() if PARALLEL_REQ: with Timer() as timer: pool.wait() logger.info("Loading {} tiles took a total of {}".format( len(tile_starts), timer.seconds() ))
def roiGen(): block_iter = block_starts.__iter__() while True: block_start = block_iter.next() block_bounds = getBlockBounds( outputSlot.meta.shape, blockshape, block_start ) block_intersecting_portion = getIntersection( block_bounds, roi ) logger.debug( "Requesting Roi: {}".format( block_bounds ) ) yield block_intersecting_portion
def _setInSlotInputHdf5(self, slot, subindex, roi, value): logger.debug("Setting block {} from hdf5".format(roi)) if self.Output.meta.has_mask: assert isinstance( value, h5py.Group ), "InputHdf5 slot requires an hdf5 Group to copy from (not a numpy masked array)." else: assert isinstance( value, h5py.Dataset ), "InputHdf5 slot requires an hdf5 Dataset to copy from (not a numpy array)." block_roi = getBlockBounds(self.Output.meta.shape, self._blockshape, roi.start) roi_is_exactly_one_block = True roi_is_exactly_one_block &= ((roi.start % self._blockshape) == 0).all() roi_is_exactly_one_block &= (block_roi == numpy.array((roi.start, roi.stop))).all() if roi_is_exactly_one_block: cachefile = self._getCacheFile(block_roi) logger.debug("Copying HDF5 data directly into block {}".format(block_roi)) if self.Output.meta.has_mask: assert len(value) == 3 for each in ["data", "mask", "fill_value"]: assert each in value assert cachefile[each].dtype == value[each].dtype assert cachefile[each].shape == value[each].shape for each in ["data", "mask", "fill_value"]: del cachefile[each] cachefile.copy(value[each], each) else: assert cachefile["data"].dtype == value.dtype assert cachefile["data"].shape == value.shape del cachefile["data"] cachefile.copy(value, "data") block_start = tuple(roi.start) self._dirtyBlocks.discard(block_start) else: # This hdf5 data does not correspond to exactly one block. # We must uncompress it and write it the "normal" way (the slow way) # FIXME: This would use less memory if we uncompressed the data block-by-block data = None if self.Output.meta.has_mask: data = numpy.ma.masked_array( value["data"][()], mask=value["mask"][()], fill_value=value["fill_value"][()], shrink=False ) else: data = value[()] self.Input[roiToSlice(roi.start, roi.stop)] = data
def _waitForBlocks(self, block_starts): """ Make sure that all blocks in the given list of blocks are present in the cache before returning. (Blocks that are not yet present will be requested from our Input slot.) """ reqPool = RequestPool() # (Do the work in parallel.) for block_start in block_starts: entire_block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, block_start ) f = partial( self._ensureCached, entire_block_roi) reqPool.add( Request(f) ) logger.debug( "Waiting for {} blocks...".format( len(block_starts) ) ) reqPool.wait()
def _executeOutputHdf5(self, roi, destination): logger.debug("Servicing request for hdf5 block {}".format( roi )) assert isinstance( destination, h5py.Group ), "OutputHdf5 slot requires an hdf5 GROUP to copy into (not a numpy array)." assert ((roi.start % self._blockshape) == 0).all(), "OutputHdf5 slot requires roi to be exactly one block." block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, roi.start ) assert (block_roi == numpy.array((roi.start, roi.stop))).all(), "OutputHdf5 slot requires roi to be exactly one block." block_roi = [roi.start, roi.stop] self._ensureCached( block_roi ) dataset = self._getBlockDataset( block_roi ) assert str(block_roi) not in destination, "destination hdf5 group already has a dataset with this block's name" destination.copy( dataset, str(block_roi) ) return destination
def roiGen(): block_iter = block_starts.__iter__() while True: try: block_start = next(block_iter) except StopIteration: # As of Python 3.7, not allowed to let StopIteration exceptions escape a generator # https://www.python.org/dev/peps/pep-0479 break else: block_bounds = getBlockBounds(outputSlot.meta.shape, blockshape, block_start) block_intersecting_portion = getIntersection(block_bounds, roi) logger.debug("Requesting Roi: {}".format(block_bounds)) yield block_intersecting_portion
def roiGen(): block_iter = self._minBlockStarts.__iter__() while True: block_start = block_iter.next() # Use offset blocking offset_block_start = block_start - self._bigRoi[0] offset_data_shape = numpy.subtract(self._bigRoi[1], self._bigRoi[0]) offset_block_bounds = getBlockBounds( offset_data_shape, minBlockShape, offset_block_start ) # Un-offset block_bounds = ( offset_block_bounds[0] + self._bigRoi[0], offset_block_bounds[1] + self._bigRoi[0] ) logger.debug( "Requesting Roi: {}".format( block_bounds ) ) yield block_bounds
def _get_features_for_block(self, block_start): """ Computes the feature matrix for the given block IFF the block is dirty. Otherwise, returns None. """ # Caller must ensure that the lock for this block already exists! with self._block_locks[block_start]: if block_start not in self._dirty_blocks: # Nothing to do if this block isn't actually dirty # (For parallel requests, its theoretically possible.) return None block_roi = getBlockBounds( self.LabelImage.meta.shape, self._blockshape, block_start ) # TODO: Shrink the requested roi using the nonzero blocks slot... # ...or just get rid of the nonzero blocks slot... labels_and_features_matrix = self._extract_feature_matrix(block_roi) return labels_and_features_matrix
def _setInSlotInputHdf5(self, slot, subindex, roi, value): logger.debug("Setting block {} from hdf5".format( roi )) assert isinstance( value, h5py.Dataset ), "InputHdf5 slot requires an hdf5 Dataset to copy from (not a numpy array)." assert ((roi.start % self._blockshape) == 0).all(), "InputHdf5 slot requires roi to be exactly one block." block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, roi.start ) assert (block_roi == numpy.array((roi.start, roi.stop))).all(), "InputHdf5 slot requires roi to be exactly one block." cachefile = self._getCacheFile( block_roi ) logger.debug( "Copying HDF5 data directly into block {}".format( block_roi ) ) assert cachefile['data'].dtype == value.dtype assert cachefile['data'].shape == value.shape del cachefile['data'] cachefile.copy( value, 'data' ) block_start = tuple(roi.start) self._dirtyBlocks.discard( block_start )
def _copyData(self, roi, destination, block_starts): # Copy data from each block # (Parallelism not needed here: h5py will serialize these requests anyway) logger.debug( "Copying data from {} blocks...".format( len(block_starts) ) ) for block_start in block_starts: entire_block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, block_start ) # This block's portion of the roi intersecting_roi = getIntersection( (roi.start, roi.stop), entire_block_roi ) # Compute slicing within destination array and slicing within this block destination_relative_intersection = numpy.subtract(intersecting_roi, roi.start) block_relative_intersection = numpy.subtract(intersecting_roi, block_start) # Copy from block to destination dataset = self._getBlockDataset( entire_block_roi ) destination[ roiToSlice(*destination_relative_intersection) ] = dataset[ roiToSlice( *block_relative_intersection ) ]
def _get_features_for_block(self, block_start): """ Computes the feature matrix for the given block IFF the block is dirty. Otherwise, returns None. """ # Caller must ensure that the lock for this block already exists! with self._block_locks[block_start]: if block_start not in self._dirty_blocks: # Nothing to do if this block isn't actually dirty # (For parallel requests, its theoretically possible.) return None block_roi = getBlockBounds(self.LabelImage.meta.shape, self._blockshape, block_start) # TODO: Shrink the requested roi using the nonzero blocks slot... # ...or just get rid of the nonzero blocks slot... labels_and_features_matrix = self._extract_feature_matrix( block_roi) return labels_and_features_matrix
def export_to_tiles(volume, tile_size, output_dir, print_progress=True): """ volume: The volume to export (either hdf5 dataset or numpy array). Must be 3D. tile_size: The width of the tiles to generate output_dir: The directory to dump the tiles to. """ assert len(volume.shape) == 3 tile_blockshape = (1, tile_size, tile_size) tile_starts = getIntersectingBlocks(tile_blockshape, roiFromShape(volume.shape)) if not os.path.exists(output_dir): os.makedirs(output_dir) logger.info("Writing {} tiles ...".format(len(tile_starts))) for tile_start in tile_starts: tile_roi = getBlockBounds(volume.shape, tile_blockshape, tile_start) if print_progress: sys.stdout.write("Tile: {} ".format(tile_roi)) sys.stdout.flush() tile_data = volume[roiToSlice(*tile_roi)] tile_data = vigra.taggedView(tile_data, 'zyx') if print_progress: sys.stdout.write('reading... ') sys.stdout.flush() tile_name = 'tile_z{:05}_y{:05}_x{:05}.png'.format(*tile_start) output_path = os.path.join(output_dir, tile_name) if print_progress: sys.stdout.write('writing... ') sys.stdout.flush() vigra.impex.writeImage(tile_data[0], output_path, dtype='NATIVE') if print_progress: sys.stdout.write('done.\n') sys.stdout.flush() logger.info("TILES COMPLETE.")
def testBasic(self): op = OpArrayPiper( graph=Graph() ) inputData = numpy.indices( (100,100) ).sum(0) op.Input.setValue( inputData ) roiList = [] block_starts = getIntersectingBlocks( [10,10], ([0,0], [100, 100]) ) for block_start in block_starts: roiList.append( getBlockBounds( [100,100], [10,10], block_start ) ) results = numpy.zeros( (100,100), dtype=numpy.int32 ) resultslock = threading.Lock() resultsCount = [0] def handleResult(roi, result): acquired = resultslock.acquire(False) assert acquired, "resultslock is contested! Access to callback is supposed to be automatically serialized." results[ roiToSlice( *roi ) ] = result logger.debug( "Got result for {}".format(roi) ) resultslock.release() resultsCount[0] += 1 progressList = [] def handleProgress( progress ): progressList.append( progress ) logger.debug( "Progress update: {}".format(progress) ) totalVolume = numpy.prod( inputData.shape ) batch = RoiRequestBatch(op.Output, roiList.__iter__(), totalVolume, batchSize=10) batch.resultSignal.subscribe( handleResult ) batch.progressSignal.subscribe( handleProgress ) batch.execute() logger.debug( "Got {} results".format( resultsCount[0] ) ) assert (results == inputData).all() # Progress reporting MUST start with 0 and end with 100 assert progressList[0] == 0, "Invalid progress reporting." assert progressList[-1] == 100, "Invalid progress reporting." # There should be some intermediate progress reporting, but exactly how much is unspecified. assert len(progressList) >= 10 logger.debug( "FINISHED" )
def autoSeedBackground(cls, laneView, foreground_label): # Seed the entire image with background labels, except for the individual label in question # To save memory, we'll do this in blocks instead of all at once volume_shape = laneView.RavelerLabels.meta.shape volume_roi = roiFromShape( volume_shape ) block_shape = (OpSplitBodyCarving.BLOCK_SIZE,) * len( volume_shape ) block_shape = numpy.minimum( block_shape, volume_shape ) block_starts = getIntersectingBlocks( block_shape, volume_roi ) logger.debug("Auto-seeding {} blocks for label".format( len(block_starts), foreground_label )) for block_index, block_start in enumerate(block_starts): block_roi = getBlockBounds( volume_shape, block_shape, block_start ) label_block = laneView.RavelerLabels(*block_roi).wait() background_block = numpy.where( label_block == foreground_label, 0, 1 ) background_block = numpy.asarray( background_block, numpy.float32 ) # Distance transform requires float if (background_block == 0.0).any(): # We need to leave a small border between the background seeds and the object membranes background_block_view = background_block.view( vigra.VigraArray ) background_block_view.axistags = copy.copy( laneView.RavelerLabels.meta.axistags ) background_block_view_4d = background_block_view.bindAxis('t', 0) background_block_view_3d = background_block_view_4d.bindAxis('c', 0) distance_transformed_block = vigra.filters.distanceTransform3D(background_block_view_3d, background=False) distance_transformed_block = distance_transformed_block.astype( numpy.uint8 ) # Create a 'hull' surrounding the foreground, but leave some space. background_seed_block = (distance_transformed_block == OpSplitBodyCarving.SEED_MARGIN) background_seed_block = background_seed_block.astype(numpy.uint8) * 1 # (In carving, background is label 1) # # Make the hull VERY sparse to avoid over-biasing graph cut toward the background class # # FIXME: Don't regenerate this random block on every loop iteration # rand_bytes = numpy.random.randint(0, 1000, background_seed_block.shape) # background_seed_block = numpy.where( rand_bytes < 1, background_seed_block, 0 ) # background_seed_block = background_seed_block.view(vigra.VigraArray) # background_seed_block.axistags = background_block_view_3d.axistags axisorder = laneView.RavelerLabels.meta.getTaggedShape().keys() logger.debug("Writing backgound seeds: {}/{}".format( block_index, len(block_starts) )) laneView.WriteSeeds[ roiToSlice( *block_roi ) ] = background_seed_block.withAxes(*axisorder) else: logger.debug("Skipping all-background block: {}/{}".format( block_index, len(block_starts) ))
def _copyData(self, roi, destination, block_starts): """ Copy data from each block into the destination array. For blocks that aren't currently stored, just write zeros. """ # (Parallelism not needed here: h5py will serialize these requests anyway) block_starts = map(tuple, block_starts) for block_start in block_starts: entire_block_roi = getBlockBounds(self.Output.meta.shape, self._blockshape, block_start) # This block's portion of the roi intersecting_roi = getIntersection((roi.start, roi.stop), entire_block_roi) # Compute slicing within destination array and slicing within this block destination_relative_intersection = numpy.subtract( intersecting_roi, roi.start) block_relative_intersection = numpy.subtract( intersecting_roi, block_start) destination_relative_intersection_slicing = roiToSlice( *destination_relative_intersection) block_relative_intersection_slicing = roiToSlice( *block_relative_intersection) if block_start in self._cacheFiles: # Copy from block to destination dataset = self._getBlockDataset(entire_block_roi) if self.Output.meta.has_mask: destination[ destination_relative_intersection_slicing] = dataset[ "data"][block_relative_intersection_slicing] destination.mask[ destination_relative_intersection_slicing] = dataset[ "mask"][block_relative_intersection_slicing] destination.fill_value = dataset["fill_value"][()] else: destination[ destination_relative_intersection_slicing] = dataset[ block_relative_intersection_slicing] else: # Not stored yet. Overwrite with zeros. destination[destination_relative_intersection_slicing] = 0
def export_to_tiles(volume, tile_size, output_dir, print_progress=True): """ volume: The volume to export (either hdf5 dataset or numpy array). Must be 3D. tile_size: The width of the tiles to generate output_dir: The directory to dump the tiles to. """ assert len(volume.shape) == 3 tile_blockshape = (1, tile_size, tile_size) tile_starts = getIntersectingBlocks( tile_blockshape, roiFromShape( volume.shape ) ) if not os.path.exists(output_dir): os.makedirs(output_dir) logger.info("Writing {} tiles ...".format( len(tile_starts) ) ) for tile_start in tile_starts: tile_roi = getBlockBounds( volume.shape, tile_blockshape, tile_start ) if print_progress: sys.stdout.write("Tile: {} ".format( tile_roi )) sys.stdout.flush() tile_data = volume[ roiToSlice(*tile_roi) ] tile_data = vigra.taggedView(tile_data, 'zyx') if print_progress: sys.stdout.write('reading... ') sys.stdout.flush() tile_name = 'tile_z{:05}_y{:05}_x{:05}.png'.format( *tile_start ) output_path = os.path.join( output_dir, tile_name ) if print_progress: sys.stdout.write('writing... ') sys.stdout.flush() vigra.impex.writeImage( tile_data[0], output_path, dtype='NATIVE' ) if print_progress: sys.stdout.write('done.\n') sys.stdout.flush() logger.info( "TILES COMPLETE." )
def _setInSlotInput(self, slot, subindex, roi, value, store_zero_blocks=True): """ Write the data in the array 'value' into the cache. If the optional store_zero_blocks param is False, then don't bother creating cache blocks for blocks that are totally zero. """ assert len(roi.stop) == len(self.Input.meta.shape), \ "roi: {} has the wrong number of dimensions for Input shape: {}"\ "".format( roi, self.Input.meta.shape ) assert numpy.less_equal(roi.stop, self.Input.meta.shape).all(), \ "roi: {} is out-of-bounds for Input shape: {}"\ "".format( roi, self.Input.meta.shape ) block_starts = getIntersectingBlocks( self._blockshape, (roi.start, roi.stop) ) block_starts = map( tuple, block_starts ) # Copy data to each block logger.debug( "Copying data INTO {} blocks...".format( len(block_starts) ) ) for block_start in block_starts: entire_block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, block_start ) # This block's portion of the roi intersecting_roi = getIntersection( (roi.start, roi.stop), entire_block_roi ) # Compute slicing within source array and slicing within this block source_relative_intersection = numpy.subtract(intersecting_roi, roi.start) block_relative_intersection = numpy.subtract(intersecting_roi, block_start) new_block_data = value[ roiToSlice(*source_relative_intersection) ] if not store_zero_blocks and new_block_data.sum() == 0 and block_start not in self._cacheFiles: # Special fast-path: If this block doesn't exist yet, # don't bother creating if we're just going to fill it with zeros # (Used by the OpCompressedUserLabelArray) pass else: # Copy from source to block dataset = self._getBlockDataset( entire_block_roi ) dataset[ roiToSlice( *block_relative_intersection ) ] = new_block_data # Here, we assume that if this function is used to update ANY PART of a # block, he is responsible for updating the ENTIRE block. # Therefore, this block is no longer 'dirty' self._dirtyBlocks.discard( block_start )
def _executeOutputHdf5(self, roi, destination): logger.debug("Servicing request for hdf5 block {}".format(roi)) assert isinstance( destination, h5py.Group ), "OutputHdf5 slot requires an hdf5 GROUP to copy into (not a numpy array)." assert ((roi.start % self._blockshape) == 0).all( ), "OutputHdf5 slot requires roi to be exactly one block." block_roi = getBlockBounds(self.Output.meta.shape, self._blockshape, roi.start) assert (block_roi == numpy.array( (roi.start, roi.stop ))).all(), "OutputHdf5 slot requires roi to be exactly one block." block_roi = [roi.start, roi.stop] self._ensureCached(block_roi) dataset = self._getBlockDataset(block_roi) assert str( block_roi ) not in destination, "destination hdf5 group already has a dataset with this block's name" destination.copy(dataset, str(block_roi)) return destination
def roiGen(): block_iter = block_starts.__iter__() while True: try: block_start = next(block_iter) except StopIteration: # As of Python 3.7, not allowed to let StopIteration exceptions escape a generator # https://www.python.org/dev/peps/pep-0479 break else: # Use offset blocking offset_block_start = block_start - self._bigRoi[0] offset_data_shape = numpy.subtract(self._bigRoi[1], self._bigRoi[0]) offset_block_bounds = getBlockBounds(offset_data_shape, blockshape, offset_block_start) # Un-offset block_bounds = ( offset_block_bounds[0] + self._bigRoi[0], offset_block_bounds[1] + self._bigRoi[0], ) logger.debug("Requesting Roi: {}".format(block_bounds)) yield block_bounds
def _copyData(self, roi, destination, block_starts): # Copy data from each block # (Parallelism not needed here: h5py will serialize these requests anyway) logger.debug("Copying data from {} blocks...".format( len(block_starts))) for block_start in block_starts: entire_block_roi = getBlockBounds(self.Output.meta.shape, self._blockshape, block_start) # This block's portion of the roi intersecting_roi = getIntersection((roi.start, roi.stop), entire_block_roi) # Compute slicing within destination array and slicing within this block destination_relative_intersection = numpy.subtract( intersecting_roi, roi.start) block_relative_intersection = numpy.subtract( intersecting_roi, block_start) destination_relative_intersection_slicing = roiToSlice( *destination_relative_intersection) block_relative_intersection_slicing = roiToSlice( *block_relative_intersection) # Copy from block to destination dataset = self._getBlockDataset(entire_block_roi) if self.Output.meta.has_mask: destination.data[ destination_relative_intersection_slicing] = dataset[ "data"][block_relative_intersection_slicing] destination.mask[ destination_relative_intersection_slicing] = dataset[ "mask"][block_relative_intersection_slicing] destination.fill_value = dataset["fill_value"][()] else: destination[ destination_relative_intersection_slicing] = dataset[ block_relative_intersection_slicing] self._last_access_times[block_start] = time.time()
def ingestData(self, slot): """ Read the data from the given slot and copy it into this cache. The rules about special pixel meanings apply here, just like setInSlot Returns: the max label found in the slot. """ assert self._blockshape is not None assert self.Output.meta.shape[: -1] == slot.meta.shape[: -1], "{} != {}".format( self.Output. meta.shape, slot.meta. shape) max_label = 0 # Get logical blocking. block_starts = getIntersectingBlocks( self._blockshape, roiFromShape(self.Output.meta.shape)) block_starts = list(map(tuple, block_starts)) # Write each block for block_start in block_starts: block_roi = getBlockBounds(self.Output.meta.shape, self._blockshape, block_start) # Request the block data block_data = slot(*block_roi).wait() # Write into the array subregion_roi = SubRegion(self.Output, *block_roi) cleaned_block_max = self._setInSlotInput(self.Input, (), subregion_roi, block_data) max_label = max(max_label, cleaned_block_max) return max_label
def _update_block(self, block_start): if block_start not in self._block_locks: with self._lock: if block_start not in self._block_locks: self._block_locks[block_start] = RequestLock() with self._block_locks[block_start]: if block_start not in self._dirty_blocks: # Nothing to do if this block isn't actually dirty # (For parallel requests, its theoretically possible.) return block_roi = getBlockBounds( self.LabelImage.meta.shape, self._blockshape, block_start ) # TODO: Shrink the requested roi using the nonzero blocks slot... # ...or just get rid of the nonzero blocks slot... labels_and_features_matrix = self._extract_feature_matrix(block_roi) with self._lock: self._dirty_blocks.remove(block_start) if labels_and_features_matrix.shape[0] > 0: self._blockwise_feature_matrices[block_start] = labels_and_features_matrix else: try: del self._blockwise_feature_matrices[block_start] except KeyError: pass
def testFailedProcessing(self): op = OpArrayPiper(graph=Graph()) inputData = numpy.indices((100, 100)).sum(0) op.Input.setValue(inputData) roiList = [] block_starts = getIntersectingBlocks([10, 10], ([0, 0], [100, 100])) for block_start in block_starts: roiList.append(getBlockBounds([100, 100], [10, 10], block_start)) class SpecialException(Exception): pass def handleResult(roi, result): raise SpecialException("Intentional Exception: raised while handling the result") totalVolume = numpy.prod(inputData.shape) batch = RoiRequestBatch(op.Output, roiList.__iter__(), totalVolume, batchSize=10, allowParallelResults=False) batch.resultSignal.subscribe(handleResult) # FIXME: There are multiple places where the RoiRequestBatch tool should be prepared to handle exceptions. # This only tests one of them (in the notify_finished() handler) with pytest.raises(SpecialException): batch.execute()
def roiGen(): block_iter = block_starts.__iter__() while True: try: block_start = next(block_iter) except StopIteration: # As of Python 3.7, not allowed to let StopIteration exceptions escape a generator # https://www.python.org/dev/peps/pep-0479 break else: # Use offset blocking offset_block_start = block_start - self._bigRoi[0] offset_data_shape = numpy.subtract( self._bigRoi[1], self._bigRoi[0]) offset_block_bounds = getBlockBounds( offset_data_shape, blockshape, offset_block_start) # Un-offset block_bounds = ( offset_block_bounds[0] + self._bigRoi[0], offset_block_bounds[1] + self._bigRoi[0], ) logger.debug("Requesting Roi: {}".format(block_bounds)) yield block_bounds
def _setInSlotInputHdf5(self, slot, subindex, roi, value): logger.debug("Setting block {} from hdf5".format( roi )) assert isinstance( value, h5py.Dataset ), "InputHdf5 slot requires an hdf5 Dataset to copy from (not a numpy array)." block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, roi.start ) roi_is_exactly_one_block = True roi_is_exactly_one_block &= ((roi.start % self._blockshape) == 0).all() roi_is_exactly_one_block &= (block_roi == numpy.array((roi.start, roi.stop))).all() if roi_is_exactly_one_block: cachefile = self._getCacheFile( block_roi ) logger.debug( "Copying HDF5 data directly into block {}".format( block_roi ) ) assert cachefile['data'].dtype == value.dtype assert cachefile['data'].shape == value.shape del cachefile['data'] cachefile.copy( value, 'data' ) block_start = tuple(roi.start) self._dirtyBlocks.discard( block_start ) else: # This hdf5 data does not correspond to exactly one block. # We must uncompress it and write it the "normal" way (the slow way) # FIXME: This would use less memory if we uncompressed the data block-by-block self.Input[roiToSlice(roi.start, roi.stop)] = value[:]
def _copyData(self, roi, destination, block_starts): # Copy data from each block # (Parallelism not needed here: h5py will serialize these requests anyway) logger.debug("Copying data from {} blocks...".format( len(block_starts))) for block_start in block_starts: entire_block_roi = getBlockBounds(self.Output.meta.shape, self._blockshape, block_start) # This block's portion of the roi intersecting_roi = getIntersection((roi.start, roi.stop), entire_block_roi) # Compute slicing within destination array and slicing within this block destination_relative_intersection = numpy.subtract( intersecting_roi, roi.start) block_relative_intersection = numpy.subtract( intersecting_roi, block_start) # Copy from block to destination dataset = self._getBlockDataset(entire_block_roi) destination[roiToSlice( *destination_relative_intersection)] = dataset[roiToSlice( *block_relative_intersection)]
def get_block_roi(self, block_start): block_shape = self._getFullShape(self._block_shape_dict) input_shape = self.RawImage.meta.shape block_stop = getBlockBounds(input_shape, block_shape, block_start)[1] block_roi = (block_start, block_stop) return block_roi
def _setInSlotInput(self, slot, subindex, roi, value, store_zero_blocks=True): """ Write the data in the array 'value' into the cache. If the optional store_zero_blocks param is False, then don't bother creating cache blocks for blocks that are totally zero. """ assert len(roi.stop) == len(self.Input.meta.shape), \ "roi: {} has the wrong number of dimensions for Input shape: {}"\ "".format( roi, self.Input.meta.shape ) assert numpy.less_equal(roi.stop, self.Input.meta.shape).all(), \ "roi: {} is out-of-bounds for Input shape: {}"\ "".format( roi, self.Input.meta.shape ) block_starts = getIntersectingBlocks( self._blockshape, (roi.start, roi.stop) ) block_starts = map( tuple, block_starts ) # Copy data to each block logger.debug( "Copying data INTO {} blocks...".format( len(block_starts) ) ) for block_start in block_starts: entire_block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, block_start ) # This block's portion of the roi intersecting_roi = getIntersection( (roi.start, roi.stop), entire_block_roi ) # Compute slicing within source array and slicing within this block source_relative_intersection = numpy.subtract(intersecting_roi, roi.start) block_relative_intersection = numpy.subtract(intersecting_roi, block_start) source_relative_intersection_slicing = roiToSlice( *source_relative_intersection ) block_relative_intersection_slicing = roiToSlice( *block_relative_intersection ) new_block_data = value[ source_relative_intersection_slicing ] new_block_sum = new_block_data.sum() if not store_zero_blocks and new_block_sum == 0 and block_start not in self._cacheFiles: # Special fast-path: If this block doesn't exist yet, # don't bother creating if we're just going to fill it with zeros. # (This feature is used by the OpCompressedUserLabelArray) pass else: # Copy from source to block dataset = self._getBlockDataset( entire_block_roi ) if self.Output.meta.has_mask: dataset["data"][ block_relative_intersection_slicing ] = new_block_data.data dataset["mask"][ block_relative_intersection_slicing ] = new_block_data.mask dataset["fill_value"][()] = new_block_data.fill_value # Untested. Write a test to use this. # # If we can, remove this block entirely. # if not store_zero_blocks and new_block_sum == 0 and (dataset["data"][:] == 0).all() and (dataset["mask"]).any() and (dataset["fill_value"] == 0).all(): # with self._lock: # with self._blockLocks[block_start]: # self._cacheFiles[block_start].close() # del self._cacheFiles[block_start] # del self._blockLocks[block_start] else: dataset[ block_relative_intersection_slicing ] = new_block_data # If we can, remove this block entirely. if not store_zero_blocks and new_block_sum == 0 and (dataset[:] == 0).all(): with self._lock: with self._blockLocks[block_start]: self._cacheFiles[block_start].close() del self._cacheFiles[block_start] del self._blockLocks[block_start] # Here, we assume that if this function is used to update ANY PART of a # block, he is responsible for updating the ENTIRE block. # Therefore, this block is no longer 'dirty' self._dirtyBlocks.discard( block_start )
def getEntireBlockRoi(self, block_start): """ Return the roi for the entire block that starts at the given coordinate. """ return getBlockBounds(self._description.view_shape, self._description.block_shape, block_start)
def read(self, view_roi, result_out): """ roi: (start, stop) tuples, ordered according to description.output_axes roi should be relative to the view """ output_axes = self.description.output_axes roi_transposed = zip(*view_roi) roi_dict = dict(zip(output_axes, roi_transposed)) view_roi = zip(*(roi_dict['z'], roi_dict['y'], roi_dict['x'])) # First, normalize roi and result to zyx order result_out = vigra.taggedView(result_out, output_axes) result_out = result_out.withAxes(*'zyx') assert numpy.array(view_roi).shape == ( 2, 3), "Invalid roi for 3D volume: {}".format(view_roi) view_roi = numpy.array(view_roi) assert (result_out.shape == (view_roi[1] - view_roi[0])).all() # User gave roi according to the view output. # Now offset it find global roi. roi = view_roi + self.description.view_origin_zyx tile_blockshape = (1, ) + tuple(self.description.tile_shape_2d_yx) tile_starts = getIntersectingBlocks(tile_blockshape, roi) pool = RequestPool() for tile_start in tile_starts: tile_roi_in = getBlockBounds(self.description.bounds_zyx, tile_blockshape, tile_start) tile_roi_in = numpy.array(tile_roi_in) # This tile's portion of the roi intersecting_roi = getIntersection(roi, tile_roi_in) intersecting_roi = numpy.array(intersecting_roi) # Compute slicing within destination array and slicing within this tile destination_relative_intersection = numpy.subtract( intersecting_roi, roi[0]) tile_relative_intersection = intersecting_roi - tile_roi_in[0] # Get a view to the output slice result_region = result_out[roiToSlice( *destination_relative_intersection)] rest_args = self._get_rest_args(tile_blockshape, tile_roi_in) if self.description.tile_url_format.startswith('http'): retrieval_fn = partial(self._retrieve_remote_tile, rest_args, tile_relative_intersection, result_region) else: retrieval_fn = partial(self._retrieve_local_tile, rest_args, tile_relative_intersection, result_region) PARALLEL_REQ = True if PARALLEL_REQ: pool.add(Request(retrieval_fn)) else: # execute serially (leave the pool empty) retrieval_fn() if PARALLEL_REQ: with Timer() as timer: pool.wait() logger.info("Loading {} tiles took a total of {}".format( len(tile_starts), timer.seconds()))
def get_block_roi(self, block_start): block_shape = self._getFullShape( self._block_shape_dict ) input_shape = self.RawImage.meta.shape block_stop = getBlockBounds( input_shape, block_shape, block_start )[1] block_roi = (block_start, block_stop) return block_roi
def read(self, roi, result_out): """ roi: (start, stop) tuples, ordered according to description.output_axes """ output_axes = self.description.output_axes roi_transposed = zip(*roi) roi_dict = dict(zip(output_axes, roi_transposed)) roi = zip(*(roi_dict['z'], roi_dict['y'], roi_dict['x'])) # First, normalize roi and result to zyx order result_out = vigra.taggedView(result_out, output_axes) result_out = result_out.withAxes(*'zyx') assert numpy.array(roi).shape == ( 2, 3), "Invalid roi for 3D volume: {}".format(roi) roi = numpy.array(roi) assert (result_out.shape == (roi[1] - roi[0])).all() tile_blockshape = (1, ) + tuple(self.description.tile_shape_2d_yx) tile_starts = getIntersectingBlocks(tile_blockshape, roi) # We use a fresh tmp dir for each read to avoid conflicts between parallel reads tmpdir = tempfile.mkdtemp() pool = RequestPool() for tile_start in tile_starts: tile_roi_in = getBlockBounds(self.description.shape_zyx, tile_blockshape, tile_start) tile_roi_in = numpy.array(tile_roi_in) # This tile's portion of the roi intersecting_roi = getIntersection(roi, tile_roi_in) intersecting_roi = numpy.array(intersecting_roi) # Compute slicing within destination array and slicing within this tile destination_relative_intersection = numpy.subtract( intersecting_roi, roi[0]) tile_relative_intersection = intersecting_roi - tile_roi_in[0] # Get a view to the output slice result_region = result_out[roiToSlice( *destination_relative_intersection)] # Special feature: # Some slices are missing, in which case we provide fake data from a different slice. # Overwrite the rest args to pull data from an alternate source tile. z_start = tile_roi_in[0][0] if z_start in self._slice_remapping: new_source_slice = self._slice_remapping[z_start] tile_roi_in[0][0] = new_source_slice tile_roi_in[1][0] = new_source_slice + 1 tile_index = numpy.array(tile_roi_in[0]) / tile_blockshape rest_args = { 'z_start': tile_roi_in[0][0], 'z_stop': tile_roi_in[1][0], 'y_start': tile_roi_in[0][1], 'y_stop': tile_roi_in[1][1], 'x_start': tile_roi_in[0][2], 'x_stop': tile_roi_in[1][2], 'z_index': tile_index[0], 'y_index': tile_index[1], 'x_index': tile_index[2] } # Quick sanity check assert rest_args['z_index'] == rest_args['z_start'] retrieval_fn = partial(self._retrieve_tile, tmpdir, rest_args, tile_relative_intersection, result_region) PARALLEL_REQ = True if PARALLEL_REQ: pool.add(Request(retrieval_fn)) else: # execute serially (leave the pool empty) retrieval_fn() pool.wait() # Clean up our temp files. shutil.rmtree(tmpdir)
def _executeProjection2D(self, roi, destination): assert sum(TinyVector(destination.shape) > 1) <= 2, "Projection result must be exactly 2D" # First, we have to determine which axis we are projecting along. # We infer this from the shape of the roi. # For example, if the roi is of shape # zyx = (1,256,256), then we know we're projecting along Z # If more than one axis has a width of 1, then we choose an # axis according to the following priority order: zyxt tagged_input_shape = self.Input.meta.getTaggedShape() tagged_result_shape = collections.OrderedDict( zip( tagged_input_shape.keys(), destination.shape ) ) nonprojection_axes = [] for key in tagged_input_shape.keys(): if (key == 'c' or tagged_input_shape[key] == 1 or tagged_result_shape[key] > 1): nonprojection_axes.append( key ) possible_projection_axes = set(tagged_input_shape) - set(nonprojection_axes) if len(possible_projection_axes) == 0: # If the image is 2D to begin with, # then the projection is simply the same as the normal output, # EXCEPT it is made binary self.Output(roi.start, roi.stop).writeInto(destination).wait() # make binary numpy.greater(destination, 0, out=destination) return for k in 'zyxt': if k in possible_projection_axes: projection_axis_key = k break # Now we know which axis we're projecting along. # Proceed with the projection, working blockwise to avoid unecessary work in unlabeled blocks projection_axis_index = self.Input.meta.getAxisKeys().index(projection_axis_key) projection_length = tagged_input_shape[projection_axis_key] input_roi = roi.copy() input_roi.start[projection_axis_index] = 0 input_roi.stop[projection_axis_index] = projection_length destination[:] = 0.0 # Get the logical blocking. block_starts = getIntersectingBlocks( self._blockshape, (input_roi.start, input_roi.stop) ) # (Parallelism wouldn't help here: h5py will serialize these requests anyway) block_starts = map( tuple, block_starts ) for block_start in block_starts: if block_start not in self._cacheFiles: # No label data in this block. Move on. continue entire_block_roi = getBlockBounds( self.Output.meta.shape, self._blockshape, block_start ) # This block's portion of the roi intersecting_roi = getIntersection( (input_roi.start, input_roi.stop), entire_block_roi ) # Compute slicing within the deep array and slicing within this block deep_relative_intersection = numpy.subtract(intersecting_roi, input_roi.start) block_relative_intersection = numpy.subtract(intersecting_roi, block_start) deep_data = self._getBlockDataset( entire_block_roi )[roiToSlice(*block_relative_intersection)] # make binary and convert to float deep_data_float = numpy.where( deep_data, numpy.float32(1.0), numpy.float32(0.0) ) # multiply by slice-index deep_data_view = numpy.rollaxis(deep_data_float, projection_axis_index, 0) min_deep_slice_index = deep_relative_intersection[0][projection_axis_index] max_deep_slice_index = deep_relative_intersection[1][projection_axis_index] def calc_color_value(slice_index): # Note 1: We assume that the colortable has at least 256 entries in it, # so, we try to ensure that all colors are above 1/256 # (we don't want colors in low slices to be rounded to 0) # Note 2: Ideally, we'd use a min projection in the code below, so that # labels in the "back" slices would appear occluded. But the # min projection would favor 0.0. Instead, we invert the # relationship between color and slice index, do a max projection, # and then re-invert the colors after everything is done. # Hence, this function starts with (1.0 - ...) return (1.0 - (float(slice_index) / projection_length)) * (1.0 - 1.0/255) + 1.0/255.0 min_color_value = calc_color_value(min_deep_slice_index) max_color_value = calc_color_value(max_deep_slice_index) num_slices = max_deep_slice_index - min_deep_slice_index deep_data_view *= numpy.linspace( min_color_value, max_color_value, num=num_slices )\ [ (slice(None),) + (None,)*(deep_data_view.ndim-1) ] # Take the max projection of this block's data. block_max_projection = numpy.amax(deep_data_float, axis=projection_axis_index, keepdims=True) # Merge this block's projection into the overall projection. destination_relative_intersection = numpy.array(deep_relative_intersection) destination_relative_intersection[:, projection_axis_index] = (0,1) destination_subview = destination[roiToSlice(*destination_relative_intersection)] numpy.maximum(block_max_projection, destination_subview, out=destination_subview) # Invert the nonzero pixels so increasing colors correspond to increasing slices. # See comment in calc_color_value(), above. destination_subview[:] = numpy.where(destination_subview, numpy.float32(1.0) - destination_subview, numpy.float32(0.0)) return
def read(self, roi, result_out): """ roi: (start, stop) tuples, ordered according to description.output_axes """ output_axes = self.description.output_axes roi_transposed = zip(*roi) roi_dict = dict( zip(output_axes, roi_transposed) ) roi = zip( *(roi_dict['z'], roi_dict['y'], roi_dict['x']) ) # First, normalize roi and result to zyx order result_out = vigra.taggedView(result_out, output_axes) result_out = result_out.withAxes(*'zyx') assert numpy.array(roi).shape == (2,3), "Invalid roi for 3D volume: {}".format( roi ) roi = numpy.array(roi) assert (result_out.shape == (roi[1] - roi[0])).all() tile_blockshape = (1,) + tuple(self.description.tile_shape_2d_yx) tile_starts = getIntersectingBlocks( tile_blockshape, roi ) # We use a fresh tmp dir for each read to avoid conflicts between parallel reads tmpdir = tempfile.mkdtemp() pool = RequestPool() for tile_start in tile_starts: tile_roi_in = getBlockBounds( self.description.shape_zyx, tile_blockshape, tile_start ) tile_roi_in = numpy.array(tile_roi_in) # This tile's portion of the roi intersecting_roi = getIntersection( roi, tile_roi_in ) intersecting_roi = numpy.array( intersecting_roi ) # Compute slicing within destination array and slicing within this tile destination_relative_intersection = numpy.subtract(intersecting_roi, roi[0]) tile_relative_intersection = intersecting_roi - tile_roi_in[0] # Get a view to the output slice result_region = result_out[roiToSlice(*destination_relative_intersection)] # Special feature: # Some slices are missing, in which case we provide fake data from a different slice. # Overwrite the rest args to pull data from an alternate source tile. z_start = tile_roi_in[0][0] if z_start in self._slice_remapping: new_source_slice = self._slice_remapping[z_start] tile_roi_in[0][0] = new_source_slice tile_roi_in[1][0] = new_source_slice+1 tile_index = numpy.array(tile_roi_in[0]) / tile_blockshape rest_args = { 'z_start' : tile_roi_in[0][0], 'z_stop' : tile_roi_in[1][0], 'y_start' : tile_roi_in[0][1], 'y_stop' : tile_roi_in[1][1], 'x_start' : tile_roi_in[0][2], 'x_stop' : tile_roi_in[1][2], 'z_index' : tile_index[0], 'y_index' : tile_index[1], 'x_index' : tile_index[2] } # Quick sanity check assert rest_args['z_index'] == rest_args['z_start'] retrieval_fn = partial( self._retrieve_tile, tmpdir, rest_args, tile_relative_intersection, result_region ) PARALLEL_REQ = True if PARALLEL_REQ: pool.add( Request( retrieval_fn ) ) else: # execute serially (leave the pool empty) retrieval_fn() pool.wait() # Clean up our temp files. shutil.rmtree(tmpdir)
def read(self, view_roi, result_out): """ roi: (start, stop) tuples, ordered according to description.output_axes roi should be relative to the view """ output_axes = self.description.output_axes roi_transposed = zip(*view_roi) roi_dict = dict( zip(output_axes, roi_transposed) ) view_roi = zip( *(roi_dict['z'], roi_dict['y'], roi_dict['x']) ) # First, normalize roi and result to zyx order result_out = vigra.taggedView(result_out, output_axes) result_out = result_out.withAxes(*'zyx') assert numpy.array(view_roi).shape == (2,3), "Invalid roi for 3D volume: {}".format( view_roi ) view_roi = numpy.array(view_roi) assert (result_out.shape == (view_roi[1] - view_roi[0])).all() # User gave roi according to the view output. # Now offset it find global roi. roi = view_roi + self.description.view_origin_zyx tile_blockshape = (1,) + tuple(self.description.tile_shape_2d_yx) tile_starts = getIntersectingBlocks( tile_blockshape, roi ) pool = RequestPool() for tile_start in tile_starts: tile_roi_in = getBlockBounds( self.description.bounds_zyx, tile_blockshape, tile_start ) tile_roi_in = numpy.array(tile_roi_in) # This tile's portion of the roi intersecting_roi = getIntersection( roi, tile_roi_in ) intersecting_roi = numpy.array( intersecting_roi ) # Compute slicing within destination array and slicing within this tile destination_relative_intersection = numpy.subtract(intersecting_roi, roi[0]) tile_relative_intersection = intersecting_roi - tile_roi_in[0] # Get a view to the output slice result_region = result_out[roiToSlice(*destination_relative_intersection)] # Special feature: # Some slices are missing, in which case we provide fake data from a different slice. # Overwrite the rest args to pull data from an alternate source tile. z_start = tile_roi_in[0][0] if z_start in self._slice_remapping: new_source_slice = self._slice_remapping[z_start] tile_roi_in[0][0] = new_source_slice tile_roi_in[1][0] = new_source_slice+1 tile_index = numpy.array(tile_roi_in[0]) / tile_blockshape rest_args = { 'z_start' : tile_roi_in[0][0], 'z_stop' : tile_roi_in[1][0], 'y_start' : tile_roi_in[0][1], 'y_stop' : tile_roi_in[1][1], 'x_start' : tile_roi_in[0][2], 'x_stop' : tile_roi_in[1][2], 'z_index' : tile_index[0], 'y_index' : tile_index[1], 'x_index' : tile_index[2] } # Apply special z_translation_function if self.description.z_translation_function is not None: z_update_func = eval(self.description.z_translation_function) rest_args['z_index'] = rest_args['z_start'] = z_update_func(rest_args['z_index']) rest_args['z_stop'] = 1 + rest_args['z_start'] # Quick sanity check assert rest_args['z_index'] == rest_args['z_start'] if self.description.tile_url_format.startswith('http'): retrieval_fn = partial( self._retrieve_remote_tile, rest_args, tile_relative_intersection, result_region ) else: retrieval_fn = partial( self._retrieve_local_tile, rest_args, tile_relative_intersection, result_region ) PARALLEL_REQ = True if PARALLEL_REQ: pool.add( Request( retrieval_fn ) ) else: # execute serially (leave the pool empty) retrieval_fn() if PARALLEL_REQ: with Timer() as timer: pool.wait() logger.info("Loading {} tiles took a total of {}".format( len(tile_starts), timer.seconds() ))