def execute(self, slot, subindex, roi, result): assert slot == self.MST, "Invalid output slot: {}".format(slot.name) #first thing, show the user that we are waiting for computations to finish self.applet.progressSignal.emit(0) volume_feat = self.Image(*roiFromShape(self.Image.meta.shape)).wait() labelVolume = self.LabelImage( *roiFromShape(self.LabelImage.meta.shape)).wait() self.applet.progress = 0 def updateProgressBar(x): #send signal iff progress is significant if x - self.applet.progress > 1 or x == 100: self.applet.progressSignal.emit(x) self.applet.progress = x #mst= MSTSegmentor(labelVolume[0,...,0], # numpy.asarray(volume_feat[0,...,0], numpy.float32), # edgeWeightFunctor = "minimum", # progressCallback = updateProgressBar) ##mst.raw is not set here in order to avoid redundant data storage #mst.raw = None newMst = WatershedSegmentor(labelVolume[0, ..., 0], numpy.asarray(volume_feat[0, ..., 0], numpy.float32), edgeWeightFunctor="minimum", progressCallback=updateProgressBar) #Output is of shape 1 #result[0] = mst newMst.raw = None result[0] = newMst return result
def execute(self,slot,subindex,roi,result): assert slot == self.MST, "Invalid output slot: {}".format(slot.name) #first thing, show the user that we are waiting for computations to finish self.applet.progressSignal.emit(0) volume_feat = self.Image( *roiFromShape( self.Image.meta.shape ) ).wait() labelVolume = self.LabelImage( *roiFromShape( self.LabelImage.meta.shape ) ).wait() self.applet.progress = 0 def updateProgressBar(x): #send signal iff progress is significant if x-self.applet.progress>1 or x==100: self.applet.progressSignal.emit(x) self.applet.progress = x mst= MSTSegmentor(labelVolume[0,...,0], numpy.asarray(volume_feat[0,...,0], numpy.float32), edgeWeightFunctor = "minimum", progressCallback = updateProgressBar) #mst.raw is not set here in order to avoid redundant data storage mst.raw = None #Output is of shape 1 result[0] = mst return result
def write(self): """ Requests the entire input and saves it to the file. This function executes synchronously. """ # TODO: Use a lazyflow.utility.BigRequestStreamer to split up # this giant request into a series of streamed subrequests. logger.warning( "The current implementation of NPY-format data export computes the entire dataset at once, which requires lots of RAM." ) path = self.Filepath.value self.progressSignal(0) final_data = numpy.zeros(self.Input.meta.shape, self.Input.meta.dtype) def handle_block_result(roi, data): slicing = roiToSlice(*roi) final_data[slicing] = data requester = BigRequestStreamer(self.Input, roiFromShape(self.Input.meta.shape)) requester.resultSignal.subscribe(handle_block_result) requester.progressSignal.subscribe(self.progressSignal) requester.execute() numpy.save(path, final_data) self.progressSignal(100)
def _calculate_probabilities(self, roi): classifier = self.Classifier.value assert isinstance( classifier, LazyflowPixelwiseClassifierABC ), f"Classifier {classifier} must be sublcass of {LazyflowPixelwiseClassifierABC}" upstream_roi = (roi.start, roi.stop) # Ask for the halo needed by the classifier axiskeys = self.Image.meta.getAxisKeys() halo_shape = classifier.get_halo_shape(axiskeys) assert len(halo_shape) == len(upstream_roi[0]) assert halo_shape[-1] == 0, "Didn't expect a non-zero halo for channel dimension." # Expand block by halo, then clip to image bounds upstream_roi = numpy.array(upstream_roi) upstream_roi[0] -= halo_shape upstream_roi[1] += halo_shape upstream_roi = getIntersection(upstream_roi, roiFromShape(self.Image.meta.shape)) upstream_roi = numpy.asarray(upstream_roi) # Determine how to extract the data from the result (without the halo) downstream_roi = numpy.array((roi.start, roi.stop)) predictions_roi = downstream_roi[:, :-1] - upstream_roi[0, :-1] # Request all upstream channels input_channels = self.Image.meta.shape[-1] upstream_roi[:, -1] = [0, input_channels] input_data = self.Image(*upstream_roi).wait() axistags = self.Image.meta.axistags probabilities = classifier.predict_probabilities_pixelwise(input_data, predictions_roi, axistags) return probabilities
def run_export(self): """ Request the volume in slices (running in parallel), and write each slice to a separate image. """ # Make the directory first if necessary export_dir = os.path.split(self.FilepathPattern.value)[0] if not os.path.exists(export_dir): os.makedirs(export_dir) # Sliceshape is the same as the input shape, except for the sliced dimension tagged_sliceshape = self.Input.meta.getTaggedShape() tagged_sliceshape[self._volume_axes[0]] = 1 slice_shape = (tagged_sliceshape.values()) # Use a request streamer to automatically request a constant batch of 4 active requests. streamer = BigRequestStreamer( self.Input, roiFromShape( self.Input.meta.shape ), slice_shape, batchSize=4 ) # Write the slices as they come in (possibly out-of-order, but probably not. streamer.resultSignal.subscribe( self._write_slice ) streamer.progressSignal.subscribe( self.progressSignal ) logger.debug("Starting Stack Export with slicing shape: {}".format( slice_shape )) streamer.execute()
def test3(self): # Generate a random dataset and see if it we get the right masking from the operator. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array( data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False ) # Provide input read all output. self.operator_identity.Input.setValue(numpy.zeros_like(data)) assert(self.operator_identity.Input.meta.has_mask) assert(self.operator_identity.Output.meta.has_mask) output = self.operator_identity.Output[None].wait() assert((output == 0).all()) assert(data.mask.shape == output.mask.shape) assert((output.mask == False).all()) # Try setInSlot data_shape_roi = roiFromShape(data.shape) data_shape_slice = roiToSlice(*data_shape_roi) self.operator_identity.Input[data_shape_slice] = data output = self.operator_identity.Output[None].wait() assert((data == output).all()) assert(data.mask.shape == output.mask.shape) assert((data.mask == output.mask).all())
def get_new_roi(self) -> Tuple[Tuple, Tuple]: # Prepare subregion operator total_roi = roiFromShape(self.Input.meta.shape) total_roi = list(map(tuple, total_roi)) # Default to full roi new_start, new_stop = total_roi if self.RegionStart.ready(): # RegionStart is permitted to contain 'None' values, which we replace with zeros new_start = [x or 0 for x in self.RegionStart.value] if self.RegionStop.ready(): # RegionStop is permitted to contain 'None' values, # which we replace with the full extent of the corresponding axis new_stop = [x_extent[0] or x_extent[1] for x_extent in zip(self.RegionStop.value, total_roi[1])] clipped_start = numpy.maximum(0, new_start) clipped_stop = numpy.minimum(total_roi[1], new_stop) if (clipped_start != new_start).any() or (clipped_stop != new_stop).any(): warnings.warn( "The ROI you are attempting to export exceeds the extents of your dataset. Clipping to dataset bounds." ) new_start, new_stop = tuple(clipped_start), tuple(clipped_stop) return new_start, new_stop
def _executeExportPath(self, result): path_format = self.OutputFilenameFormat.value file_extension = self._export_impls[self.OutputFormat.value][0] # Remove existing extension (if present) and add the correct extension (if any) if file_extension: path_format = os.path.splitext(path_format)[0] path_format += '.' + file_extension # Provide the TOTAL path (including dataset name) if self.OutputFormat.value == 'hdf5': path_format += '/' + self.OutputInternalPath.value roi = numpy.array(roiFromShape(self.Input.meta.shape)) # Intermediate state can cause coordinate offset and input shape to be mismatched. # Just don't use the offset if it looks wrong. # (The client will provide a valid offset later on.) if self.CoordinateOffset.ready() and len( self.CoordinateOffset.value) == len(roi[0]): offset = self.CoordinateOffset.value assert len(roi[0] == len(offset)) roi += offset optional_replacements = {} optional_replacements['roi'] = map(tuple, roi) for key, (start, stop) in zip(self.Input.meta.getAxisKeys(), roi.transpose()): optional_replacements[key + '_start'] = start optional_replacements[key + '_stop'] = stop formatted_path = format_known_keys(path_format, optional_replacements) result[0] = formatted_path return result
def execute(self, slot, subindex, rroi, result): self.progressSignal(0) # Save the axistags as a dataset attribute self.d.attrs['axistags'] = self.Image.meta.axistags.toJSON() def handle_block_result(roi, data): slicing = roiToSlice(*roi) if data.flags.c_contiguous: self.d.write_direct(data.view(numpy.ndarray), dest_sel=slicing) else: self.d[slicing] = data batch_size = None if self.BatchSize.ready(): batch_size = self.BatchSize.value requester = BigRequestStreamer(self.Image, roiFromShape(self.Image.meta.shape), batchSize=batch_size) requester.resultSignal.subscribe(handle_block_result) requester.progressSignal.subscribe(self.progressSignal) requester.execute() # Be paranoid: Flush right now. self.f.file.flush() # We're finished. result[0] = True self.progressSignal(100)
def test_via_OpExportSlot(self): data = 255 * numpy.random.random( (64, 128, 128, 1) ) data = data.astype( numpy.uint8 ) data = vigra.taggedView( data, vigra.defaultAxistags('zyxc') ) graph = Graph() opPiper = OpArrayPiper(graph=graph) opPiper.Input.setValue( data ) opExport = OpExportSlot(graph=graph) opExport.Input.connect( opPiper.Output ) opExport.OutputFormat.setValue( 'dvid' ) url = 'http://{hostname}/api/node/{data_uuid}/{data_name}'.format( **self.__dict__ ) opExport.OutputFilenameFormat.setValue( url ) assert opExport.ExportPath.ready() assert opExport.ExportPath.value == url opExport.run_export() opRead = OpInputDataReader( graph=graph ) try: opRead.FilePath.setValue( opExport.ExportPath.value ) expected_data = data.view(numpy.ndarray) read_data = opRead.Output( *roiFromShape(data.shape) ).wait() assert (read_data == expected_data).all(), "Read data didn't match exported data!" finally: opRead.cleanUp()
def _executeExportPath(self, result): path_format = self.OutputFilenameFormat.value file_extension = self._export_impls[ self.OutputFormat.value ][0] # Remove existing extension (if present) and add the correct extension path_format = os.path.splitext(path_format)[0] path_format += '.' + file_extension # Provide the TOTAL path (including dataset name) if self.OutputFormat.value == 'hdf5': path_format += '/' + self.OutputInternalPath.value roi = numpy.array( roiFromShape(self.Input.meta.shape) ) if self.CoordinateOffset.ready(): offset = self.CoordinateOffset.value assert len(roi[0] == len(offset)) roi += offset optional_replacements = {} optional_replacements['roi'] = map(tuple, roi) for key, (start, stop) in zip( self.Input.meta.getAxisKeys(), roi.transpose() ): optional_replacements[key + '_start'] = start optional_replacements[key + '_stop'] = stop formatted_path = format_known_keys( path_format, optional_replacements ) result[0] = formatted_path return result
def _executeExportPath(self, result): path_format = self.OutputFilenameFormat.value file_extension = self._export_impls[ self.OutputFormat.value ][0] # Remove existing extension (if present) and add the correct extension (if any) if file_extension: path_format = os.path.splitext(path_format)[0] path_format += '.' + file_extension # Provide the TOTAL path (including dataset name) if self.OutputFormat.value == 'hdf5': path_format += '/' + self.OutputInternalPath.value roi = numpy.array( roiFromShape(self.Input.meta.shape) ) # Intermediate state can cause coordinate offset and input shape to be mismatched. # Just don't use the offset if it looks wrong. # (The client will provide a valid offset later on.) if self.CoordinateOffset.ready() and len(self.CoordinateOffset.value) == len(roi[0]): offset = self.CoordinateOffset.value assert len(roi[0] == len(offset)) roi += offset optional_replacements = {} optional_replacements['roi'] = map(tuple, roi) for key, (start, stop) in zip( self.Input.meta.getAxisKeys(), roi.transpose() ): optional_replacements[key + '_start'] = start optional_replacements[key + '_stop'] = stop formatted_path = format_known_keys( path_format, optional_replacements ) result[0] = formatted_path return result
def __init__(self, slot, start = None, stop = None, pslice = None): super(SubRegion,self).__init__(slot) shape = None if slot is not None: shape = slot.meta.shape if pslice != None or start is not None and stop is None and pslice is None: if pslice is None: pslice = start if shape is None: # Okay to use a shapeless slot if the key is bounded # AND if the key has the correct length assert slicingtools.is_bounded(pslice) # Supply a dummy shape shape = [0] * len(pslice) self.start, self.stop = sliceToRoi(pslice,shape) elif start is None and pslice is None: assert shape is not None, "Can't create a default subregion without a slot and a shape." self.start, self.stop = roiFromShape(shape) else: self.start = TinyVector(start) self.stop = TinyVector(stop) self.dim = len(self.start) for start, stop in zip(self.start, self.stop): assert isinstance(start, (int, long, numpy.integer)), "Roi contains non-integers: {}".format( self ) assert isinstance(start, (int, long, numpy.integer)), "Roi contains non-integers: {}".format( self )
def test_via_OpExportSlot(self): data = 255 * numpy.random.random((64, 128, 128, 1)) data = data.astype(numpy.uint8) data = vigra.taggedView(data, vigra.defaultAxistags("zyxc")) graph = Graph() opPiper = OpArrayPiper(graph=graph) opPiper.Input.setValue(data) opExport = OpExportSlot(graph=graph) opExport.Input.connect(opPiper.Output) opExport.OutputFormat.setValue("dvid") url = "http://{hostname}/api/node/{data_uuid}/{data_name}".format( **self.__dict__) opExport.OutputFilenameFormat.setValue(url) assert opExport.ExportPath.ready() assert opExport.ExportPath.value == url opExport.run_export() opRead = OpInputDataReader(graph=graph) try: opRead.FilePath.setValue(opExport.ExportPath.value) expected_data = data.view(numpy.ndarray) read_data = opRead.Output(*roiFromShape(data.shape)).wait() assert (read_data == expected_data ).all(), "Read data didn't match exported data!" finally: opRead.cleanUp()
def ingestData(self, slot): """ Read the data from the given slot and copy it into this cache. The rules about special pixel meanings apply here, just like setInSlot Returns: the max label found in the slot. """ assert self._blockshape is not None assert self.Input.meta.shape == slot.meta.shape max_label = 0 # Get logical blocking. block_starts = getIntersectingBlocks( self._blockshape, roiFromShape(self.Input.meta.shape) ) block_starts = map( tuple, block_starts ) # Write each block for block_start in block_starts: block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, block_start ) # Request the block data block_data = slot(*block_roi).wait() # Write into the array subregion_roi = SubRegion(self.Input, *block_roi) cleaned_block_data = self._setInSlotInput( self.Input, (), subregion_roi, block_data ) max_label = max( max_label, cleaned_block_data.max() ) return max_label
def test3(self): # Generate a random dataset and see if it we get the right masking from the operator. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) data = numpy.ma.masked_array(data, mask=numpy.zeros(data.shape, dtype=bool), shrink=False) # Provide input read all output. self.operator_identity.Input.setValue(numpy.zeros_like(data)) assert self.operator_identity.Input.meta.has_mask assert self.operator_identity.Output.meta.has_mask output = self.operator_identity.Output[None].wait() assert (output == 0).all() assert data.mask.shape == output.mask.shape assert (output.mask == False).all() # Try setInSlot data_shape_roi = roiFromShape(data.shape) data_shape_slice = roiToSlice(*data_shape_roi) self.operator_identity.Input[data_shape_slice] = data output = self.operator_identity.Output[None].wait() assert (data == output).all() assert data.mask.shape == output.mask.shape assert (data.mask == output.mask).all()
def execute(self, slot, subindex, rroi, result): self.progressSignal(0) # Save the axistags as a dataset attribute self.d.attrs['axistags'] = self.Image.meta.axistags.toJSON() def handle_block_result(roi, data): slicing = roiToSlice(*roi) if data.flags.c_contiguous: self.d.write_direct(data.view(numpy.ndarray), dest_sel=slicing) else: self.d[slicing] = data batch_size = None if self.BatchSize.ready(): batch_size = self.BatchSize.value requester = BigRequestStreamer( self.Image, roiFromShape( self.Image.meta.shape ), batchSize=batch_size ) requester.resultSignal.subscribe( handle_block_result ) requester.progressSignal.subscribe( self.progressSignal ) requester.execute() # Be paranoid: Flush right now. self.f.file.flush() # We're finished. result[0] = True self.progressSignal(100)
def write(self): """ Requests the entire input and saves it to the file. This function executes synchronously. """ # TODO: Use a lazyflow.utility.BigRequestStreamer to split up # this giant request into a series of streamed subrequests. logger.warn("The current implementation of NPY-format data export computes the entire dataset at once, which requires lots of RAM.") path = self.Filepath.value self.progressSignal(0) final_data = numpy.zeros( self.Input.meta.shape, self.Input.meta.dtype ) def handle_block_result(roi, data): slicing = roiToSlice(*roi) final_data[slicing] = data requester = BigRequestStreamer( self.Input, roiFromShape( self.Input.meta.shape ) ) requester.resultSignal.subscribe( handle_block_result ) requester.progressSignal.subscribe( self.progressSignal ) requester.execute() numpy.save(path, final_data) self.progressSignal(100)
def run_export(self): """ Request the volume in slices (running in parallel), and write each slice to the correct page. Note: We can't use BigRequestStreamer here, because the data for each slice wouldn't be guaranteed to arrive in the correct order. """ # Make the directory first if necessary export_dir = os.path.split(self.FilepathPattern.value)[0] if not os.path.exists(export_dir): os.makedirs(export_dir) # Blockshape is the same as the input shape, except for the sliced dimension step_axis = self._volume_axes[0] tagged_blockshape = self.Input.meta.getTaggedShape() tagged_blockshape[step_axis] = 1 block_shape = (tagged_blockshape.values()) logger.debug("Starting Multipage Sequence Export with block shape: {}".format( block_shape )) # Block step is all zeros except step axis, e.g. (0, 1, 0, 0, 0) block_step = numpy.array( self.Input.meta.getAxisKeys() ) == step_axis block_step = block_step.astype(int) filepattern = self.FilepathPattern.value # If the user didn't provide custom formatting for the slice field, # auto-format to include zero-padding if '{slice_index}' in filepattern: filepattern = filepattern.format( slice_index='{' + 'slice_index:0{}'.format(self._max_slice_digits) + '}' ) self.progressSignal(0) # Nothing fancy here: Just loop over the blocks in order. tagged_shape = self.Input.meta.getTaggedShape() for block_index in xrange( tagged_shape[step_axis] ): roi = numpy.array(roiFromShape(block_shape)) roi += block_index*block_step roi = map(tuple, roi) try: opSubregion = OpSubRegion( parent=self ) opSubregion.Roi.setValue( roi ) opSubregion.Input.connect( self.Input ) formatted_path = filepattern.format( slice_index=(block_index + self.SliceIndexOffset.value) ) opExportBlock = OpExportMultipageTiff( parent=self ) opExportBlock.Input.connect( opSubregion.Output ) opExportBlock.Filepath.setValue( formatted_path ) block_start_progress = 100*block_index // tagged_shape[step_axis] def _handleBlockProgress(block_progress): self.progressSignal( block_start_progress + block_progress//tagged_shape[step_axis] ) opExportBlock.progressSignal.subscribe( _handleBlockProgress ) # Run the export for this block opExportBlock.run_export() finally: opExportBlock.cleanUp() opSubregion.cleanUp() self.progressSignal(100)
def run_export(self): self.progressSignal(0) url = self.NodeDataUrl.value url_path = url.split('://')[1] hostname, api, node, uuid, dataname = url_path.split('/') assert api == 'api' assert node == 'node' axiskeys = self.Input.meta.getAxisKeys() shape = self.Input.meta.shape if self._transpose_axes: axiskeys = reversed(axiskeys) shape = tuple(reversed(shape)) axiskeys = "".join( axiskeys ) if self.OffsetCoord.ready(): offset_start = self.OffsetCoord.value else: offset_start = (0,) * len( self.Input.meta.shape ) self.progressSignal(5) # Get the dataset details try: metadata = VoxelsAccessor.get_metadata(hostname, uuid, dataname) except VoxelsAccessor.BadRequestError as ex: # Dataset doesn't exist yet. Let's create it. metadata = VoxelsMetadata.create_default_metadata( shape, self.Input.meta.dtype, axiskeys, 0.0, "" ) VoxelsAccessor.create_new(hostname, uuid, dataname, metadata) # Since this class is generally used to push large blocks of data, # we'll be nice and set throttle=True client = VoxelsAccessor( hostname, uuid, dataname, throttle=True ) def handle_block_result(roi, data): # Send it to dvid roi = numpy.asarray(roi) roi += offset_start start, stop = roi if self._transpose_axes: data = data.transpose() start = tuple(reversed(start)) stop = tuple(reversed(stop)) client.post_ndarray( start, stop, data ) requester = BigRequestStreamer( self.Input, roiFromShape( self.Input.meta.shape ) ) requester.resultSignal.subscribe( handle_block_result ) requester.progressSignal.subscribe( self.progressSignal ) requester.execute() self.progressSignal(100)
def setInSlot(self, slot, subindex, roi, value): # Forward to the output self.Output[roiToSlice(roi.start, roi.stop)] = value entire_roi = roiFromShape(self.Input.meta.shape) if (numpy.array((roi.start, roi.stop)) == entire_roi).all(): # Nothing is dirty any more. self._init_fixed_dirty_roi()
def autoSeedBackground(cls, laneView, foreground_label): # Seed the entire image with background labels, except for the individual label in question # To save memory, we'll do this in blocks instead of all at once volume_shape = laneView.RavelerLabels.meta.shape volume_roi = roiFromShape(volume_shape) block_shape = (OpSplitBodyCarving.BLOCK_SIZE, ) * len(volume_shape) block_shape = numpy.minimum(block_shape, volume_shape) block_starts = getIntersectingBlocks(block_shape, volume_roi) logger.debug("Auto-seeding {} blocks for label".format( len(block_starts), foreground_label)) for block_index, block_start in enumerate(block_starts): block_roi = getBlockBounds(volume_shape, block_shape, block_start) label_block = laneView.RavelerLabels(*block_roi).wait() background_block = numpy.where(label_block == foreground_label, 0, 1) background_block = numpy.asarray( background_block, numpy.float32) # Distance transform requires float if (background_block == 0.0).any(): # We need to leave a small border between the background seeds and the object membranes background_block_view = background_block.view(vigra.VigraArray) background_block_view.axistags = copy.copy( laneView.RavelerLabels.meta.axistags) background_block_view_4d = background_block_view.bindAxis( 't', 0) background_block_view_3d = background_block_view_4d.bindAxis( 'c', 0) distance_transformed_block = vigra.filters.distanceTransform( background_block_view_3d, background=False) distance_transformed_block = distance_transformed_block.astype( numpy.uint8) # Create a 'hull' surrounding the foreground, but leave some space. background_seed_block = (distance_transformed_block == OpSplitBodyCarving.SEED_MARGIN) background_seed_block = background_seed_block.astype( numpy.uint8) * 1 # (In carving, background is label 1) # # Make the hull VERY sparse to avoid over-biasing graph cut toward the background class # # FIXME: Don't regenerate this random block on every loop iteration # rand_bytes = numpy.random.randint(0, 1000, background_seed_block.shape) # background_seed_block = numpy.where( rand_bytes < 1, background_seed_block, 0 ) # background_seed_block = background_seed_block.view(vigra.VigraArray) # background_seed_block.axistags = background_block_view_3d.axistags axisorder = laneView.RavelerLabels.meta.getTaggedShape().keys() logger.debug("Writing backgound seeds: {}/{}".format( block_index, len(block_starts))) laneView.WriteSeeds[roiToSlice( *block_roi)] = background_seed_block.withAxes(*axisorder) else: logger.debug("Skipping all-background block: {}/{}".format( block_index, len(block_starts)))
def run_export(self): self.progressSignal(0) url = self.NodeDataUrl.value url_path = url.split('://')[1] hostname, api, node, uuid, dataname = url_path.split('/') assert api == 'api' assert node == 'node' axiskeys = self.Input.meta.getAxisKeys() shape = self.Input.meta.shape if self._transpose_axes: axiskeys = reversed(axiskeys) shape = tuple(reversed(shape)) axiskeys = "".join(axiskeys) if self.OffsetCoord.ready(): offset_start = self.OffsetCoord.value else: offset_start = (0, ) * len(self.Input.meta.shape) self.progressSignal(5) # Get the dataset details try: metadata = VoxelsAccessor.get_metadata(hostname, uuid, dataname) except DVIDException as ex: if ex.status != 404: raise # Dataset doesn't exist yet. Let's create it. metadata = VoxelsMetadata.create_default_metadata( shape, self.Input.meta.dtype, axiskeys, 0.0, "") VoxelsAccessor.create_new(hostname, uuid, dataname, metadata) # Since this class is generally used to push large blocks of data, # we'll be nice and set throttle=True client = VoxelsAccessor(hostname, uuid, dataname, throttle=True) def handle_block_result(roi, data): # Send it to dvid roi = numpy.asarray(roi) roi += offset_start start, stop = roi if self._transpose_axes: data = data.transpose() start = tuple(reversed(start)) stop = tuple(reversed(stop)) client.post_ndarray(start, stop, data) requester = BigRequestStreamer(self.Input, roiFromShape(self.Input.meta.shape)) requester.resultSignal.subscribe(handle_block_result) requester.progressSignal.subscribe(self.progressSignal) requester.execute() self.progressSignal(100)
def _purge_label(self, label_to_purge, decrement_remaining, replacement_value=0): """ Scan through all labeled pixels. (1) Reassign all pixels of the given value (set to replacement_value) (2) If decrement_remaining=True, decrement all labels above that value so the set of stored labels remains consecutive. Note that the decrement is performed AFTER replacement. """ changed_block_rois = [] # stored_block_rois = self.CleanBlocks.value stored_block_roi_destination = [None] self.execute(self.CleanBlocks, (), SubRegion(self.Output, (0, ), (1, )), stored_block_roi_destination) stored_block_rois = stored_block_roi_destination[0] for block_roi in stored_block_rois: # Get data block_shape = numpy.subtract(block_roi[1], block_roi[0]) block = self.Output.stype.allocateDestination( SubRegion(self.Output, *roiFromShape(block_shape))) self.execute(self.Output, (), SubRegion(self.Output, *block_roi), block) # Locate pixels to change matching_label_coords = numpy.nonzero(block == label_to_purge) # Change the data block[matching_label_coords] = replacement_value coords_to_decrement = block > label_to_purge if decrement_remaining: block[coords_to_decrement] -= numpy.uint8(1) # Update cache with the new data (only if something really changed) if len(matching_label_coords[0]) > 0 or ( decrement_remaining and coords_to_decrement.sum() > 0): super(OpCompressedUserLabelArray, self)._setInSlotInput(self.Input, (), SubRegion(self.Output, *block_roi), block, store_zero_blocks=False) changed_block_rois.append(block_roi) for block_roi in changed_block_rois: # FIXME: Shouldn't this dirty notification be handled in OpUnmanagedCompressedCache? self.Output.setDirty(*block_roi)
def testBasic_Dvid(self): if _skip_dvid: raise nose.SkipTest # Spin up a mock dvid server to test with. dvid_dataset, data_uuid, data_name = "datasetA", "abcde", "indices_data" mockserver_data_file = self._tmpdir + '/mockserver_data.h5' with H5MockServerDataFile(mockserver_data_file) as test_h5file: test_h5file.add_node(dvid_dataset, data_uuid) server_proc, shutdown_event = H5MockServer.create_and_start( mockserver_data_file, "localhost", 8000, same_process=False, disable_server_logging=True) try: data = 255 * numpy.random.random((100, 100, 4)) data = data.astype(numpy.uint8) data = vigra.taggedView(data, vigra.defaultAxistags('xyc')) graph = Graph() opPiper = OpArrayPiper(graph=graph) opPiper.Input.setValue(data) opExport = OpExportSlot(graph=graph) opExport.Input.connect(opPiper.Output) opExport.OutputFormat.setValue('dvid') url = 'http://localhost:8000/api/node/{data_uuid}/{data_name}'.format( **locals()) opExport.OutputFilenameFormat.setValue(url) assert opExport.ExportPath.ready() assert opExport.ExportPath.value == url opExport.run_export() try: opRead = OpInputDataReader(graph=graph) opRead.FilePath.setValue(opExport.ExportPath.value) expected_data = data.view(numpy.ndarray) read_data = opRead.Output(*roiFromShape(data.shape)).wait() assert (read_data == expected_data ).all(), "Read data didn't match exported data!" finally: opRead.cleanUp() finally: shutdown_event.set() server_proc.join()
def run_export(self): """ Request the volume in slices (running in parallel), and write each slice to a separate image. """ # Make the directory first if necessary export_dir = os.path.split(self.FilepathPattern.value)[0] if not os.path.exists(export_dir): os.makedirs(export_dir) # Sliceshape is the same as the input shape, except for the sliced dimension tagged_sliceshape = self.Input.meta.getTaggedShape() tagged_sliceshape[self._volume_axes[0]] = 1 slice_shape = (list(tagged_sliceshape.values())) parallel_requests = 4 # If ram usage info is available, make a better guess about how many requests we can launch in parallel ram_usage_per_requested_pixel = self.Input.meta.ram_usage_per_requested_pixel if ram_usage_per_requested_pixel is not None: pixels_per_slice = numpy.prod(slice_shape) if 'c' in tagged_sliceshape: pixels_per_slice //= tagged_sliceshape['c'] ram_usage_per_slice = pixels_per_slice * ram_usage_per_requested_pixel # Fudge factor: Reduce RAM usage by a bit available_ram = psutil.virtual_memory().available available_ram *= 0.5 parallel_requests = int(available_ram // ram_usage_per_slice) if parallel_requests < 1: raise MemoryError( 'Not enough RAM to export to the selected format. ' 'Consider exporting to hdf5 (h5).' ) streamer = BigRequestStreamer( self.Input, roiFromShape( self.Input.meta.shape ), slice_shape, parallel_requests ) # Write the slices as they come in (possibly out-of-order, but probably not) streamer.resultSignal.subscribe( self._write_slice ) streamer.progressSignal.subscribe( self.progressSignal ) logger.debug("Starting Stack Export with slicing shape: {}".format( slice_shape )) streamer.execute()
def run_export_to_array(self): # Allocate result final_result = numpy.ndarray( dtype=self.Input.meta.dtype, shape=self.Input.meta.shape ) # Prepare streamer streamer = BigRequestStreamer( self.Input, roiFromShape(self.Input.meta.shape), allowParallelResults=True ) def handle_block_result(roi, block_result): final_result[roiToSlice(*roi)] = block_result streamer.resultSignal.subscribe( handle_block_result ) streamer.progressSignal.subscribe( self.progressSignal ) # Perform export streamer.execute() return final_result
def run_export_to_array(self): # Allocate result final_result = numpy.ndarray(dtype=self.Input.meta.dtype, shape=self.Input.meta.shape) # Prepare streamer streamer = BigRequestStreamer(self.Input, roiFromShape(self.Input.meta.shape), allowParallelResults=True) def handle_block_result(roi, block_result): final_result[roiToSlice(*roi)] = block_result streamer.resultSignal.subscribe(handle_block_result) streamer.progressSignal.subscribe(self.progressSignal) # Perform export streamer.execute() return final_result
def execute(self, slot, subindex, roi, result): classifier_factory = self.ClassifierFactory.value assert issubclass(type(classifier_factory), LazyflowPixelwiseClassifierFactoryABC), \ "Factory is of type {}, which does not satisfy the LazyflowPixelwiseClassifierFactoryABC interface."\ "".format( type(classifier_factory) ) # Accumulate all non-zero blocks of each image into lists label_data_blocks = [] image_data_blocks = [] for image_slot, label_slot, nonzero_block_slot in zip(self.Images, self.Labels, self.nonzeroLabelBlocks): block_slicings = nonzero_block_slot.value for block_slicing in block_slicings: block_label_roi = sliceToRoi( block_slicing, image_slot.meta.shape ) # Ask for the halo needed by the classifier axiskeys = image_slot.meta.getAxisKeys() halo_shape = classifier_factory.get_halo_shape(axiskeys) assert len(halo_shape) == len( block_label_roi[0] ) assert halo_shape[-1] == 0, "Didn't expect a non-zero halo for channel dimension." # Expand block by halo, then clip to image bounds block_label_roi = numpy.array( block_label_roi ) block_label_roi[0] -= halo_shape block_label_roi[1] += halo_shape block_label_roi = getIntersection( block_label_roi, roiFromShape(image_slot.meta.shape) ) block_image_roi = numpy.array( block_label_roi ) assert (block_image_roi[:, -1] == [0,1]).all() num_channels = image_slot.meta.shape[-1] block_image_roi[:, -1] = [0, num_channels] # Ensure the results are plain ndarray, not VigraArray, # which some classifiers might have trouble with. block_label_data = numpy.asarray( label_slot(*block_label_roi).wait() ) block_image_data = numpy.asarray( image_slot(*block_image_roi).wait() ) label_data_blocks.append( block_label_data ) image_data_blocks.append( block_image_data ) logger.debug("Training new classifier: {}".format( classifier_factory.description )) classifier = classifier_factory.create_and_train_pixelwise( image_data_blocks, label_data_blocks ) assert issubclass(type(classifier), LazyflowPixelwiseClassifierABC), \ "Classifier is of type {}, which does not satisfy the LazyflowPixelwiseClassifierABC interface."\ "".format( type(classifier) ) result[0] = classifier return result
def test3(self): # Generate a random dataset and see if it we get the right masking from the operator. data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32) # Provide input read all output. self.operator_identity.Input.setValue(numpy.zeros_like(data)) output = self.operator_identity.Output[None].wait() assert (output == 0).all() # Try setInSlot data_shape_roi = roiFromShape(data.shape) data_shape_slice = roiToSlice(*data_shape_roi) self.operator_identity.Input[data_shape_slice] = data output = self.operator_identity.Output[None].wait() assert (data == output).all()
def propagateDirty(self, slot, subindex, roi): dirty_roi = self._standardize_roi( roi.start, roi.stop ) maximum_roi = roiFromShape(self.Input.meta.shape) maximum_roi = self._standardize_roi( *maximum_roi ) if dirty_roi == maximum_roi: # Optimize the common case: # Everything is dirty, so no need to loop self._resetBlocks() else: # FIXME: This is O(N) for now. # We should speed this up by maintaining a bookkeeping data structure in execute(). for block_roi in self._block_data.keys(): if getIntersection(block_roi, dirty_roi, assertIntersect=False): self.freeBlock(block_roi) self.Output.setDirty( roi.start, roi.stop )
def propagateDirty(self, slot, subindex, roi): dirty_roi = self._standardize_roi(roi.start, roi.stop) maximum_roi = roiFromShape(self.Input.meta.shape) maximum_roi = self._standardize_roi(*maximum_roi) if dirty_roi == maximum_roi: # Optimize the common case: # Everything is dirty, so no need to loop self._resetBlocks() else: # FIXME: This is O(N) for now. # We should speed this up by maintaining a bookkeeping data structure in execute(). for block_roi in self._block_data.keys(): if getIntersection(block_roi, dirty_roi, assertIntersect=False): self.freeBlock(block_roi) self.Output.setDirty(roi.start, roi.stop)
def _setInSlotInput(self, slot, subindex, roi, new_pixels): """ Since this is a label array, inserting pixels has a special meaning: We only overwrite the new non-zero pixels. In the new data, zeros mean "don't change". So, here's what each pixel we're adding means: 0: don't change 1: change to 1 2: change to 2 ... N: change to N magic_eraser_value: change to 0 """ if isinstance(new_pixels, vigra.VigraArray): new_pixels = new_pixels.view(numpy.ndarray) # Extract the data to modify original_data = self.Output.stype.allocateDestination( SubRegion(self.Output, *roiFromShape(new_pixels.shape))) self.execute(self.Output, (), roi, original_data) # Reset the pixels we need to change (so we can use |= below) original_data[new_pixels.nonzero()] = 0 # Update original_data |= new_pixels # Replace 'eraser' values with zeros. cleaned_data = original_data.copy() cleaned_data[original_data == self._eraser_magic_value] = 0 # Set in the cache (our superclass). super(OpCompressedUserLabelArray, self)._setInSlotInput(slot, subindex, roi, cleaned_data, store_zero_blocks=False) # FIXME: Shouldn't this notification be triggered from within OpUnmanagedCompressedCache? self.Output.setDirty(roi.start, roi.stop) return cleaned_data # Internal use: Return the cleaned_data
def autoSeedBackground(cls, laneView, foreground_label): # Seed the entire image with background labels, except for the individual label in question # To save memory, we'll do this in blocks instead of all at once volume_shape = laneView.RavelerLabels.meta.shape volume_roi = roiFromShape( volume_shape ) block_shape = (OpSplitBodyCarving.BLOCK_SIZE,) * len( volume_shape ) block_shape = numpy.minimum( block_shape, volume_shape ) block_starts = getIntersectingBlocks( block_shape, volume_roi ) logger.debug("Auto-seeding {} blocks for label".format( len(block_starts), foreground_label )) for block_index, block_start in enumerate(block_starts): block_roi = getBlockBounds( volume_shape, block_shape, block_start ) label_block = laneView.RavelerLabels(*block_roi).wait() background_block = numpy.where( label_block == foreground_label, 0, 1 ) background_block = numpy.asarray( background_block, numpy.float32 ) # Distance transform requires float if (background_block == 0.0).any(): # We need to leave a small border between the background seeds and the object membranes background_block_view = background_block.view( vigra.VigraArray ) background_block_view.axistags = copy.copy( laneView.RavelerLabels.meta.axistags ) background_block_view_4d = background_block_view.bindAxis('t', 0) background_block_view_3d = background_block_view_4d.bindAxis('c', 0) distance_transformed_block = vigra.filters.distanceTransform3D(background_block_view_3d, background=False) distance_transformed_block = distance_transformed_block.astype( numpy.uint8 ) # Create a 'hull' surrounding the foreground, but leave some space. background_seed_block = (distance_transformed_block == OpSplitBodyCarving.SEED_MARGIN) background_seed_block = background_seed_block.astype(numpy.uint8) * 1 # (In carving, background is label 1) # # Make the hull VERY sparse to avoid over-biasing graph cut toward the background class # # FIXME: Don't regenerate this random block on every loop iteration # rand_bytes = numpy.random.randint(0, 1000, background_seed_block.shape) # background_seed_block = numpy.where( rand_bytes < 1, background_seed_block, 0 ) # background_seed_block = background_seed_block.view(vigra.VigraArray) # background_seed_block.axistags = background_block_view_3d.axistags axisorder = laneView.RavelerLabels.meta.getTaggedShape().keys() logger.debug("Writing backgound seeds: {}/{}".format( block_index, len(block_starts) )) laneView.WriteSeeds[ roiToSlice( *block_roi ) ] = background_seed_block.withAxes(*axisorder) else: logger.debug("Skipping all-background block: {}/{}".format( block_index, len(block_starts) ))
def run_export(self): """ Request the volume in slices (running in parallel), and write each slice to a separate image. """ # Make the directory first if necessary export_dir = os.path.split(self.FilepathPattern.value)[0] if not os.path.exists(export_dir): os.makedirs(export_dir) # Sliceshape is the same as the input shape, except for the sliced dimension tagged_sliceshape = self.Input.meta.getTaggedShape() tagged_sliceshape[self._volume_axes[0]] = 1 slice_shape = list(tagged_sliceshape.values()) parallel_requests = 4 # If ram usage info is available, make a better guess about how many requests we can launch in parallel ram_usage_per_requested_pixel = self.Input.meta.ram_usage_per_requested_pixel if ram_usage_per_requested_pixel is not None: pixels_per_slice = numpy.prod(slice_shape) if "c" in tagged_sliceshape: pixels_per_slice //= tagged_sliceshape["c"] ram_usage_per_slice = pixels_per_slice * ram_usage_per_requested_pixel # Fudge factor: Reduce RAM usage by a bit available_ram = psutil.virtual_memory().available available_ram *= 0.5 parallel_requests = int(available_ram // ram_usage_per_slice) if parallel_requests < 1: raise MemoryError( "Not enough RAM to export to the selected format. " "Consider exporting to hdf5 (h5)." ) streamer = BigRequestStreamer(self.Input, roiFromShape(self.Input.meta.shape), slice_shape, parallel_requests) # Write the slices as they come in (possibly out-of-order, but probably not) streamer.resultSignal.subscribe(self._write_slice) streamer.progressSignal.subscribe(self.progressSignal) logger.debug(f"Starting Stack Export with slicing shape: {slice_shape}") streamer.execute()
def export_to_tiles(volume, tile_size, output_dir, print_progress=True): """ volume: The volume to export (either hdf5 dataset or numpy array). Must be 3D. tile_size: The width of the tiles to generate output_dir: The directory to dump the tiles to. """ assert len(volume.shape) == 3 tile_blockshape = (1, tile_size, tile_size) tile_starts = getIntersectingBlocks(tile_blockshape, roiFromShape(volume.shape)) if not os.path.exists(output_dir): os.makedirs(output_dir) logger.info("Writing {} tiles ...".format(len(tile_starts))) for tile_start in tile_starts: tile_roi = getBlockBounds(volume.shape, tile_blockshape, tile_start) if print_progress: sys.stdout.write("Tile: {} ".format(tile_roi)) sys.stdout.flush() tile_data = volume[roiToSlice(*tile_roi)] tile_data = vigra.taggedView(tile_data, 'zyx') if print_progress: sys.stdout.write('reading... ') sys.stdout.flush() tile_name = 'tile_z{:05}_y{:05}_x{:05}.png'.format(*tile_start) output_path = os.path.join(output_dir, tile_name) if print_progress: sys.stdout.write('writing... ') sys.stdout.flush() vigra.impex.writeImage(tile_data[0], output_path, dtype='NATIVE') if print_progress: sys.stdout.write('done.\n') sys.stdout.flush() logger.info("TILES COMPLETE.")
def testBasic_Dvid(self): if _skip_dvid: raise nose.SkipTest # Spin up a mock dvid server to test with. dvid_dataset, data_uuid, data_name = "datasetA", "abcde", "indices_data" mockserver_data_file = self._tmpdir + '/mockserver_data.h5' with H5MockServerDataFile( mockserver_data_file ) as test_h5file: test_h5file.add_node( dvid_dataset, data_uuid ) server_proc, shutdown_event = H5MockServer.create_and_start( mockserver_data_file, "localhost", 8000, same_process=False, disable_server_logging=True ) try: data = 255 * numpy.random.random( (100,100, 4) ) data = data.astype( numpy.uint8 ) data = vigra.taggedView( data, vigra.defaultAxistags('xyc') ) graph = Graph() opPiper = OpArrayPiper(graph=graph) opPiper.Input.setValue( data ) opExport = OpExportSlot(graph=graph) opExport.Input.connect( opPiper.Output ) opExport.OutputFormat.setValue( 'dvid' ) url = 'http://localhost:8000/api/node/{data_uuid}/{data_name}'.format( **locals() ) opExport.OutputFilenameFormat.setValue( url ) assert opExport.ExportPath.ready() assert opExport.ExportPath.value == url opExport.run_export() try: opRead = OpInputDataReader( graph=graph ) opRead.FilePath.setValue( opExport.ExportPath.value ) expected_data = data.view(numpy.ndarray) read_data = opRead.Output( *roiFromShape(data.shape) ).wait() assert (read_data == expected_data).all(), "Read data didn't match exported data!" finally: opRead.cleanUp() finally: shutdown_event.set() server_proc.join()
def export_to_tiles(volume, tile_size, output_dir, print_progress=True): """ volume: The volume to export (either hdf5 dataset or numpy array). Must be 3D. tile_size: The width of the tiles to generate output_dir: The directory to dump the tiles to. """ assert len(volume.shape) == 3 tile_blockshape = (1, tile_size, tile_size) tile_starts = getIntersectingBlocks( tile_blockshape, roiFromShape( volume.shape ) ) if not os.path.exists(output_dir): os.makedirs(output_dir) logger.info("Writing {} tiles ...".format( len(tile_starts) ) ) for tile_start in tile_starts: tile_roi = getBlockBounds( volume.shape, tile_blockshape, tile_start ) if print_progress: sys.stdout.write("Tile: {} ".format( tile_roi )) sys.stdout.flush() tile_data = volume[ roiToSlice(*tile_roi) ] tile_data = vigra.taggedView(tile_data, 'zyx') if print_progress: sys.stdout.write('reading... ') sys.stdout.flush() tile_name = 'tile_z{:05}_y{:05}_x{:05}.png'.format( *tile_start ) output_path = os.path.join( output_dir, tile_name ) if print_progress: sys.stdout.write('writing... ') sys.stdout.flush() vigra.impex.writeImage( tile_data[0], output_path, dtype='NATIVE' ) if print_progress: sys.stdout.write('done.\n') sys.stdout.flush() logger.info( "TILES COMPLETE." )
def _purge_label(self, label_to_purge, decrement_remaining, replacement_value=0): """ Scan through all labeled pixels. (1) Reassign all pixels of the given value (set to replacement_value) (2) If decrement_remaining=True, decrement all labels above that value so the set of stored labels remains consecutive. Note that the decrement is performed AFTER replacement. """ changed_block_rois = [] # stored_block_rois = self.CleanBlocks.value stored_block_roi_destination = [None] self.execute(self.CleanBlocks, (), SubRegion(self.Output, (0,), (1,)), stored_block_roi_destination) stored_block_rois = stored_block_roi_destination[0] for block_roi in stored_block_rois: # Get data block_shape = numpy.subtract(block_roi[1], block_roi[0]) block = self.Output.stype.allocateDestination(SubRegion(self.Output, *roiFromShape(block_shape))) self.execute(self.Output, (), SubRegion(self.Output, *block_roi), block) # Locate pixels to change matching_label_coords = numpy.nonzero(block == label_to_purge) # Change the data block[matching_label_coords] = replacement_value coords_to_decrement = block > label_to_purge if decrement_remaining: block[coords_to_decrement] -= numpy.uint8(1) # Update cache with the new data (only if something really changed) if len(matching_label_coords[0]) > 0 or (decrement_remaining and coords_to_decrement.sum() > 0): super(OpCompressedUserLabelArray, self)._setInSlotInput( self.Input, (), SubRegion(self.Output, *block_roi), block, store_zero_blocks=False ) changed_block_rois.append(block_roi) for block_roi in changed_block_rois: # FIXME: Shouldn't this dirty notification be handled in OpUnmanagedCompressedCache? self.Output.setDirty(*block_roi)
def run_export(self): self.progressSignal(0) url = self.NodeDataUrl.value url_path = url.split('://')[1] hostname, api, node, uuid, dataname = url_path.split('/') assert api == 'api' assert node == 'node' # Request the data axiskeys = self.Input.meta.getAxisKeys() data = self.Input[:].wait() if self._transpose_axes: data = data.transpose() axiskeys = reversed(axiskeys) axiskeys = "".join( axiskeys ) # FIXME: We assume the dataset needs to be created first. # If it already existed, this (presumably) causes an error on the DVID side. metadata = pydvid.voxels.VoxelsMetadata.create_default_metadata( data.shape, data.dtype.type, axiskeys, 0.0, "" ) connection = httplib.HTTPConnection( hostname ) with contextlib.closing( connection ): self.progressSignal(5) pydvid.voxels.create_new(connection, uuid, dataname, metadata) client = pydvid.voxels.VoxelsAccessor( connection, uuid, dataname ) # For now, we send the whole darn thing at once. # TODO: Stream it over in blocks... # Send it to dvid start, stop = roiFromShape(data.shape) client.post_ndarray( start, stop, data ) self.progressSignal(100)
def execute(self, slot, subindex, rroi, result): self.progressSignal(0) # Save the axistags as a dataset attribute self.d.attrs["axistags"] = self.Image.meta.axistags.toJSON() if isinstance(self.d, h5py.Dataset): for index, tag in enumerate(self.Image.meta.axistags): self.d.dims[index].label = tag.key else: # if n5 dataset, apply neuroglancer's axes tags convention self.d.attrs["axes"] = "".join( tag.key for tag in self.Image.meta.axistags)[::-1] drange = self.Image.meta.get("drange") if drange: self.d.attrs["drange"] = drange def handle_block_result(roi, data): slicing = roiToSlice(*roi) if data.flags.c_contiguous: self.d.write_direct(data.view(numpy.ndarray), dest_sel=slicing) else: self.d[slicing] = data batch_size = None if self.BatchSize.ready(): batch_size = self.BatchSize.value requester = BigRequestStreamer(self.Image, roiFromShape(self.Image.meta.shape), batchSize=batch_size) requester.resultSignal.subscribe(handle_block_result) requester.progressSignal.subscribe(self.progressSignal) requester.execute() # Be paranoid: Flush right now. if isinstance(self.f, h5py.File): self.f.file.flush() # not available in z5py # We're finished. result[0] = True self.progressSignal(100)
def run_export(self): self.progressSignal(0) url = self.NodeDataUrl.value url_path = url.split('://')[1] hostname, api, node, uuid, dataname = url_path.split('/') assert api == 'api' assert node == 'node' # Request the data axiskeys = self.Input.meta.getAxisKeys() data = self.Input[:].wait() if self._transpose_axes: data = data.transpose() axiskeys = reversed(axiskeys) axiskeys = "".join(axiskeys) # FIXME: We assume the dataset needs to be created first. # If it already existed, this (presumably) causes an error on the DVID side. metadata = pydvid.voxels.VoxelsMetadata.create_default_metadata( data.shape, data.dtype.type, axiskeys, 0.0, "") connection = httplib.HTTPConnection(hostname) with contextlib.closing(connection): self.progressSignal(5) pydvid.voxels.create_new(connection, uuid, dataname, metadata) client = pydvid.voxels.VoxelsAccessor(connection, uuid, dataname) # For now, we send the whole darn thing at once. # TODO: Stream it over in blocks... # Send it to dvid start, stop = roiFromShape(data.shape) client.post_ndarray(start, stop, data) self.progressSignal(100)
graph = Graph() opReader = OpInputDataReader(graph=graph) opReader.WorkingDirectory.setValue( cwd ) opReader.FilePath.setValue( label_data_path ) print "Reading label volume: {}".format( label_data_path ) label_volume = opReader.Output[:].wait() axiskeys = opPixelClassification raw_shape = opPixelClassification.InputImages[lane].meta.shape if label_volume.ndim != len(raw_shape): # Append a singleton channel axis assert label_volume.ndim == len(raw_shape)-1 label_volume = label_volume[...,None] print "Applying label volume to lane #{}".format(lane) entire_volume_slicing = roiToSlice(*roiFromShape(label_volume.shape)) opPixelClassification.LabelInputs[lane][entire_volume_slicing] = label_volume ## ## REQUEST TRAINED CLASSIFIER ## opPixelClassification.FreezePredictions.setValue(False) _ = opPixelClassification.Classifier.value ## ## SAVE PROJECT ## # save project file (saves new classifier). shell.projectManager.saveProject(force_all_save=False)
def _init_fixed_dirty_roi(self): # Intentionally flipped: nothing is dirty at first. entire_roi = roiFromShape(self.Input.meta.shape) self._fixed_dirty_roi = (entire_roi[1], entire_roi[0])
def _deserializeFromHdf5(self, topGroup, groupVersion, hdf5File, projectFilePath): obj = topGroup["objects"] for imageIndex, opCarving in enumerate(self._o.innerOperators): mst = opCarving._mst for i, name in enumerate(obj): print " loading object with name='%s'" % name try: g = obj[name] fg_voxels = g["fg_voxels"] bg_voxels = g["bg_voxels"] fg_voxels = [fg_voxels[:, k] for k in range(3)] bg_voxels = [bg_voxels[:, k] for k in range(3)] sv = g["sv"].value mst.object_names[name] = i + 1 mst.object_seeds_fg_voxels[name] = fg_voxels mst.object_seeds_bg_voxels[name] = bg_voxels mst.object_lut[name] = sv mst.bg_priority[name] = g["bg_prio"].value mst.no_bias_below[name] = g["no_bias_below"].value print "[CarvingSerializer] de-serializing %s, with opCarving=%d, mst=%d" % ( name, id(opCarving), id(mst), ) print " %d voxels labeled with green seed" % fg_voxels[0].shape[0] print " %d voxels labeled with red seed" % bg_voxels[0].shape[0] print " object is made up of %d supervoxels" % sv.size print " bg priority = %f" % mst.bg_priority[name] print " no bias below = %d" % mst.no_bias_below[name] except Exception as e: print "object %s could not be loaded due to exception: %s" % (name, e) shape = opCarving.opLabelArray.Output.meta.shape dtype = opCarving.opLabelArray.Output.meta.dtype fg_voxels = None if "fg_voxels" in topGroup.keys(): fg_voxels = topGroup["fg_voxels"] fg_voxels = [fg_voxels[:, k] for k in range(3)] bg_voxels = None if "bg_voxels" in topGroup.keys(): bg_voxels = topGroup["bg_voxels"] bg_voxels = [bg_voxels[:, k] for k in range(3)] # Start with inverse roi total_roi = roiFromShape(opCarving.opLabelArray.Output.meta.shape) bounding_box_roi = numpy.array([total_roi[1][1:4], total_roi[0][1:4]]) if fg_voxels is not None and len(fg_voxels[0]) > 0: fg_bounding_box_start = numpy.array(map(numpy.min, fg_voxels)) fg_bounding_box_stop = 1 + numpy.array(map(numpy.max, fg_voxels)) bounding_box_roi[0] = numpy.minimum(bounding_box_roi[0], fg_bounding_box_start) bounding_box_roi[1] = numpy.maximum(bounding_box_roi[1], fg_bounding_box_stop) if bg_voxels is not None and len(bg_voxels[0]) > 0: bg_bounding_box_start = numpy.array(map(numpy.min, bg_voxels)) bg_bounding_box_stop = 1 + numpy.array(map(numpy.max, bg_voxels)) bounding_box_roi[0] = numpy.minimum(bounding_box_roi[0], bg_bounding_box_start) bounding_box_roi[1] = numpy.maximum(bounding_box_roi[1], bg_bounding_box_stop) if (bounding_box_roi[1] > bounding_box_roi[0]).all(): z = numpy.zeros(bounding_box_roi[1] - bounding_box_roi[0], dtype=dtype) if fg_voxels is not None: fg_voxels = fg_voxels - numpy.array([bounding_box_roi[0]]).transpose() z[fg_voxels] = 2 if bg_voxels is not None: bg_voxels = bg_voxels - numpy.array([bounding_box_roi[0]]).transpose() z[bg_voxels] = 1 bounding_box_slicing = roiToSlice(bounding_box_roi[0], bounding_box_roi[1]) opCarving.WriteSeeds[(slice(0, 1),) + bounding_box_slicing + (slice(0, 1),)] = z[ numpy.newaxis, :, :, :, numpy.newaxis ] print "restored seeds" opCarving._buildDone()
def generate_trained_project_file(new_project_path, raw_data_paths, label_data_paths, feature_selections, classifier_factory=None): """ Create a new project file from scratch, add the given raw data files, inject the corresponding labels, configure the given feature selections, and (if provided) override the classifier type ('factory'). Finally, request the classifier object from the pipeline (which forces training), and save the project. new_project_path: Where to save the new project file raw_data_paths: A list of paths to the raw data images to train with label_data_paths: A list of paths to the label image data to train with feature_selections: A matrix of bool, representing the selected features classifier_factory: Override the classifier type. Must be a subclass of either: - lazyflow.classifiers.LazyflowVectorwiseClassifierFactoryABC - lazyflow.classifiers.LazyflowPixelwiseClassifierFactoryABC """ assert len(raw_data_paths) == len( label_data_paths ), "Number of label images must match number of raw images." import ilastik_main from ilastik.workflows.pixelClassification import PixelClassificationWorkflow from lazyflow.graph import Graph from lazyflow.operators.ioOperators import OpInputDataReader from lazyflow.roi import roiToSlice, roiFromShape ## ## CREATE PROJECT ## # Manually configure the arguments to ilastik, as if they were parsed from the command line. # (Start with empty args and fill in below.) ilastik_args = ilastik_main.parse_args([]) ilastik_args.new_project = new_project_path ilastik_args.headless = True ilastik_args.workflow = "Pixel Classification" shell = ilastik_main.main(ilastik_args) assert isinstance(shell.workflow, PixelClassificationWorkflow) ## ## CONFIGURE GRAYSCALE INPUT ## data_selection_applet = shell.workflow.dataSelectionApplet # To configure data selection, start with empty cmdline args and manually fill them in data_selection_args, _ = data_selection_applet.parse_known_cmdline_args( [], PixelClassificationWorkflow.ROLE_NAMES) data_selection_args.raw_data = raw_data_paths data_selection_args.preconvert_stacks = True # Simplest thing here is to configure using cmd-line interface data_selection_applet.configure_operator_with_parsed_args( data_selection_args) ## ## APPLY FEATURE MATRIX (from matrix above) ## opFeatures = shell.workflow.featureSelectionApplet.topLevelOperator opFeatures.Scales.setValue(ScalesList) opFeatures.FeatureIds.setValue(FeatureIds) opFeatures.SelectionMatrix.setValue(feature_selections) ## ## CUSTOMIZE CLASSIFIER TYPE ## opPixelClassification = shell.workflow.pcApplet.topLevelOperator if classifier_factory is not None: opPixelClassification.ClassifierFactory.setValue(classifier_factory) ## ## READ/APPLY LABEL VOLUMES ## # Read each label volume and inject the label data into the appropriate training slot cwd = os.getcwd() max_label_class = 0 for lane, label_data_path in enumerate(label_data_paths): graph = Graph() opReader = OpInputDataReader(graph=graph) try: opReader.WorkingDirectory.setValue(cwd) opReader.FilePath.setValue(label_data_path) print("Reading label volume: {}".format(label_data_path)) label_volume = opReader.Output[:].wait() finally: opReader.cleanUp() raw_shape = opPixelClassification.InputImages[lane].meta.shape if label_volume.ndim != len(raw_shape): # Append a singleton channel axis assert label_volume.ndim == len(raw_shape) - 1 label_volume = label_volume[..., None] # Auto-calculate the max label value max_label_class = max(max_label_class, label_volume.max()) print("Applying label volume to lane #{}".format(lane)) entire_volume_slicing = roiToSlice(*roiFromShape(label_volume.shape)) opPixelClassification.LabelInputs[lane][ entire_volume_slicing] = label_volume assert max_label_class > 1, "Not enough label classes were found in your label data." label_names = list(map(str, list(range(max_label_class)))) opPixelClassification.LabelNames.setValue(label_names) ## ## TRAIN CLASSIFIER ## # Make sure the caches in the pipeline are not 'frozen'. # (This is the equivalent of 'live update' mode in the GUI.) opPixelClassification.FreezePredictions.setValue(False) # Request the classifier object from the pipeline. # This forces the pipeline to produce (train) the classifier. _ = opPixelClassification.Classifier.value ## ## SAVE PROJECT ## # save project file (includes the new classifier). shell.projectManager.saveProject(force_all_save=False)
def generate_trained_project_file( new_project_path, raw_data_paths, label_data_paths, feature_selections, classifier_factory=None ): """ Create a new project file from scratch, add the given raw data files, inject the corresponding labels, configure the given feature selections, and (if provided) override the classifier type ('factory'). Finally, request the classifier object from the pipeline (which forces training), and save the project. new_project_path: Where to save the new project file raw_data_paths: A list of paths to the raw data images to train with label_data_paths: A list of paths to the label image data to train with feature_selections: A matrix of bool, representing the selected features classifier_factory: Override the classifier type. Must be a subclass of either: - lazyflow.classifiers.LazyflowVectorwiseClassifierFactoryABC - lazyflow.classifiers.LazyflowPixelwiseClassifierFactoryABC """ assert len(raw_data_paths) == len(label_data_paths), "Number of label images must match number of raw images." import ilastik_main from ilastik.workflows.pixelClassification import PixelClassificationWorkflow from lazyflow.graph import Graph from lazyflow.operators.ioOperators import OpInputDataReader from lazyflow.roi import roiToSlice, roiFromShape ## ## CREATE PROJECT ## # Manually configure the arguments to ilastik, as if they were parsed from the command line. # (Start with empty args and fill in below.) ilastik_args = ilastik_main.parser.parse_args([]) ilastik_args.new_project = new_project_path ilastik_args.headless = True ilastik_args.workflow = "Pixel Classification" shell = ilastik_main.main(ilastik_args) assert isinstance(shell.workflow, PixelClassificationWorkflow) ## ## CONFIGURE GRAYSCALE INPUT ## data_selection_applet = shell.workflow.dataSelectionApplet # To configure data selection, start with empty cmdline args and manually fill them in data_selection_args, _ = data_selection_applet.parse_known_cmdline_args([], PixelClassificationWorkflow.ROLE_NAMES) data_selection_args.raw_data = raw_data_paths data_selection_args.preconvert_stacks = True # Simplest thing here is to configure using cmd-line interface data_selection_applet.configure_operator_with_parsed_args(data_selection_args) ## ## APPLY FEATURE MATRIX (from matrix above) ## opFeatures = shell.workflow.featureSelectionApplet.topLevelOperator opFeatures.Scales.setValue(ScalesList) opFeatures.FeatureIds.setValue(FeatureIds) opFeatures.SelectionMatrix.setValue(feature_selections) ## ## CUSTOMIZE CLASSIFIER TYPE ## opPixelClassification = shell.workflow.pcApplet.topLevelOperator if classifier_factory is not None: opPixelClassification.ClassifierFactory.setValue(classifier_factory) ## ## READ/APPLY LABEL VOLUMES ## # Read each label volume and inject the label data into the appropriate training slot cwd = os.getcwd() max_label_class = 0 for lane, label_data_path in enumerate(label_data_paths): graph = Graph() opReader = OpInputDataReader(graph=graph) try: opReader.WorkingDirectory.setValue(cwd) opReader.FilePath.setValue(label_data_path) print "Reading label volume: {}".format(label_data_path) label_volume = opReader.Output[:].wait() finally: opReader.cleanUp() raw_shape = opPixelClassification.InputImages[lane].meta.shape if label_volume.ndim != len(raw_shape): # Append a singleton channel axis assert label_volume.ndim == len(raw_shape) - 1 label_volume = label_volume[..., None] # Auto-calculate the max label value max_label_class = max(max_label_class, label_volume.max()) print "Applying label volume to lane #{}".format(lane) entire_volume_slicing = roiToSlice(*roiFromShape(label_volume.shape)) opPixelClassification.LabelInputs[lane][entire_volume_slicing] = label_volume assert max_label_class > 1, "Not enough label classes were found in your label data." label_names = map(str, range(max_label_class)) opPixelClassification.LabelNames.setValue(label_names) ## ## TRAIN CLASSIFIER ## # Make sure the caches in the pipeline are not 'frozen'. # (This is the equivalent of 'live update' mode in the GUI.) opPixelClassification.FreezePredictions.setValue(False) # Request the classifier object from the pipeline. # This forces the pipeline to produce (train) the classifier. _ = opPixelClassification.Classifier.value ## ## SAVE PROJECT ## # save project file (includes the new classifier). shell.projectManager.saveProject(force_all_save=False)
def import_labeling_layer(labelLayer, labelingSlots, parent_widget=None): """ Prompt the user for layer import settings, and perform the layer import. :param labelLayer: The top label layer source :param labelingSlots: An instance of LabelingGui.LabelingSlots :param parent_widget: The Qt GUI parent object """ writeSeeds = labelingSlots.labelInput assert isinstance(writeSeeds, lazyflow.graph.Slot), "slot is of type %r" % (type(writeSeeds)) opLabels = writeSeeds.getRealOperator() assert isinstance(opLabels, lazyflow.graph.Operator), "slot's operator is of type %r" % (type(opLabels)) recentlyImported = PreferencesManager().get('labeling', 'recently imported') mostRecentProjectPath = PreferencesManager().get('shell', 'recently opened') mostRecentImageFile = PreferencesManager().get( 'DataSelection', 'recent image' ) if recentlyImported: defaultDirectory = os.path.split(recentlyImported)[0] elif mostRecentProjectPath: defaultDirectory = os.path.split(mostRecentProjectPath)[0] elif mostRecentImageFile: defaultDirectory = os.path.split(mostRecentImageFile)[0] else: defaultDirectory = os.path.expanduser('~') fileNames = DataSelectionGui.getImageFileNamesToOpen(parent_widget, defaultDirectory) fileNames = map(str, fileNames) if not fileNames: return PreferencesManager().set('labeling', 'recently imported', fileNames[0]) try: # Initialize operators opImport = OpInputDataReader( parent=opLabels.parent ) opCache = OpArrayCache( parent=opLabels.parent ) opMetadataInjector = OpMetadataInjector( parent=opLabels.parent ) opReorderAxes = OpReorderAxes( parent=opLabels.parent ) # Set up the pipeline as follows: # # opImport --> opCache --> opMetadataInjector --------> opReorderAxes --(inject via setInSlot)--> labelInput # / / # User-specified axisorder labelInput.meta.axistags opImport.WorkingDirectory.setValue(defaultDirectory) opImport.FilePath.setValue(fileNames[0] if len(fileNames) == 1 else os.path.pathsep.join(fileNames)) assert opImport.Output.ready() opCache.blockShape.setValue( opImport.Output.meta.shape ) opCache.Input.connect( opImport.Output ) assert opCache.Output.ready() opMetadataInjector.Input.connect( opCache.Output ) metadata = opCache.Output.meta.copy() opMetadataInjector.Metadata.setValue( metadata ) opReorderAxes.Input.connect( opMetadataInjector.Output ) # Transpose the axes for assignment to the labeling operator. opReorderAxes.AxisOrder.setValue( writeSeeds.meta.getAxisKeys() ) # We'll show a little window with a busy indicator while the data is loading busy_dlg = QProgressDialog(parent=parent_widget) busy_dlg.setLabelText("Importing Label Data...") busy_dlg.setCancelButton(None) busy_dlg.setMinimum(100) busy_dlg.setMaximum(100) def close_busy_dlg(*args): QApplication.postEvent(busy_dlg, QCloseEvent()) # Load the data from file into our cache # When it's done loading, close the progress dialog. req = opCache.Output[:] req.notify_finished( close_busy_dlg ) req.notify_failed( close_busy_dlg ) req.submit() busy_dlg.exec_() readData = req.result maxLabels = len(labelingSlots.labelNames.value) # Can't use return_counts feature because that requires numpy >= 1.9 #unique_read_labels, readLabelCounts = numpy.unique(readData, return_counts=True) # This does the same as the above, albeit slower, and probably with more ram. unique_read_labels = numpy.unique(readData) readLabelCounts = vigra_bincount(readData)[unique_read_labels] labelInfo = (maxLabels, (unique_read_labels, readLabelCounts)) del readData # Ask the user how to interpret the data. settingsDlg = LabelImportOptionsDlg( parent_widget, fileNames, opMetadataInjector.Output, labelingSlots.labelInput, labelInfo ) def handle_updated_axes(): # The user is specifying a new interpretation of the file's axes updated_axisorder = str(settingsDlg.axesEdit.text()) metadata = opMetadataInjector.Metadata.value.copy() metadata.axistags = vigra.defaultAxistags(updated_axisorder) opMetadataInjector.Metadata.setValue( metadata ) if opReorderAxes._invalid_axes: settingsDlg.buttonBox.button(QDialogButtonBox.Ok).setEnabled(False) # Red background settingsDlg.axesEdit.setStyleSheet("QLineEdit { background: rgb(255, 128, 128);" "selection-background-color: rgb(128, 128, 255); }") settingsDlg.axesEdit.editingFinished.connect( handle_updated_axes ) # Initialize handle_updated_axes() dlg_result = settingsDlg.exec_() if dlg_result != LabelImportOptionsDlg.Accepted: return # Get user's chosen label mapping from dlg labelMapping = settingsDlg.labelMapping # Get user's chosen offsets. # Offsets in dlg only include the file axes, not the 5D axes expected by the label input, # so expand them to full 5D axes_5d = opReorderAxes.Output.meta.getAxisKeys() tagged_offsets = collections.OrderedDict( zip( axes_5d, [0]*len(axes_5d) ) ) tagged_offsets.update( dict( zip( opMetadataInjector.Output.meta.getAxisKeys(), settingsDlg.imageOffsets ) ) ) imageOffsets = tagged_offsets.values() # Optimization if mapping is identity if labelMapping.keys() == labelMapping.values(): labelMapping = None # This will be fast (it's already cached) label_data = opReorderAxes.Output[:].wait() # Map input labels to output labels if labelMapping: # There are other ways to do a relabeling (e.g skimage.segmentation.relabel_sequential) # But this supports potentially huge values of unique_read_labels (in the billions), # without needing GB of RAM. mapping_indexes = numpy.searchsorted(unique_read_labels, label_data) new_labels = numpy.array([labelMapping[x] for x in unique_read_labels]) label_data[:] = new_labels[mapping_indexes] label_roi = numpy.array( roiFromShape(opReorderAxes.Output.meta.shape) ) label_roi += imageOffsets label_slice = roiToSlice(*label_roi) writeSeeds[label_slice] = label_data finally: opReorderAxes.cleanUp() opMetadataInjector.cleanUp() opCache.cleanUp() opImport.cleanUp()
def execute(self, slot, subindex, roi, result): classifier = self.Classifier.value # Training operator may return 'None' if there was no data to train with skip_prediction = (classifier is None) # Shortcut: If the mask is totally zero, skip this request entirely if not skip_prediction and self.PredictionMask.ready(): mask_roi = numpy.array((roi.start, roi.stop)) mask_roi[:, -1:] = [[0], [1]] start, stop = map(tuple, mask_roi) mask = self.PredictionMask(start, stop).wait() skip_prediction = not numpy.any(mask) if skip_prediction: result[:] = 0.0 return result assert issubclass(type(classifier), LazyflowPixelwiseClassifierABC), \ "Classifier is of type {}, which does not satisfy the LazyflowPixelwiseClassifierABC interface."\ "".format( type(classifier) ) upstream_roi = (roi.start, roi.stop) # Ask for the halo needed by the classifier axiskeys = self.Image.meta.getAxisKeys() halo_shape = classifier.get_halo_shape(axiskeys) assert len(halo_shape) == len(upstream_roi[0]) assert halo_shape[ -1] == 0, "Didn't expect a non-zero halo for channel dimension." # Expand block by halo, then clip to image bounds upstream_roi = numpy.array(upstream_roi) upstream_roi[0] -= halo_shape upstream_roi[1] += halo_shape upstream_roi = getIntersection(upstream_roi, roiFromShape(self.Image.meta.shape)) upstream_roi = numpy.asarray(upstream_roi) # Determine how to extract the data from the result (without the halo) downstream_roi = numpy.array((roi.start, roi.stop)) downstream_channels = self.PMaps.meta.shape[-1] roi_within_result = downstream_roi - upstream_roi[0] roi_within_result[:, -1] = [0, downstream_channels] # Request all upstream channels input_channels = self.Image.meta.shape[-1] upstream_roi[:, -1] = [0, input_channels] # Request the data input_data = self.Image(*upstream_roi).wait() axistags = self.Image.meta.axistags probabilities = classifier.predict_probabilities_pixelwise( input_data, axistags) # We're expecting a channel for each label class. # If we didn't provide at least one sample for each label, # we may get back fewer channels. if probabilities.shape[-1] != self.PMaps.meta.shape[-1]: # Copy to an array of the correct shape # This is slow, but it's an unusual case assert probabilities.shape[-1] == len(classifier.known_classes) full_probabilities = numpy.zeros(probabilities.shape[:-1] + (self.PMaps.meta.shape[-1], ), dtype=numpy.float32) for i, label in enumerate(classifier.known_classes): full_probabilities[..., label - 1] = probabilities[..., i] probabilities = full_probabilities # Extract requested region (discard halo) probabilities = probabilities[roiToSlice(*roi_within_result)] # Copy only the prediction channels the client requested. result[...] = probabilities[..., roi.start[-1]:roi.stop[-1]] return result
def create_slice_req( index ): roi = roiFromShape(slice_shape) roi += index*slice_step return self.Input(*roi)
def import_labeling_layer(labelLayer, labelingSlots, parent_widget=None): """ Prompt the user for layer import settings, and perform the layer import. :param labelLayer: The top label layer source :param labelingSlots: An instance of LabelingGui.LabelingSlots :param parent_widget: The Qt GUI parent object """ writeSeeds = labelingSlots.labelInput assert isinstance( writeSeeds, lazyflow.graph.Slot), "slot is of type %r" % (type(writeSeeds)) opLabels = writeSeeds.getRealOperator() assert isinstance(opLabels, lazyflow.graph.Operator ), "slot's operator is of type %r" % (type(opLabels)) recentlyImported = PreferencesManager().get('labeling', 'recently imported') mostRecentProjectPath = PreferencesManager().get('shell', 'recently opened') mostRecentImageFile = PreferencesManager().get('DataSelection', 'recent image') if recentlyImported: defaultDirectory = os.path.split(recentlyImported)[0] elif mostRecentProjectPath: defaultDirectory = os.path.split(mostRecentProjectPath)[0] elif mostRecentImageFile: defaultDirectory = os.path.split(mostRecentImageFile)[0] else: defaultDirectory = os.path.expanduser('~') fileNames = DataSelectionGui.getImageFileNamesToOpen( parent_widget, defaultDirectory) fileNames = map(str, fileNames) if not fileNames: return PreferencesManager().set('labeling', 'recently imported', fileNames[0]) try: # Initialize operators opImport = OpInputDataReader(parent=opLabels.parent) opCache = OpArrayCache(parent=opLabels.parent) opMetadataInjector = OpMetadataInjector(parent=opLabels.parent) opReorderAxes = OpReorderAxes(parent=opLabels.parent) # Set up the pipeline as follows: # # opImport --> opCache --> opMetadataInjector --------> opReorderAxes --(inject via setInSlot)--> labelInput # / / # User-specified axisorder labelInput.meta.axistags opImport.WorkingDirectory.setValue(defaultDirectory) opImport.FilePath.setValue(fileNames[0] if len(fileNames) == 1 else os.path.pathsep.join(fileNames)) assert opImport.Output.ready() opCache.blockShape.setValue(opImport.Output.meta.shape) opCache.Input.connect(opImport.Output) assert opCache.Output.ready() opMetadataInjector.Input.connect(opCache.Output) metadata = opCache.Output.meta.copy() opMetadataInjector.Metadata.setValue(metadata) opReorderAxes.Input.connect(opMetadataInjector.Output) # Transpose the axes for assignment to the labeling operator. opReorderAxes.AxisOrder.setValue(writeSeeds.meta.getAxisKeys()) # We'll show a little window with a busy indicator while the data is loading busy_dlg = QProgressDialog(parent=parent_widget) busy_dlg.setLabelText("Importing Label Data...") busy_dlg.setCancelButton(None) busy_dlg.setMinimum(100) busy_dlg.setMaximum(100) def close_busy_dlg(*args): QApplication.postEvent(busy_dlg, QCloseEvent()) # Load the data from file into our cache # When it's done loading, close the progress dialog. req = opCache.Output[:] req.notify_finished(close_busy_dlg) req.notify_failed(close_busy_dlg) req.submit() busy_dlg.exec_() readData = req.result maxLabels = len(labelingSlots.labelNames.value) # Can't use return_counts feature because that requires numpy >= 1.9 #unique_read_labels, readLabelCounts = numpy.unique(readData, return_counts=True) # This does the same as the above, albeit slower, and probably with more ram. unique_read_labels = numpy.unique(readData) readLabelCounts = numpy.bincount(readData.flat)[unique_read_labels] labelInfo = (maxLabels, (unique_read_labels, readLabelCounts)) del readData # Ask the user how to interpret the data. settingsDlg = LabelImportOptionsDlg(parent_widget, fileNames, opMetadataInjector.Output, labelingSlots.labelInput, labelInfo) def handle_updated_axes(): # The user is specifying a new interpretation of the file's axes updated_axisorder = str(settingsDlg.axesEdit.text()) metadata = opMetadataInjector.Metadata.value.copy() metadata.axistags = vigra.defaultAxistags(updated_axisorder) opMetadataInjector.Metadata.setValue(metadata) settingsDlg.axesEdit.editingFinished.connect(handle_updated_axes) dlg_result = settingsDlg.exec_() if dlg_result != LabelImportOptionsDlg.Accepted: return # Get user's chosen label mapping from dlg labelMapping = settingsDlg.labelMapping # Get user's chosen offsets. # Offsets in dlg only include the file axes, not the 5D axes expected by the label input, # so expand them to full 5D axes_5d = opReorderAxes.Output.meta.getAxisKeys() tagged_offsets = collections.OrderedDict( zip(axes_5d, [0] * len(axes_5d))) tagged_offsets.update( dict( zip(opMetadataInjector.Output.meta.getAxisKeys(), settingsDlg.imageOffsets))) imageOffsets = tagged_offsets.values() # Optimization if mapping is identity if labelMapping.keys() == labelMapping.values(): labelMapping = None # This will be fast (it's already cached) label_data = opReorderAxes.Output[:].wait() # Map input labels to output labels if labelMapping: # There are other ways to do a relabeling (e.g skimage.segmentation.relabel_sequential) # But this supports potentially huge values of unique_read_labels (in the billions), # without needing GB of RAM. mapping_indexes = numpy.searchsorted(unique_read_labels, label_data) new_labels = numpy.array( [labelMapping[x] for x in unique_read_labels]) label_data[:] = new_labels[mapping_indexes] label_roi = numpy.array(roiFromShape(opReorderAxes.Output.meta.shape)) label_roi += imageOffsets label_slice = roiToSlice(*label_roi) writeSeeds[label_slice] = label_data finally: opReorderAxes.cleanUp() opMetadataInjector.cleanUp() opCache.cleanUp() opImport.cleanUp()