def testSetDirty(self): g = Graph() vol = np.zeros((5, 2, 200, 100, 10)) vol = vol.astype(np.uint8) vol = vigra.taggedView(vol, axistags="tcxyz") vol[:200, ...] = 1 vol[800:, ...] = 1 op = OpLabelVolume(graph=g) op.Method.setValue(self.method) op.Input.setValue(vol) opCheck = DirtyAssert(graph=g) opCheck.Input.connect(op.Output) opCheck.willBeDirty(1, 1) roi = SubRegion(op.Input, start=(1, 1, 0, 0, 0), stop=(2, 2, 200, 100, 10)) with pytest.raises(PropagateDirtyCalled): op.Input.setDirty(roi) opCheck.Input.disconnect() opCheck.Input.connect(op.CachedOutput) opCheck.willBeDirty(1, 1) out = op.Output[...].wait() roi = SubRegion(op.Input, start=(1, 1, 0, 0, 0), stop=(2, 2, 200, 100, 10)) with pytest.raises(PropagateDirtyCalled): op.Input.setDirty(roi)
def _purge_label(self, label_to_purge): """ Scan through all labeled pixels. (1) Clear all pixels of the given value (set to 0) (2) Decrement all labels above that value so the set of stored labels is consecutive """ changed_block_rois = [] #stored_block_rois = self.CleanBlocks.value stored_block_roi_destination = [None] self.execute(self.CleanBlocks, (), SubRegion( self.Output, (0,),(1,) ), stored_block_roi_destination) stored_block_rois = stored_block_roi_destination[0] for block_roi in stored_block_rois: # Get data block_shape = numpy.subtract( block_roi[1], block_roi[0] ) block = numpy.ndarray( shape=block_shape, dtype=self.Output.meta.dtype ) self.execute(self.Output, (), SubRegion( self.Output, *block_roi ), block) # Locate pixels to change matching_label_coords = numpy.nonzero( block == label_to_purge ) coords_to_decrement = block > label_to_purge # Change the data block[matching_label_coords] = 0 block = numpy.where( coords_to_decrement, block-1, block ) # Update cache with the new data (only if something really changed) if len(matching_label_coords[0]) > 0 or len(coords_to_decrement[0]) > 0: super( OpCompressedUserLabelArray, self )._setInSlotInput( self.Input, (), SubRegion( self.Output, *block_roi ), block, store_zero_blocks=False ) changed_block_rois.append( block_roi ) for block_roi in changed_block_rois: # FIXME: Shouldn't this dirty notification be handled in OpCompressedCache? self.Output.setDirty( *block_roi )
def _export(): t_from = None # determine from_time (it could has been changed in the GUI meanwhile) for t_from, label2color_at in enumerate( self.mainOperator.label2color): if len(label2color_at) == 0: continue else: break if t_from == None: return print "Saving first label image..." key = [] for idx, flag in enumerate( axisTagsToString( self.mainOperator.LabelImage.meta.axistags)): if flag is 't': key.append(slice(t_from, t_from + 1)) elif flag is 'c': key.append(slice(0, 1)) else: key.append( slice(0, self.mainOperator.LabelImage.meta.shape[idx])) roi = SubRegion(self.mainOperator.LabelImage, key) labelImage = self.mainOperator.LabelImage.get(roi).wait() labelImage = labelImage[0, ..., 0] try: write_events([], str(directory), t_from, labelImage) events = self.mainOperator.events print "Saving events..." print "Length of events " + str(len(events)) num_files = float(len(events)) for i, events_at in enumerate(events): t = t_from + i key[0] = slice(t + 1, t + 2) roi = SubRegion(self.mainOperator.LabelImage, key) labelImage = self.mainOperator.LabelImage.get(roi).wait() labelImage = labelImage[0, ..., 0] if self.withMergers: write_events(events_at, str(directory), t + 1, labelImage, self.mainOperator.mergers) else: write_events(events_at, str(directory), t + 1, labelImage) _handle_progress(i / num_files * 100) except IOError as e: self._criticalMessage("Cannot export the tracking results. Maybe these files already exist. "\ "Please delete them or choose a different directory.") return
def testNoFullBlocks(self): op = OpSplitRequestsBlockwise(False, graph=self.g) op.Input.connect(self.piper.Output) op.BlockShape.setValue((10, 15, 20)) op.Output[0:20, 30:45, 10:30].wait() slot = self.piper.Output expected = [SubRegion(slot, (0, 30, 10), (10, 45, 20)), SubRegion(slot, (0, 30, 20), (10, 45, 30)), SubRegion(slot, (10, 30, 10), (20, 45, 20)), SubRegion(slot, (10, 30, 20), (20, 45, 30))] for req in self.piper.requests: print(req) for roi in expected: filtered = [x for x in self.piper.requests if x == roi] assert len(filtered) == 1, "missing roi {}".format(roi) self.piper.requests = [] op.Output[5:14, 32:44, 17:21].wait() expected = [SubRegion(slot, (5, 32, 17), (10, 44, 20)), SubRegion(slot, (5, 32, 20), (10, 44, 21)), SubRegion(slot, (10, 32, 17), (14, 44, 20)), SubRegion(slot, (10, 32, 20), (14, 44, 21))] for roi in expected: filtered = [x for x in self.piper.requests if x == roi] assert len(filtered) == 1, "missing roi {}".format(roi)
def _purge_label(self, label_to_purge, decrement_remaining, replacement_value=0): """ Scan through all labeled pixels. (1) Reassign all pixels of the given value (set to replacement_value) (2) If decrement_remaining=True, decrement all labels above that value so the set of stored labels remains consecutive. Note that the decrement is performed AFTER replacement. """ changed_block_rois = [] # stored_block_rois = self.CleanBlocks.value stored_block_roi_destination = [None] self.execute(self.CleanBlocks, (), SubRegion(self.Output, (0, ), (1, )), stored_block_roi_destination) stored_block_rois = stored_block_roi_destination[0] for block_roi in stored_block_rois: # Get data block_shape = numpy.subtract(block_roi[1], block_roi[0]) block = self.Output.stype.allocateDestination( SubRegion(self.Output, *roiFromShape(block_shape))) self.execute(self.Output, (), SubRegion(self.Output, *block_roi), block) # Locate pixels to change matching_label_coords = numpy.nonzero(block == label_to_purge) # Change the data block[matching_label_coords] = replacement_value coords_to_decrement = block > label_to_purge if decrement_remaining: block[coords_to_decrement] -= numpy.uint8(1) # Update cache with the new data (only if something really changed) if len(matching_label_coords[0]) > 0 or ( decrement_remaining and coords_to_decrement.sum() > 0): super(OpCompressedUserLabelArray, self)._setInSlotInput(self.Input, (), SubRegion(self.Output, *block_roi), block, store_zero_blocks=False) changed_block_rois.append(block_roi) for block_roi in changed_block_rois: # FIXME: Shouldn't this dirty notification be handled in OpUnmanagedCompressedCache? self.Output.setDirty(*block_roi)
def testDirty(self): opSlicer = self.opSlicer dirtyRois = {} def handleDirty(i, slot, roi): assert opSlicer.Slices[i] == slot dirtyRois[i] = roi for i, slot in enumerate(opSlicer.Slices): slot.notifyDirty( partial(handleDirty, i) ) dirtyInputRoi = SubRegion(slot=opSlicer.Input, start=[4,3,2,1], stop=[6,5,4,3]) opSlicer.Input.setDirty(dirtyInputRoi) assert len(dirtyRois) == 2 assert dirtyRois[1].start == [4,3,2,0] assert dirtyRois[1].stop == [6,5,4,1] assert dirtyRois[2].start == [4,3,2,0] assert dirtyRois[2].stop == [6,5,4,1] # Reset dirtyRois = {} opSlicer.AxisFlag.setValue('x') assert len(dirtyRois) == 3 assert dirtyRois[0].start == [0,0,0,0] assert dirtyRois[0].stop == [10,10,10,3] assert dirtyRois[1].start == [0,0,0,0] assert dirtyRois[1].stop == [10,10,10,3] assert dirtyRois[2].start == [0,0,0,0] assert dirtyRois[2].stop == [10,10,10,3]
def _getObjects(self, trange, misdet_idx): filtered_labels = {} oid2tids = {} alltids = set() for t in range(trange[0], trange[1]): count = 0 filtered_labels[t] = [] oid2tids[t] = {} troi = SubRegion( self.LabelImage, start=[t] + [0] * len(self.LabelImage.meta.shape[1:]), stop=[t + 1] + list(self.LabelImage.meta.shape[1:]), ) max_oid = np.max(self.LabelImage.get(troi).wait()) for idx in range(max_oid + 1): oid = int(idx) + 1 if t in list(self.labels.keys()) and oid in list( self.labels[t].keys()): if misdet_idx not in self.labels[t][oid]: oid2tids[t][oid] = self.labels[t][oid] for l in self.labels[t][oid]: alltids.add(l) count += 1 logger.info("at timestep {}, {} traxels found".format(t, count)) return oid2tids, alltids
def ingestData(self, slot): """ Read the data from the given slot and copy it into this cache. The rules about special pixel meanings apply here, just like setInSlot Returns: the max label found in the slot. """ assert self._blockshape is not None assert self.Input.meta.shape == slot.meta.shape max_label = 0 # Get logical blocking. block_starts = getIntersectingBlocks( self._blockshape, roiFromShape(self.Input.meta.shape) ) block_starts = map( tuple, block_starts ) # Write each block for block_start in block_starts: block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, block_start ) # Request the block data block_data = slot(*block_roi).wait() # Write into the array subregion_roi = SubRegion(self.Input, *block_roi) cleaned_block_data = self._setInSlotInput( self.Input, (), subregion_roi, block_data ) max_label = max( max_label, cleaned_block_data.max() ) return max_label
def _chunkIndexToRoi(self, index): shape = self._shape start = self._chunkShape * np.asarray(index) stop = self._chunkShape * (np.asarray(index) + 1) stop = np.where(stop > shape, shape, stop) roi = SubRegion(self.Input, start=tuple(start), stop=tuple(stop)) return roi
def _export(): num_files = float(len(label2color)) for t, label2color_at in enumerate(label2color): if len(label2color_at) == 0: continue print 'exporting tiffs for t = ' + str(t) roi = SubRegion(self.mainOperator.LabelImage, start=[ t, ] + 4 * [ 0, ], stop=[ t + 1, ] + list(lshape[1:])) labelImage = self.mainOperator.LabelImage.get(roi).wait() relabeled = relabel(labelImage[0, ..., 0], label2color_at) for i in range(relabeled.shape[2]): out_im = relabeled[:, :, i] out_fn = str(directory) + '/vis_t' + str(t).zfill( 4) + '_z' + str(i).zfill(4) + '.tif' vigra.impex.writeImage(np.asarray(out_im, dtype=np.uint32), out_fn) _handle_progress(t / num_files * 100) print 'Tiffs exported.'
def copy_block(full_block_roi, clipped_block_roi): full_block_roi = numpy.asarray(full_block_roi) clipped_block_roi = numpy.asarray(clipped_block_roi) output_roi = numpy.asarray(clipped_block_roi) - roi.start # If data data exists already or we can just fetch it without needing extra scratch space, # just call the base class block_roi = self._get_containing_block_roi(clipped_block_roi) if block_roi is not None or (full_block_roi == clipped_block_roi).all(): self._execute_Output_impl(clipped_block_roi, result[roiToSlice(*output_roi)]) elif self.Input.meta.dontcache: # Data isn't in the cache, but we don't need it in the cache anyway. self.Input(*clipped_block_roi).writeInto( result[roiToSlice(*output_roi)]).block() else: # Data doesn't exist yet in the cache. # Request the full block, but then discard the parts we don't need. # (We use allocateDestination() here to support MaskedArray types.) # TODO: We should probably just get rid of MaskedArray support altogether... full_block_data = self.Output.stype.allocateDestination( SubRegion(self.Output, *full_block_roi)) self._execute_Output_impl(full_block_roi, full_block_data) roi_within_block = clipped_block_roi - full_block_roi[0] self.Output.stype.copy_data( result[roiToSlice(*output_roi)], full_block_data[roiToSlice(*roi_within_block)])
def _label(self, roi, result): result = vigra.taggedView(result, axistags=self.Output.meta.axistags) # get the background values bg = self.Background[...].wait() bg = vigra.taggedView(bg, axistags=self.Background.meta.axistags) bg = bg.withAxes(*"ct") assert np.all( self.Background.meta.shape[0] == self.Input.meta.shape[0] ), "Shape of background values incompatible to shape of Input" assert np.all( self.Background.meta.shape[4] == self.Input.meta.shape[4] ), "Shape of background values incompatible to shape of Input" # do labeling in parallel over channels and time slices pool = RequestPool() start = np.asarray(roi.start, dtype=np.int) stop = np.asarray(roi.stop, dtype=np.int) for ti, t in enumerate(range(roi.start[0], roi.stop[0])): start[0], stop[0] = t, t + 1 for ci, c in enumerate(range(roi.start[4], roi.stop[4])): start[4], stop[4] = c, c + 1 newRoi = SubRegion(self.Output, start=tuple(start), stop=tuple(stop)) resView = result[ti, ..., ci].withAxes(*"xyz") req = Request(partial(self._label3d, newRoi, bg[c, t], resView)) pool.add(req) logger.debug("{}: Computing connected components for ROI {} ...".format(self.name, roi)) pool.wait() pool.clean() logger.debug("{}: Connected components computed.".format(self.name))
def test(self, position5d_start, position5d_stop): print "test" roi = SubRegion(self.op.Density, position5d_start, position5d_stop) key = roi.toSlice() key = tuple(k for k in key if k != slice(0,0, None)) newKey = [] for k in key: if k != slice(0,0,None): if k.stop < k.start: k = slice(k.stop, k.start) newKey.append(k) newKey = tuple(newKey) try: density = numpy.sum(self.op.Density[newKey].wait()) / 255 strdensity = "{0:.2f}".format(density) self._labelControlUi.CountText.setText(strdensity) except: pass
def test2(self, position5d_start, position5d_stop): print "test2" roi = SubRegion(self.op.LabelInputs, position5d_start, position5d_stop) key = roi.toSlice() #key = tuple(k for k in key if k != slice(0,0, None)) newKey = [] for k in key: if k.stop < k.start: k = slice(k.stop, k.start) newKey.append(k) newKey = tuple(newKey) self.boxes[self.activeBox] = newKey #self.op.BoxLabelImages[newKey] = self.activeBox + 2 #self.op.BoxLabelImages labelShape = tuple([position5d_stop[i] + 1 - position5d_start[i] for i in range(5)]) labels = numpy.ones((labelShape), dtype = numpy.uint8) * (self.activeBox + 3) self.boxlabelsrc.put(newKey, labels)
def execute(self, slot, subindex, roi, result): sigma = 3.0 roi_with_halo, result_roi = enlargeRoiForHalo( roi.start, roi.stop, self.Input.meta.shape, sigma, return_result_roi=True ) start, stop = roi_with_halo newroi = SubRegion(self.Input, start=start, stop=stop) data = self.Input.get(newroi).wait() time.sleep(self.delay) result[:] = data[roiToSlice(*result_roi)]
def propagateDirty(self, slot, subindex, roi): key = roi.toSlice() if slot == self.inputs["Input"] and self._forward_dirty: if not self._fixed: self.outputs["Output"].setDirty(key) elif self._blockShape is not None: # Find the block key roi = SubRegion(slot, pslice=key) start, stop = roi.start, roi.stop blockStart = (start / self._blockShape) blockStop = (stop * 1.0 / self._blockShape).ceil() with self._lock: # check whether the dirty region encompasses the whole cache if (blockStart == 0).all() and (blockStop == self._dirtyShape).all(): self._fixed_all_dirty = True # shortcut, if everything is dirty already, dont loop over the blocks if self._fixed_all_dirty is False: innerBlocks = self._get_block_numbers( blockStart, blockStop) for block_index in innerBlocks.flat: self._fixed_dirty_blocks.add(block_index) if slot == self.fixAtCurrent: self._fixed = self.fixAtCurrent.value if not self._fixed: # We've become unfixed. # Take the superset of all the blocks that became dirty in the meantime and notify our output dirtystart, dirtystop = (None, None) with self._lock: if self._fixed_all_dirty is True: dirtystop = self.Output.meta.shape dirtystart = [0] * len(self.Output.meta.shape) elif len(self._fixed_dirty_blocks) > 0: dirtystart = self.Output.meta.shape dirtystop = [0] * len(self.Output.meta.shape) for block_index in self._fixed_dirty_blocks: offset = self._blockShape * self._get_block_multi_index( block_index) bigstart = offset bigstop = numpy.minimum(offset + self._blockShape, self.Output.meta.shape) dirtystart = numpy.minimum(bigstart, dirtystart) dirtystop = numpy.maximum(bigstop, dirtystop) self._fixed_dirty_blocks = set() # reset all dirty state to false self._fixed_all_dirty = False if dirtystart is not None: self.Output.setDirty(dirtystart, dirtystop)
def test(self, position5d_start, position5d_stop): from lazyflow.rtype import SubRegion import numpy from sitecustomize import debug_trace roi = SubRegion(self.op.Density, position5d_start, position5d_stop) key = roi.toSlice() key = tuple(k for k in key if k != slice(0,0, None)) newKey = [] for k in key: if k != slice(0,0,None): if k.stop < k.start: k = slice(k.stop, k.start) newKey.append(k) newKey = tuple(newKey) try: density = numpy.sum(self.op.Density[newKey].wait()) self._labelControlUi.CountText.setText(str(density)) except: debug_trace()
def testPassThroughAndFixed(self): self.op.fixAtCurrent.setValue(False) # we want the last operator to throw an exception when its # propagateDirty is called, so we can catch it below def foo(): raise PropagateDirtyCalled() self.call.function = foo with self.assertRaises(PropagateDirtyCalled): self.piper.Input.setValue(self.vol) roi = self.call.roi assert_array_equal(roi.start, (0, 0, 0)) assert_array_equal(roi.stop, self.vol.shape) with self.assertRaises(PropagateDirtyCalled): self.piper.Input.setDirty(slice(None)) roi = self.call.roi assert_array_equal(roi.start, (0, 0, 0)) assert_array_equal(roi.stop, self.vol.shape) self.op.fixAtCurrent.setValue(True) try: roi1 = SubRegion(self.piper.Input, (0, 10, 20), (15, 15, 35)) self.piper.Input.setDirty(roi1) roi2 = SubRegion(self.piper.Input, (15, 9, 1), (20, 14, 20)) self.piper.Input.setDirty(roi2) except PropagateDirtyCalled: raise AssertionError("dirtyness is not fixed") with self.assertRaises(PropagateDirtyCalled): self.op.fixAtCurrent.setValue(False) roi = self.call.roi assert_array_equal(roi.start, (0, 9, 1)) assert_array_equal(roi.stop, (20, 15, 35))
def testFullBlocks(self): op = OpSplitRequestsBlockwise(True, graph=self.g) op.Input.connect(self.piper.Output) op.BlockShape.setValue((10, 15, 20)) op.Output[0:20, 30:45, 10:30].wait() slot = self.piper.Output expected = [SubRegion(slot, (0, 30, 0), (10, 45, 20)), SubRegion(slot, (0, 30, 20), (10, 45, 40)), SubRegion(slot, (10, 30, 0), (20, 45, 20)), SubRegion(slot, (10, 30, 20), (20, 45, 40))] for roi in expected: filtered = filter(lambda x: x == roi, self.piper.requests) assert len(filtered) == 1, "missing roi {}".format(roi) self.piper.requests = [] op.Output[5:14, 32:44, 17:21].wait() for roi in expected: filtered = filter(lambda x: x == roi, self.piper.requests) assert len(filtered) == 1, "missing roi {}".format(roi)
def _calculateFeatures(self, interactive=True): mainOperator = self.topLevelOperatorView mainOperator.ObjectCenterImage.setDirty( SubRegion(mainOperator.ObjectCenterImage)) current_t = self.editor.posModel.time def _handle_all_finished(*args): self._lock.acquire() self.applet.progressSignal(100) self.topLevelOperatorView._opRegFeats.fixed = True feats = self.topLevelOperatorView.RegionFeatures[0].wait() nfeatures = 0 nchannels = 0 try: for pname, pfeats in feats[0].items(): if pname != default_features_key: for featname, feat in pfeats.items(): nchannels += feat.shape[1] nfeatures += 1 if interactive: self._drawer.featuresSelected.setText( "{} features computed, {} channels in total".format( nfeatures, nchannels)) logger.info('Object Extraction: done.') success = True except AttributeError: if interactive: self._drawer.featuresSelected.setText( "Feature computation failed (most likely due to memory issues)" ) logger.error('Object Extraction: failed.') success = False self.applet.appletStateUpdateRequested() self._lock.release() self.applet.progressSignal(0) self.applet.progressSignal(-1) reqs = [] self.already_done = 0 req = mainOperator.RegionFeatures([current_t]) req.submit() req.notify_failed(self.handleFeatureComputationFailure) req.notify_finished(_handle_all_finished) reqs.append(req)
def _prepareTaskInfos(self, roiList): # Divide up the workload into large pieces logger.info("Dividing into {} node jobs.".format(len(roiList))) taskInfos = collections.OrderedDict() for roiIndex, roi in enumerate(roiList): roi = (tuple(roi[0]), tuple(roi[1])) taskInfo = OpClusterize.TaskInfo() taskInfo.subregion = SubRegion(None, start=roi[0], stop=roi[1]) taskName = "J{:02}".format(roiIndex) commandArgs = [] commandArgs.append("--option_config_file=" + self.ConfigFilePath.value) commandArgs.append("--project=" + self.ProjectFilePath.value) commandArgs.append("--_node_work_=\"" + Roi.dumps(taskInfo.subregion) + "\"") commandArgs.append("--process_name={}".format(taskName)) commandArgs.append("--output_description_file={}".format( self.OutputDatasetDescription.value)) for slot in self.SecondaryOutputDescriptions: commandArgs.append( "--secondary_output_description_file={}".format( slot.value)) # Check the command format string: We need to know where to put our args... commandFormat = self._config.command_format assert commandFormat.find("{task_args}") != -1 # Output log directory might be a relative path (relative to config file) absLogDir, relLogDir = getPathVariants( self._config.output_log_directory, os.path.split(self.ConfigFilePath.value)[0]) taskOutputLogFilename = taskName + ".log" taskOutputLogPath = os.path.join(absLogDir, taskOutputLogFilename) allArgs = " " + " ".join(commandArgs) + " " taskInfo.taskName = taskName taskInfo.command = commandFormat.format( task_args=allArgs, task_name=taskName, task_output_file=taskOutputLogPath) taskInfos[roi] = taskInfo return taskInfos
def _setInSlotInput(self, slot, subindex, roi, new_pixels): """ Since this is a label array, inserting pixels has a special meaning: We only overwrite the new non-zero pixels. In the new data, zeros mean "don't change". So, here's what each pixel we're adding means: 0: don't change 1: change to 1 2: change to 2 ... N: change to N magic_eraser_value: change to 0 """ if isinstance(new_pixels, vigra.VigraArray): new_pixels = new_pixels.view(numpy.ndarray) # Extract the data to modify original_data = self.Output.stype.allocateDestination( SubRegion(self.Output, *roiFromShape(new_pixels.shape))) self.execute(self.Output, (), roi, original_data) # Reset the pixels we need to change (so we can use |= below) original_data[new_pixels.nonzero()] = 0 # Update original_data |= new_pixels # Replace 'eraser' values with zeros. cleaned_data = original_data.copy() cleaned_data[original_data == self._eraser_magic_value] = 0 # Set in the cache (our superclass). super(OpCompressedUserLabelArray, self)._setInSlotInput(slot, subindex, roi, cleaned_data, store_zero_blocks=False) # FIXME: Shouldn't this notification be triggered from within OpUnmanagedCompressedCache? self.Output.setDirty(roi.start, roi.stop) return cleaned_data # Internal use: Return the cleaned_data
def propagateDirty(self, slot, subindex, roi): # the roi here is a list of time steps if slot is self.RegionCenters: roi = list(roi) t = roi[0] T = t + 1 a = t for b in roi[1:]: assert b - 1 == a, "List roi must be contiguous" a = b T += 1 time_index = self.BinaryImage.meta.axistags.index('t') stop = numpy.asarray(self.BinaryImage.meta.shape, dtype=numpy.int) start = numpy.zeros_like(stop) stop[time_index] = T start[time_index] = t roi = SubRegion(self.Output, tuple(start), tuple(stop)) self.Output.setDirty(roi)
def _export(): self.applet.busy = True self.applet.appletStateUpdateRequested.emit() if hasattr(self.mainOperator, "RelabeledImage"): labelImageSlot = self.mainOperator.RelabeledImage else: labelImageSlot = self.mainOperator.LabelImage logger.info("Saving first label image...") key = [] for idx, flag in enumerate( axisTagsToString(labelImageSlot.meta.axistags)): if flag is 't': key.append(slice(0, labelImageSlot.meta.shape[idx]) ) #slice(t_from,t_from+1)) elif flag is 'c': key.append(slice(0, 1)) else: key.append(slice(0, labelImageSlot.meta.shape[idx])) try: events = self.mainOperator.EventsVector.value logger.info("Saving events...") logger.info("Length of events " + str(len(events))) num_files = float(len(events)) for i in sorted(events.keys()): events_at = events[i] i = int(i) t = i key[0] = slice(t, t + 1) roi = SubRegion(labelImageSlot, key) labelImage = labelImageSlot.get(roi).wait() labelImage = labelImage[0, ..., 0] write_events(events_at, str(directory), t, labelImage) _handle_progress(i / num_files * 100) except IOError as e: self._criticalMessage("Cannot export the tracking results. Maybe these files already exist. "\ "Please delete them or choose a different directory.") return
def propagateDirty(self, slot, subindex, roi): # all input slots affect the (global) graph cut computation if slot == self.Beta: # beta value affects the whole volume self.Output.setDirty(slice(None)) elif slot == self.Prediction: # time-channel slices are pairwise independent # determine t, c from input volume t_ind = 0 c_ind = 4 t = (roi.start[t_ind], roi.stop[t_ind]) c = (roi.start[c_ind], roi.stop[c_ind]) # set output dirty start = t[0:1] + (0,)*3 + c[0:1] stop = t[1:2] + self.Output.meta.shape[1:4] + c[1:2] roi = SubRegion(self.Output, start=start, stop=stop) self.Output.setDirty(roi)
def propagateDirty(self, slot, subindex, roi): super(OpObjectsSegment, self).propagateDirty(slot, subindex, roi) if slot == self.LabelImage: # time-channel slices are pairwise independent # determine t, c from input volume t_ind = 0 c_ind = 4 t = (roi.start[t_ind], roi.stop[t_ind]) c = (roi.start[c_ind], roi.stop[c_ind]) # set output dirty start = t[0:1] + (0, ) * 3 + c[0:1] stop = t[1:2] + self.Output.meta.shape[1:4] + c[1:2] roi = SubRegion(self.Output, start=start, stop=stop) self.Output.setDirty(roi) elif slot == self.Margin: # margin affects the whole volume self.Output.setDirty(slice(None))
def testSetDirty(self): g = Graph() vol = np.zeros((200, 100, 10)) vol = vol.astype(np.uint8) vol = vigra.taggedView(vol, axistags="zyx") vol[:200, ...] = 1 vol[800:, ...] = 1 op = OpLazyCC(graph=g) op.ChunkShape.setValue((100, 20, 5)) op.Input.setValue(vol) opCheck = DirtyAssert(graph=g) opCheck.Input.connect(op.Output) out = op.Output[:100, :20, :5].wait() roi = SubRegion(op.Input, start=(0, 0, 0), stop=(200, 100, 10)) with self.assertRaises(PropagateDirtyCalled): op.Input.setDirty(roi)
def _calculateFeatures(self, interactive=True): mainOperator = self.topLevelOperatorView mainOperator.ObjectCenterImage.setDirty(SubRegion(mainOperator.ObjectCenterImage)) maxt = mainOperator.LabelImage.meta.shape[0] def _handle_one_finished(*args): self.already_done = self.already_done+1 self.applet.progressSignal.emit(int(self.already_done*100./maxt)) if self.already_done == maxt: _handle_all_finished(*args) def _handle_all_finished(*args): self.applet.progressSignal.emit(100) self.topLevelOperatorView._opRegFeats.fixed = True feats = self.topLevelOperatorView.RegionFeatures[0].wait() nfeatures = 0 nchannels = 0 for pname, pfeats in feats[0].iteritems(): if pname!='Default features': for featname, feat in pfeats.iteritems(): nchannels += feat.shape[1] nfeatures += 1 if interactive: self._drawer.featuresSelected.setText("{} features computed, {} channels in total".format(nfeatures, nchannels)) logger.info('Object Extraction: done.') self.applet.progressSignal.emit(0) self.applet.progressSignal.emit(-1) reqs = [] self.already_done = 0 for t in range(maxt): req = mainOperator.RegionFeatures([t]) req.submit() reqs.append(req) for req in reqs: req.notify_failed( self.handleFeatureComputationFailure ) req.notify_finished( _handle_one_finished )
def execute(self, slot, subindex, roi, result): if slot == self.Dirty: assert False # Shouldn't get to this line because the dirty output is given a value directly if slot == self.OutputDataPath: assert False # This slot is already set via setupOutputs if slot == self.ExportResult: # We can stop now if the output isn't dirty if not self.Dirty.value: result[0] = True return exportFormat = self.Format.value # Export H5 if exportFormat == ExportFormat.H5: pathComp = PathComponents(self.OutputDataPath.value) # Ensure the directory exists if not os.path.exists(pathComp.externalDirectory): with self._createDirLock: # Check again now that we have the lock. if not os.path.exists(pathComp.externalDirectory): os.makedirs(pathComp.externalDirectory) # Open the file try: hdf5File = h5py.File(pathComp.externalPath) except: logger.error("Unable to open hdf5File: " + pathComp.externalPath) logger.error(traceback.format_exc()) result[0] = False return # Set up the write operator opH5Writer = OpH5WriterBigDataset(parent=self) opH5Writer.hdf5File.setValue(hdf5File) opH5Writer.hdf5Path.setValue(pathComp.internalPath) #opH5Writer.Image.connect( self.ImageToExport ) opH5Writer.Image.connect(self.ImageCache.Output) print "computing predictions for the selected slices:" self.ImageCache.fixAtCurrent.setValue(False) #check readiness for inp in self.ImageCache.inputs: print inp, self.ImageCache.inputs[inp].ready() print "input shape:", self.ImageCache.Input.meta.shape print "output shape:", self.ImageCache.Output.meta.shape selectedSlices = self.SelectedSlices.value zaxis = self.ImageToExport.meta.axistags.index('z') for isl, sl in enumerate(selectedSlices): print "computing for slice ...", isl start = [0] * len(self.ImageToExport.meta.shape) start[zaxis] = sl stop = list(self.ImageToExport.meta.shape) stop[zaxis] = sl + 1 roi = SubRegion(self.ImageCache, start=start, stop=stop) print roi temp = self.ImageCache.Output[roi.toSlice()].wait() #print temp self.ImageCache.fixAtCurrent.setValue(True) #tstart = [0]*len(self.ImageToExport.meta.shape) #tstop = list(self.ImageToExport.meta.shape) #troi = SubRegion(self.ImageCache, start=tstart, stop=tstop) #tttemp = self.ImageCache.Output[troi.toSlice()].wait() #print tttemp # The H5 Writer provides it's own progress signal, so just connect ours to it. opH5Writer.progressSignal.subscribe(self.progressSignal) # Trigger the write self.Dirty.setValue(not opH5Writer.WriteImage.value) hdf5File.close() opH5Writer.cleanUp() # elif exportFormat == ExportFormat.Npy: # assert False # TODO # elif exportFormat == ExportFormat.Npy: # assert False # TODO else: assert False, "Unknown export format" result[0] = not self.Dirty.value
def execute(self, slot, subindex, slot_roi, target): assert slot == self.Features or slot == self.Output if slot == self.Features: feature_slice = roiToSlice(slot_roi.start, slot_roi.stop) index = subindex[0] feature_slice = list(feature_slice) # Translate channel slice of this feature to the channel slice of the output slot. output_channel_offset = self.featureOutputChannels[index][0] feature_slice[1] = slice( output_channel_offset + feature_slice[1].start, output_channel_offset + feature_slice[1].stop) slot_roi = SubRegion(self.Output, pslice=feature_slice) # Get output slot region for this channel return self.execute(self.Output, (), slot_roi, target) elif slot == self.Output: # Correlation of variable 'families' representing reference frames: # ______________________________ # | input/output frame | input/output shape given by slots # | _________________________ | # | | smooth frame | | pre-smoothing op needs halo around filter roi # | | ____________________ | | # | | |filter frame | | | filter needs halo around target roi # | | | _______________ | | | # | | | | target frame | | | | target is given by output_roi # note: The 'full_' variable prefix refers to the full 5D shape (tczyx), without 'full_' variables mostly # refer to the 3D space subregion (zyx) full_output_slice = slot_roi.toSlice() logger.debug( f"OpPixelFeaturesPresmoothed: request {slot_roi.pprint()}") assert (slot_roi.stop <= self.Output.meta.shape).all() full_output_shape = self.Output.meta.shape full_output_start, full_output_stop = sliceToRoi( full_output_slice, full_output_shape) assert len(full_output_shape) == 5 if all(self.ComputeIn2d.value ): # todo: check for this particular slice axes2enlarge = (0, 1, 1) else: axes2enlarge = (1, 1, 1) output_shape = full_output_shape[2:] output_start = full_output_start[2:] output_stop = full_output_stop[2:] axistags = self.Output.meta.axistags target = target.view(vigra.VigraArray) target.axistags = copy.copy(axistags) # filter roi in input frame # sigma = 0.7, because the features receive a pre-smoothed array and don't need much of a neighborhood input_filter_start, input_filter_stop = roi.enlargeRoiForHalo( output_start, output_stop, output_shape, 0.7, self.WINDOW_SIZE, enlarge_axes=axes2enlarge) # smooth roi in input frame input_smooth_start, input_smooth_stop = roi.enlargeRoiForHalo( input_filter_start, input_filter_stop, output_shape, self.max_sigma, self.WINDOW_SIZE, enlarge_axes=axes2enlarge, ) # target roi in filter frame filter_target_start = roi.TinyVector(output_start - input_filter_start) filter_target_stop = roi.TinyVector(output_stop - input_filter_start) # filter roi in smooth frame smooth_filter_start = roi.TinyVector(input_filter_start - input_smooth_start) smooth_filter_stop = roi.TinyVector(input_filter_stop - input_smooth_start) filter_target_slice = roi.roiToSlice(filter_target_start, filter_target_stop) input_smooth_slice = roi.roiToSlice(input_smooth_start, input_smooth_stop) # pre-smooth for all requested time slices and all channels full_input_smooth_slice = (full_output_slice[0], slice(None), *input_smooth_slice) req = self.Input[full_input_smooth_slice] source = req.wait() req.clean() req.destination = None if source.dtype != numpy.float32: sourceF = source.astype(numpy.float32) try: source.resize((1, ), refcheck=False) except Exception: pass del source source = sourceF sourceV = source.view(vigra.VigraArray) sourceV.axistags = copy.copy(self.Input.meta.axistags) dimCol = len(self.scales) dimRow = self.matrix.shape[0] presmoothed_source = [None] * dimCol source_smooth_shape = tuple(smooth_filter_stop - smooth_filter_start) full_source_smooth_shape = ( full_output_stop[0] - full_output_start[0], self.Input.meta.shape[1], ) + source_smooth_shape try: for j in range(dimCol): for i in range(dimRow): if self.matrix[i, j]: # There is at least one filter op with this scale break else: # There is no filter op at this scale continue if self.scales[j] > 1.0: tempSigma = math.sqrt(self.scales[j]**2 - 1.0) else: tempSigma = self.scales[j] presmoothed_source[j] = numpy.ndarray( full_source_smooth_shape, numpy.float32) droi = ( (0, *tuple(smooth_filter_start._asint())), (sourceV.shape[1], *tuple(smooth_filter_stop._asint())), ) for i, vsa in enumerate(sourceV.timeIter()): presmoothed_source[j][ i, ...] = self._computeGaussianSmoothing( vsa, tempSigma, droi, in2d=self.ComputeIn2d.value[j]) except RuntimeError as e: if "kernel longer than line" in str(e): raise RuntimeError( "Feature computation error:\nYour image is too small to apply a filter with " f"sigma={self.scales[j]:.1f}. Please select features with smaller sigmas." ) else: raise e del sourceV try: source.resize((1, ), refcheck=False) except ValueError: # Sometimes this fails, but that's okay. logger.debug("Failed to free array memory.") del source cnt = 0 written = 0 closures = [] # connect individual operators for i in range(dimRow): for j in range(dimCol): if self.matrix[i, j]: oslot = self.featureOps[i][j].Output req = None slices = oslot.meta.shape[1] if (cnt + slices >= slot_roi.start[1] and slot_roi.start[1] - cnt < slices and slot_roi.start[1] + written < slot_roi.stop[1]): begin = 0 if cnt < slot_roi.start[1]: begin = slot_roi.start[1] - cnt end = slices if cnt + end > slot_roi.stop[1]: end = slot_roi.stop[1] - cnt # feature slice in output frame feature_slice = (slice(None), slice( written, written + end - begin)) + (slice(None), ) * 3 subtarget = target[feature_slice] # readjust the roi for the new source array full_filter_target_slice = [ full_output_slice[0], slice(begin, end), *filter_target_slice ] filter_target_roi = SubRegion( oslot, pslice=full_filter_target_slice) closure = partial( oslot.operator.execute, oslot, (), filter_target_roi, subtarget, sourceArray=presmoothed_source[j], ) closures.append(closure) written += end - begin cnt += slices pool = RequestPool() for c in closures: pool.request(c) pool.wait() pool.clean() for i in range(len(presmoothed_source)): if presmoothed_source[i] is not None: try: presmoothed_source[i].resize((1, )) except Exception: presmoothed_source[i] = None
def execute(self, slot, subindex, roi, result): if slot == self.Dirty: assert False # Shouldn't get to this line because the dirty output is given a value directly if slot == self.OutputDataPath: assert False # This slot is already set via setupOutputs if slot == self.ExportResult: # We can stop now if the output isn't dirty if not self.Dirty.value: result[0] = True return exportFormat = self.Format.value # Export H5 if exportFormat == ExportFormat.H5: pathComp = PathComponents(self.OutputDataPath.value) # Ensure the directory exists if not os.path.exists(pathComp.externalDirectory): with self._createDirLock: # Check again now that we have the lock. if not os.path.exists(pathComp.externalDirectory): os.makedirs(pathComp.externalDirectory) # Open the file try: hdf5File = h5py.File(pathComp.externalPath) except: logger.error("Unable to open hdf5File: " + pathComp.externalPath) logger.error( traceback.format_exc() ) result[0] = False return # Set up the write operator opH5Writer = OpH5WriterBigDataset(parent=self) opH5Writer.hdf5File.setValue( hdf5File ) opH5Writer.hdf5Path.setValue( pathComp.internalPath ) #opH5Writer.Image.connect( self.ImageToExport ) opH5Writer.Image.connect(self.ImageCache.Output) print "computing predictions for the selected slices:" self.ImageCache.fixAtCurrent.setValue(False) #check readiness for inp in self.ImageCache.inputs: print inp, self.ImageCache.inputs[inp].ready() print "input shape:", self.ImageCache.Input.meta.shape print "output shape:", self.ImageCache.Output.meta.shape selectedSlices = self.SelectedSlices.value zaxis = self.ImageToExport.meta.axistags.index('z') for isl, sl in enumerate(selectedSlices): print "computing for slice ...", isl start = [0]*len(self.ImageToExport.meta.shape) start[zaxis]=sl stop = list(self.ImageToExport.meta.shape) stop[zaxis]=sl+1 roi = SubRegion(self.ImageCache, start=start, stop=stop) print roi temp = self.ImageCache.Output[roi.toSlice()].wait() #print temp self.ImageCache.fixAtCurrent.setValue(True) #tstart = [0]*len(self.ImageToExport.meta.shape) #tstop = list(self.ImageToExport.meta.shape) #troi = SubRegion(self.ImageCache, start=tstart, stop=tstop) #tttemp = self.ImageCache.Output[troi.toSlice()].wait() #print tttemp # The H5 Writer provides it's own progress signal, so just connect ours to it. opH5Writer.progressSignal.subscribe( self.progressSignal ) # Trigger the write self.Dirty.setValue( not opH5Writer.WriteImage.value ) hdf5File.close() opH5Writer.cleanUp() # elif exportFormat == ExportFormat.Npy: # assert False # TODO # elif exportFormat == ExportFormat.Npy: # assert False # TODO else: assert False, "Unknown export format" result[0] = not self.Dirty.value
def execute(self, slot, subindex, slot_roi, target): assert slot == self.Features or slot == self.Output if slot == self.Features: feature_slice = roiToSlice(slot_roi.start, slot_roi.stop) index = subindex[0] feature_slice = list(feature_slice) # Translate channel slice of this feature to the channel slice of the output slot. output_channel_offset = self.featureOutputChannels[index][0] feature_slice[1] = slice( output_channel_offset + feature_slice[1].start, output_channel_offset + feature_slice[1].stop ) slot_roi = SubRegion(self.Output, pslice=feature_slice) # Get output slot region for this channel return self.execute(self.Output, (), slot_roi, target) elif slot == self.Output: # Correlation of variable 'families' representing reference frames: # ______________________________ # | input/output frame | input/output shape given by slots # | _________________________ | # | | smooth frame | | pre-smoothing op needs halo around filter roi # | | ____________________ | | # | | |filter frame | | | filter needs halo around target roi # | | | _______________ | | | # | | | | target frame | | | | target is given by output_roi # note: The 'full_' variable prefix refers to the full 5D shape (tczyx), without 'full_' variables mostly # refer to the 3D space subregion (zyx) full_output_slice = slot_roi.toSlice() logger.debug(f"OpPixelFeaturesPresmoothed: request {slot_roi.pprint()}") assert (slot_roi.stop <= self.Output.meta.shape).all() full_output_shape = self.Output.meta.shape full_output_start, full_output_stop = sliceToRoi(full_output_slice, full_output_shape) assert len(full_output_shape) == 5 if all(self.ComputeIn2d.value): # todo: check for this particular slice axes2enlarge = (0, 1, 1) else: axes2enlarge = (1, 1, 1) output_shape = full_output_shape[2:] output_start = full_output_start[2:] output_stop = full_output_stop[2:] axistags = self.Output.meta.axistags target = target.view(vigra.VigraArray) target.axistags = copy.copy(axistags) # filter roi in input frame # sigma = 0.7, because the features receive a pre-smoothed array and don't need much of a neighborhood input_filter_start, input_filter_stop = roi.enlargeRoiForHalo( output_start, output_stop, output_shape, 0.7, self.WINDOW_SIZE, enlarge_axes=axes2enlarge ) # smooth roi in input frame input_smooth_start, input_smooth_stop = roi.enlargeRoiForHalo( input_filter_start, input_filter_stop, output_shape, self.max_sigma, self.WINDOW_SIZE, enlarge_axes=axes2enlarge, ) # target roi in filter frame filter_target_start = roi.TinyVector(output_start - input_filter_start) filter_target_stop = roi.TinyVector(output_stop - input_filter_start) # filter roi in smooth frame smooth_filter_start = roi.TinyVector(input_filter_start - input_smooth_start) smooth_filter_stop = roi.TinyVector(input_filter_stop - input_smooth_start) filter_target_slice = roi.roiToSlice(filter_target_start, filter_target_stop) input_smooth_slice = roi.roiToSlice(input_smooth_start, input_smooth_stop) # pre-smooth for all requested time slices and all channels full_input_smooth_slice = (full_output_slice[0], slice(None), *input_smooth_slice) req = self.Input[full_input_smooth_slice] source = req.wait() req.clean() req.destination = None if source.dtype != numpy.float32: sourceF = source.astype(numpy.float32) try: source.resize((1,), refcheck=False) except Exception: pass del source source = sourceF sourceV = source.view(vigra.VigraArray) sourceV.axistags = copy.copy(self.Input.meta.axistags) dimCol = len(self.scales) dimRow = self.matrix.shape[0] presmoothed_source = [None] * dimCol source_smooth_shape = tuple(smooth_filter_stop - smooth_filter_start) full_source_smooth_shape = ( full_output_stop[0] - full_output_start[0], self.Input.meta.shape[1], ) + source_smooth_shape try: for j in range(dimCol): for i in range(dimRow): if self.matrix[i, j]: # There is at least one filter op with this scale break else: # There is no filter op at this scale continue if self.scales[j] > 1.0: tempSigma = math.sqrt(self.scales[j] ** 2 - 1.0) else: tempSigma = self.scales[j] presmoothed_source[j] = numpy.ndarray(full_source_smooth_shape, numpy.float32) droi = ( (0, *tuple(smooth_filter_start._asint())), (sourceV.shape[1], *tuple(smooth_filter_stop._asint())), ) for i, vsa in enumerate(sourceV.timeIter()): presmoothed_source[j][i, ...] = self._computeGaussianSmoothing( vsa, tempSigma, droi, in2d=self.ComputeIn2d.value[j] ) except RuntimeError as e: if "kernel longer than line" in str(e): raise RuntimeError( "Feature computation error:\nYour image is too small to apply a filter with " f"sigma={self.scales[j]:.1f}. Please select features with smaller sigmas." ) else: raise e del sourceV try: source.resize((1,), refcheck=False) except ValueError: # Sometimes this fails, but that's okay. logger.debug("Failed to free array memory.") del source cnt = 0 written = 0 closures = [] # connect individual operators for i in range(dimRow): for j in range(dimCol): if self.matrix[i, j]: oslot = self.featureOps[i][j].Output req = None slices = oslot.meta.shape[1] if ( cnt + slices >= slot_roi.start[1] and slot_roi.start[1] - cnt < slices and slot_roi.start[1] + written < slot_roi.stop[1] ): begin = 0 if cnt < slot_roi.start[1]: begin = slot_roi.start[1] - cnt end = slices if cnt + end > slot_roi.stop[1]: end = slot_roi.stop[1] - cnt # feature slice in output frame feature_slice = (slice(None), slice(written, written + end - begin)) + (slice(None),) * 3 subtarget = target[feature_slice] # readjust the roi for the new source array full_filter_target_slice = [full_output_slice[0], slice(begin, end), *filter_target_slice] filter_target_roi = SubRegion(oslot, pslice=full_filter_target_slice) closure = partial( oslot.operator.execute, oslot, (), filter_target_roi, subtarget, sourceArray=presmoothed_source[j], ) closures.append(closure) written += end - begin cnt += slices pool = RequestPool() for c in closures: pool.request(c) pool.wait() pool.clean() for i in range(len(presmoothed_source)): if presmoothed_source[i] is not None: try: presmoothed_source[i].resize((1,)) except Exception: presmoothed_source[i] = None
def _setInSlotInput(self, slot, subindex, roi, new_pixels): """ Since this is a label array, inserting pixels has a special meaning: We only overwrite the new non-zero pixels. In the new data, zeros mean "don't change". So, here's what each pixel we're adding means: 0: don't change 1: change to 1 2: change to 2 ... N: change to N eraser_magic_value: change to 0 """ if isinstance(new_pixels, vigra.VigraArray): new_pixels = new_pixels.view(numpy.ndarray) # Get logical blocking. block_rois = getIntersectingRois(self.Output.meta.shape, self._blockshape, (roi.start, roi.stop)) # Convert to tuples block_rois = [(tuple(start), tuple(stop)) for start, stop in block_rois] max_label = 0 for block_roi in block_rois: roi_within_data = numpy.array(block_roi) - roi.start new_block_pixels = new_pixels[roiToSlice(*roi_within_data)] # Shortcut: Nothing to change if this block is all zeros. if not new_block_pixels.any(): continue block_slot_roi = SubRegion(self.Output, *block_roi) # Extract the data to modify original_block_data = self.Output.stype.allocateDestination(block_slot_roi) self.execute(self.Output, (), block_slot_roi, original_block_data) # Reset the pixels we need to change (so we can use |= below) original_block_data[new_block_pixels.nonzero()] = 0 # Update original_block_data |= new_block_pixels # Replace 'eraser' values with zeros. cleaned_block_data = original_block_data.copy() cleaned_block_data[original_block_data == self._eraser_magic_value] = 0 # Set in the cache (our superclass). super(OpCompressedUserLabelArray, self)._setInSlotInput( slot, subindex, block_slot_roi, cleaned_block_data, store_zero_blocks=False ) max_label = max(max_label, cleaned_block_data.max()) # We could wait to send out one big dirty notification (instead of one per block), # But that might result in a lot of unecessarily dirty pixels in cases when the # new_pixels were mostly empty (such as when importing labels from disk). # That's bad for downstream operators like OpFeatureMatrixCache # So instead, we only send notifications for the blocks that were touched. # During labeling, it makes no difference. # During project import, this is slightly worse. # But during label import from disk, this is very important.a # FIXME: Shouldn't this notification be triggered from within OpUnmanagedCompressedCache? self.Output.setDirty(*block_roi) return max_label # Internal use: Return max label
def _export(): self.applet.busy = True self.applet.appletStateUpdateRequested.emit() t_from = None # determine from_time (it could has been changed in the GUI meanwhile) for t_from, label2color_at in enumerate( self.mainOperator.label2color): if len(label2color_at) == 0: continue else: break if t_from is None: self._criticalMessage("There is nothing to export.") return t_from = int(t_from) logger.info("Saving first label image...") key = [] for idx, flag in enumerate( axisTagsToString( self.mainOperator.LabelImage.meta.axistags)): if flag is 't': key.append(slice(t_from, t_from + 1)) elif flag is 'c': key.append(slice(0, 1)) else: key.append( slice(0, self.mainOperator.LabelImage.meta.shape[idx])) roi = SubRegion(self.mainOperator.LabelImage, key) labelImage = self.mainOperator.LabelImage.get(roi).wait() labelImage = labelImage[0, ..., 0] try: # write_events([], str(directory), t_from, labelImage) events = self.mainOperator.EventsVector.value logger.info("Saving events...") logger.info("Length of events " + str(len(events))) num_files = float(len(events)) for i in sorted(events.keys()): events_at = events[i] i = int(i) t = t_from + i key[0] = slice(t, t + 1) roi = SubRegion(self.mainOperator.LabelImage, key) labelImage = self.mainOperator.LabelImage.get(roi).wait() labelImage = labelImage[0, ..., 0] if self.withMergers: write_events(events_at, str(directory), t, labelImage, self.mainOperator.mergers) else: write_events(events_at, str(directory), t, labelImage) _handle_progress(i / num_files * 100) except IOError as e: self._criticalMessage("Cannot export the tracking results. Maybe these files already exist. "\ "Please delete them or choose a different directory.") return
def execute(self, slot, subindex, rroi, result): assert slot == self.Features or slot == self.Output if slot == self.Features: key = roiToSlice(rroi.start, rroi.stop) index = subindex[0] subslot = self.Features[index] key = list(key) channelIndex = self.Input.meta.axistags.index('c') # Translate channel slice to the correct location for the output slot. key[channelIndex] = slice(self.featureOutputChannels[index][0] + key[channelIndex].start, self.featureOutputChannels[index][0] + key[channelIndex].stop) rroi = SubRegion(subslot, pslice=key) # Get output slot region for this channel return self.execute(self.Output, (), rroi, result) elif slot == self.outputs["Output"]: key = rroi.toSlice() cnt = 0 written = 0 assert (rroi.stop<=self.outputs["Output"].meta.shape).all() flag = 'c' channelAxis=self.inputs["Input"].meta.axistags.index('c') axisindex = channelAxis oldkey = list(key) oldkey.pop(axisindex) inShape = self.inputs["Input"].meta.shape shape = self.outputs["Output"].meta.shape axistags = self.inputs["Input"].meta.axistags result = result.view(vigra.VigraArray) result.axistags = copy.copy(axistags) hasTimeAxis = self.inputs["Input"].meta.axistags.axisTypeCount(vigra.AxisType.Time) timeAxis=self.inputs["Input"].meta.axistags.index('t') subkey = popFlagsFromTheKey(key,axistags,'c') subshape=popFlagsFromTheKey(shape,axistags,'c') at2 = copy.copy(axistags) at2.dropChannelAxis() subshape=popFlagsFromTheKey(subshape,at2,'t') subkey = popFlagsFromTheKey(subkey,at2,'t') oldstart, oldstop = roi.sliceToRoi(key, shape) start, stop = roi.sliceToRoi(subkey,subkey) maxSigma = max(0.7,self.maxSigma) # The region of the smoothed image we need to give to the feature filter (in terms of INPUT coordinates) vigOpSourceStart, vigOpSourceStop = roi.extendSlice(start, stop, subshape, 0.7, window = 2) # The region of the input that we need to give to the smoothing operator (in terms of INPUT coordinates) newStart, newStop = roi.extendSlice(vigOpSourceStart, vigOpSourceStop, subshape, maxSigma, window = 3.5) # Translate coordinates (now in terms of smoothed image coordinates) vigOpSourceStart = roi.TinyVector(vigOpSourceStart - newStart) vigOpSourceStop = roi.TinyVector(vigOpSourceStop - newStart) readKey = roi.roiToSlice(newStart, newStop) writeNewStart = start - newStart writeNewStop = writeNewStart + stop - start treadKey=list(readKey) if hasTimeAxis: if timeAxis < channelAxis: treadKey.insert(timeAxis, key[timeAxis]) else: treadKey.insert(timeAxis-1, key[timeAxis]) if self.inputs["Input"].meta.axistags.axisTypeCount(vigra.AxisType.Channels) == 0: treadKey = popFlagsFromTheKey(treadKey,axistags,'c') else: treadKey.insert(channelAxis, slice(None,None,None)) treadKey=tuple(treadKey) req = self.inputs["Input"][treadKey].allocate() sourceArray = req.wait() req.result = None req.destination = None if sourceArray.dtype != numpy.float32: sourceArrayF = sourceArray.astype(numpy.float32) sourceArray.resize((1,), refcheck = False) del sourceArray sourceArray = sourceArrayF sourceArrayV = sourceArray.view(vigra.VigraArray) sourceArrayV.axistags = copy.copy(axistags) dimCol = len(self.scales) dimRow = self.matrix.shape[0] sourceArraysForSigmas = [None]*dimCol #connect individual operators for j in range(dimCol): hasScale = False for i in range(dimRow): if self.matrix[i,j]: hasScale = True if not hasScale: continue destSigma = 1.0 if self.scales[j] > destSigma: tempSigma = math.sqrt(self.scales[j]**2 - destSigma**2) else: destSigma = 0.0 tempSigma = self.scales[j] vigOpSourceShape = list(vigOpSourceStop - vigOpSourceStart) if hasTimeAxis: if timeAxis < channelAxis: vigOpSourceShape.insert(timeAxis, ( oldstop - oldstart)[timeAxis]) else: vigOpSourceShape.insert(timeAxis-1, ( oldstop - oldstart)[timeAxis]) vigOpSourceShape.insert(channelAxis, inShape[channelAxis]) sourceArraysForSigmas[j] = numpy.ndarray(tuple(vigOpSourceShape),numpy.float32) for i,vsa in enumerate(sourceArrayV.timeIter()): droi = (tuple(vigOpSourceStart._asint()), tuple(vigOpSourceStop._asint())) tmp_key = getAllExceptAxis(len(sourceArraysForSigmas[j].shape),timeAxis, i) sourceArraysForSigmas[j][tmp_key] = vigra.filters.gaussianSmoothing(vsa,tempSigma, roi = droi, window_size = 3.5 ) else: droi = (tuple(vigOpSourceStart._asint()), tuple(vigOpSourceStop._asint())) #print droi, sourceArray.shape, tempSigma,self.scales[j] sourceArraysForSigmas[j] = vigra.filters.gaussianSmoothing(sourceArrayV, sigma = tempSigma, roi = droi, window_size = 3.5) #sourceArrayForSigma = sourceArrayForSigma.view(numpy.ndarray) del sourceArrayV try: sourceArray.resize((1,), refcheck = False) except ValueError: # Sometimes this fails, but that's okay. logger.debug("Failed to free array memory.") del sourceArray closures = [] #connect individual operators for i in range(dimRow): for j in range(dimCol): val=self.matrix[i,j] if val: vop= self.featureOps[i][j] oslot = vop.outputs["Output"] req = None inTagKeys = [ax.key for ax in oslot.meta.axistags] if flag in inTagKeys: slices = oslot.meta.shape[axisindex] if cnt + slices >= rroi.start[axisindex] and rroi.start[axisindex]-cnt<slices and rroi.start[axisindex]+written<rroi.stop[axisindex]: begin = 0 if cnt < rroi.start[axisindex]: begin = rroi.start[axisindex] - cnt end = slices if cnt + end > rroi.stop[axisindex]: end -= cnt + end - rroi.stop[axisindex] key_ = copy.copy(oldkey) key_.insert(axisindex, slice(begin, end, None)) reskey = [slice(None, None, None) for x in range(len(result.shape))] reskey[axisindex] = slice(written, written+end-begin, None) destArea = result[tuple(reskey)] roi_ = SubRegion(self.Input, pslice=key_) closure = partial(oslot.operator.execute, oslot, (), roi_, destArea, sourceArray = sourceArraysForSigmas[j]) closures.append(closure) written += end - begin cnt += slices else: if cnt>=rroi.start[axisindex] and rroi.start[axisindex] + written < rroi.stop[axisindex]: reskey = copy.copy(oldkey) reskey.insert(axisindex, written) #print "key: ", key, "reskey: ", reskey, "oldkey: ", oldkey #print "result: ", result.shape, "inslot:", inSlot.shape destArea = result[tuple(reskey)] logger.debug(oldkey, destArea.shape, sourceArraysForSigmas[j].shape) oldroi = SubRegion(self.Input, pslice=oldkey) closure = partial(oslot.operator.execute, oslot, (), oldroi, destArea, sourceArray = sourceArraysForSigmas[j]) closures.append(closure) written += 1 cnt += 1 pool = Pool() for c in closures: r = pool.request(c) pool.wait() pool.clean() for i in range(len(sourceArraysForSigmas)): if sourceArraysForSigmas[i] is not None: try: sourceArraysForSigmas[i].resize((1,)) except: sourceArraysForSigmas[i] = None
def execute(self, slot, subindex, roi, result): assert (roi.start >= 0).all(), \ "Requested roi is out-of-bounds: [{}, {}]".format( roi.start, roi.stop ) assert (roi.stop <= self.Input.meta.shape).all(), \ "Requested roi is out-of-bounds: [{}, {}] (exceeds shape {})"\ .format( roi.start, roi.stop, self.Input.meta.shape ) if not self._configured: # this happens when the operator is not yet fully configured due to fixAtCurrent == True result[:] = 0 return t = time.time() start, stop = roi.start, roi.stop blockStart = (start / self._blockShape) blockStop = (stop * 1.0 / self._blockShape).ceil() innerBlocks = self._get_block_numbers(blockStart, blockStop) pool = RequestPool() for block_index in innerBlocks.flat: #which part of the original key does this block fill? block_multi_index = self._get_block_multi_index(block_index) offset = self._blockShape * block_multi_index bigstart = numpy.maximum(offset, start) bigstop = numpy.minimum(offset + self._blockShape, stop) smallstart = bigstart - offset smallstop = bigstop - offset bigkey = roiToSlice(bigstart - start, bigstop - start) with self._lock: if not self._fixed: if not self._cache_list.has_key(block_index): self._opSub_list[block_index] = generic.OpSubRegion( parent=self) self._opSub_list[block_index].inputs["Input"].connect( self.inputs["Input"]) tstart = self._blockShape * block_multi_index tstop = numpy.minimum((block_multi_index + numpy.ones( block_multi_index.shape, numpy.uint8)) * self._blockShape, self.shape) self._opSub_list[block_index].Roi.setValue( (tuple(tstart), tuple(tstop))) self._cache_list[block_index] = OpArrayCache( parent=self) self._cache_list[block_index].inputs["Input"].connect( self._opSub_list[block_index].outputs["Output"]) self._cache_list[block_index].inputs[ "fixAtCurrent"].connect(self.fixAtCurrent) self._cache_list[block_index].inputs[ "blockShape"].setValue( self.inputs["innerBlockShape"].value) # we dont register a callback for dirtyness, since we already forward the signal # Forward value changed notifications to our own output. self._cache_list[ block_index].Output.notifyValueChanged( self.Output._sig_value_changed) if self._cache_list.has_key(block_index): op = self._cache_list[block_index] #req = self._cache_list[block_index].outputs["Output"][smallkey].writeInto(result[bigkey]) smallroi = SubRegion(op.outputs["Output"], start=smallstart, stop=smallstop) req = op.Output(smallroi.start, smallroi.stop) req.writeInto(result[bigkey]) pool.add(req) #op.execute(op.outputs["Output"], (), smallroi, result[bigkey]) ####op.getOutSlot(op.outputs["Output"],smallkey,result[bigkey]) #requests.append(req) else: #When this block has never been in the cache and the current #value is fixed (fixAtCurrent=True), return 0 values #This prevents random noise appearing in such cases. result[bigkey] = 0 with self._lock: # Since a downstream operator has expressed an interest in this block, # mark it to be signaled as dirty when we become unfixed. # Otherwise, downstream operators won't know when there's valid data in this block. self._fixed_dirty_blocks.add(block_index) pool.wait() self.logger.debug("read %r took %f msec." % (roi.pprint(), 1000.0 * (time.time() - t)))