def testSetDirty(self):
g = Graph()
vol = np.zeros((5, 2, 200, 100, 10))
vol = vol.astype(np.uint8)
vol = vigra.taggedView(vol, axistags="tcxyz")
vol[:200, ...] = 1
vol[800:, ...] = 1
op = OpLabelVolume(graph=g)
op.Method.setValue(self.method)
op.Input.setValue(vol)
opCheck = DirtyAssert(graph=g)
opCheck.Input.connect(op.Output)
opCheck.willBeDirty(1, 1)
roi = SubRegion(op.Input,
start=(1, 1, 0, 0, 0),
stop=(2, 2, 200, 100, 10))
with pytest.raises(PropagateDirtyCalled):
op.Input.setDirty(roi)
opCheck.Input.disconnect()
opCheck.Input.connect(op.CachedOutput)
opCheck.willBeDirty(1, 1)
out = op.Output[...].wait()
roi = SubRegion(op.Input,
start=(1, 1, 0, 0, 0),
stop=(2, 2, 200, 100, 10))
with pytest.raises(PropagateDirtyCalled):
op.Input.setDirty(roi)
def _purge_label(self, label_to_purge):
"""
Scan through all labeled pixels.
(1) Clear all pixels of the given value (set to 0)
(2) Decrement all labels above that value so the set of stored labels is consecutive
"""
changed_block_rois = []
#stored_block_rois = self.CleanBlocks.value
stored_block_roi_destination = [None]
self.execute(self.CleanBlocks, (), SubRegion( self.Output, (0,),(1,) ), stored_block_roi_destination)
stored_block_rois = stored_block_roi_destination[0]
for block_roi in stored_block_rois:
# Get data
block_shape = numpy.subtract( block_roi[1], block_roi[0] )
block = numpy.ndarray( shape=block_shape, dtype=self.Output.meta.dtype )
self.execute(self.Output, (), SubRegion( self.Output, *block_roi ), block)
# Locate pixels to change
matching_label_coords = numpy.nonzero( block == label_to_purge )
coords_to_decrement = block > label_to_purge
# Change the data
block[matching_label_coords] = 0
block = numpy.where( coords_to_decrement, block-1, block )
# Update cache with the new data (only if something really changed)
if len(matching_label_coords[0]) > 0 or len(coords_to_decrement[0]) > 0:
super( OpCompressedUserLabelArray, self )._setInSlotInput( self.Input, (), SubRegion( self.Output, *block_roi ), block, store_zero_blocks=False )
changed_block_rois.append( block_roi )
for block_roi in changed_block_rois:
# FIXME: Shouldn't this dirty notification be handled in OpCompressedCache?
self.Output.setDirty( *block_roi )
def _export():
t_from = None
# determine from_time (it could has been changed in the GUI meanwhile)
for t_from, label2color_at in enumerate(
self.mainOperator.label2color):
if len(label2color_at) == 0:
continue
else:
break
if t_from == None:
return
print "Saving first label image..."
key = []
for idx, flag in enumerate(
axisTagsToString(
self.mainOperator.LabelImage.meta.axistags)):
if flag is 't':
key.append(slice(t_from, t_from + 1))
elif flag is 'c':
key.append(slice(0, 1))
else:
key.append(
slice(0, self.mainOperator.LabelImage.meta.shape[idx]))
roi = SubRegion(self.mainOperator.LabelImage, key)
labelImage = self.mainOperator.LabelImage.get(roi).wait()
labelImage = labelImage[0, ..., 0]
try:
write_events([], str(directory), t_from, labelImage)
events = self.mainOperator.events
print "Saving events..."
print "Length of events " + str(len(events))
num_files = float(len(events))
for i, events_at in enumerate(events):
t = t_from + i
key[0] = slice(t + 1, t + 2)
roi = SubRegion(self.mainOperator.LabelImage, key)
labelImage = self.mainOperator.LabelImage.get(roi).wait()
labelImage = labelImage[0, ..., 0]
if self.withMergers:
write_events(events_at, str(directory), t + 1,
labelImage, self.mainOperator.mergers)
else:
write_events(events_at, str(directory), t + 1,
labelImage)
_handle_progress(i / num_files * 100)
except IOError as e:
self._criticalMessage("Cannot export the tracking results. Maybe these files already exist. "\
"Please delete them or choose a different directory.")
return
def testNoFullBlocks(self):
op = OpSplitRequestsBlockwise(False, graph=self.g)
op.Input.connect(self.piper.Output)
op.BlockShape.setValue((10, 15, 20))
op.Output[0:20, 30:45, 10:30].wait()
slot = self.piper.Output
expected = [SubRegion(slot, (0, 30, 10), (10, 45, 20)),
SubRegion(slot, (0, 30, 20), (10, 45, 30)),
SubRegion(slot, (10, 30, 10), (20, 45, 20)),
SubRegion(slot, (10, 30, 20), (20, 45, 30))]
for req in self.piper.requests:
print(req)
for roi in expected:
filtered = [x for x in self.piper.requests if x == roi]
assert len(filtered) == 1, "missing roi {}".format(roi)
self.piper.requests = []
op.Output[5:14, 32:44, 17:21].wait()
expected = [SubRegion(slot, (5, 32, 17), (10, 44, 20)),
SubRegion(slot, (5, 32, 20), (10, 44, 21)),
SubRegion(slot, (10, 32, 17), (14, 44, 20)),
SubRegion(slot, (10, 32, 20), (14, 44, 21))]
for roi in expected:
filtered = [x for x in self.piper.requests if x == roi]
assert len(filtered) == 1, "missing roi {}".format(roi)
def _purge_label(self,
label_to_purge,
decrement_remaining,
replacement_value=0):
"""
Scan through all labeled pixels.
(1) Reassign all pixels of the given value (set to replacement_value)
(2) If decrement_remaining=True, decrement all labels above that
value so the set of stored labels remains consecutive.
Note that the decrement is performed AFTER replacement.
"""
changed_block_rois = []
# stored_block_rois = self.CleanBlocks.value
stored_block_roi_destination = [None]
self.execute(self.CleanBlocks, (), SubRegion(self.Output, (0, ),
(1, )),
stored_block_roi_destination)
stored_block_rois = stored_block_roi_destination[0]
for block_roi in stored_block_rois:
# Get data
block_shape = numpy.subtract(block_roi[1], block_roi[0])
block = self.Output.stype.allocateDestination(
SubRegion(self.Output, *roiFromShape(block_shape)))
self.execute(self.Output, (), SubRegion(self.Output, *block_roi),
block)
# Locate pixels to change
matching_label_coords = numpy.nonzero(block == label_to_purge)
# Change the data
block[matching_label_coords] = replacement_value
coords_to_decrement = block > label_to_purge
if decrement_remaining:
block[coords_to_decrement] -= numpy.uint8(1)
# Update cache with the new data (only if something really changed)
if len(matching_label_coords[0]) > 0 or (
decrement_remaining and coords_to_decrement.sum() > 0):
super(OpCompressedUserLabelArray,
self)._setInSlotInput(self.Input, (),
SubRegion(self.Output, *block_roi),
block,
store_zero_blocks=False)
changed_block_rois.append(block_roi)
for block_roi in changed_block_rois:
# FIXME: Shouldn't this dirty notification be handled in OpUnmanagedCompressedCache?
self.Output.setDirty(*block_roi)
def testDirty(self):
opSlicer = self.opSlicer
dirtyRois = {}
def handleDirty(i, slot, roi):
assert opSlicer.Slices[i] == slot
dirtyRois[i] = roi
for i, slot in enumerate(opSlicer.Slices):
slot.notifyDirty( partial(handleDirty, i) )
dirtyInputRoi = SubRegion(slot=opSlicer.Input, start=[4,3,2,1], stop=[6,5,4,3])
opSlicer.Input.setDirty(dirtyInputRoi)
assert len(dirtyRois) == 2
assert dirtyRois[1].start == [4,3,2,0]
assert dirtyRois[1].stop == [6,5,4,1]
assert dirtyRois[2].start == [4,3,2,0]
assert dirtyRois[2].stop == [6,5,4,1]
# Reset
dirtyRois = {}
opSlicer.AxisFlag.setValue('x')
assert len(dirtyRois) == 3
assert dirtyRois[0].start == [0,0,0,0]
assert dirtyRois[0].stop == [10,10,10,3]
assert dirtyRois[1].start == [0,0,0,0]
assert dirtyRois[1].stop == [10,10,10,3]
assert dirtyRois[2].start == [0,0,0,0]
assert dirtyRois[2].stop == [10,10,10,3]
def _getObjects(self, trange, misdet_idx):
filtered_labels = {}
oid2tids = {}
alltids = set()
for t in range(trange[0], trange[1]):
count = 0
filtered_labels[t] = []
oid2tids[t] = {}
troi = SubRegion(
self.LabelImage,
start=[t] + [0] * len(self.LabelImage.meta.shape[1:]),
stop=[t + 1] + list(self.LabelImage.meta.shape[1:]),
)
max_oid = np.max(self.LabelImage.get(troi).wait())
for idx in range(max_oid + 1):
oid = int(idx) + 1
if t in list(self.labels.keys()) and oid in list(
self.labels[t].keys()):
if misdet_idx not in self.labels[t][oid]:
oid2tids[t][oid] = self.labels[t][oid]
for l in self.labels[t][oid]:
alltids.add(l)
count += 1
logger.info("at timestep {}, {} traxels found".format(t, count))
return oid2tids, alltids
def ingestData(self, slot):
"""
Read the data from the given slot and copy it into this cache.
The rules about special pixel meanings apply here, just like setInSlot
Returns: the max label found in the slot.
"""
assert self._blockshape is not None
assert self.Input.meta.shape == slot.meta.shape
max_label = 0
# Get logical blocking.
block_starts = getIntersectingBlocks( self._blockshape, roiFromShape(self.Input.meta.shape) )
block_starts = map( tuple, block_starts )
# Write each block
for block_start in block_starts:
block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, block_start )
# Request the block data
block_data = slot(*block_roi).wait()
# Write into the array
subregion_roi = SubRegion(self.Input, *block_roi)
cleaned_block_data = self._setInSlotInput( self.Input, (), subregion_roi, block_data )
max_label = max( max_label, cleaned_block_data.max() )
return max_label
def _chunkIndexToRoi(self, index):
shape = self._shape
start = self._chunkShape * np.asarray(index)
stop = self._chunkShape * (np.asarray(index) + 1)
stop = np.where(stop > shape, shape, stop)
roi = SubRegion(self.Input, start=tuple(start), stop=tuple(stop))
return roi
def _export():
num_files = float(len(label2color))
for t, label2color_at in enumerate(label2color):
if len(label2color_at) == 0:
continue
print 'exporting tiffs for t = ' + str(t)
roi = SubRegion(self.mainOperator.LabelImage,
start=[
t,
] + 4 * [
0,
],
stop=[
t + 1,
] + list(lshape[1:]))
labelImage = self.mainOperator.LabelImage.get(roi).wait()
relabeled = relabel(labelImage[0, ..., 0], label2color_at)
for i in range(relabeled.shape[2]):
out_im = relabeled[:, :, i]
out_fn = str(directory) + '/vis_t' + str(t).zfill(
4) + '_z' + str(i).zfill(4) + '.tif'
vigra.impex.writeImage(np.asarray(out_im, dtype=np.uint32),
out_fn)
_handle_progress(t / num_files * 100)
print 'Tiffs exported.'
def copy_block(full_block_roi, clipped_block_roi):
full_block_roi = numpy.asarray(full_block_roi)
clipped_block_roi = numpy.asarray(clipped_block_roi)
output_roi = numpy.asarray(clipped_block_roi) - roi.start
# If data data exists already or we can just fetch it without needing extra scratch space,
# just call the base class
block_roi = self._get_containing_block_roi(clipped_block_roi)
if block_roi is not None or (full_block_roi
== clipped_block_roi).all():
self._execute_Output_impl(clipped_block_roi,
result[roiToSlice(*output_roi)])
elif self.Input.meta.dontcache:
# Data isn't in the cache, but we don't need it in the cache anyway.
self.Input(*clipped_block_roi).writeInto(
result[roiToSlice(*output_roi)]).block()
else:
# Data doesn't exist yet in the cache.
# Request the full block, but then discard the parts we don't need.
# (We use allocateDestination() here to support MaskedArray types.)
# TODO: We should probably just get rid of MaskedArray support altogether...
full_block_data = self.Output.stype.allocateDestination(
SubRegion(self.Output, *full_block_roi))
self._execute_Output_impl(full_block_roi, full_block_data)
roi_within_block = clipped_block_roi - full_block_roi[0]
self.Output.stype.copy_data(
result[roiToSlice(*output_roi)],
full_block_data[roiToSlice(*roi_within_block)])
def _label(self, roi, result):
result = vigra.taggedView(result, axistags=self.Output.meta.axistags)
# get the background values
bg = self.Background[...].wait()
bg = vigra.taggedView(bg, axistags=self.Background.meta.axistags)
bg = bg.withAxes(*"ct")
assert np.all(
self.Background.meta.shape[0] == self.Input.meta.shape[0]
), "Shape of background values incompatible to shape of Input"
assert np.all(
self.Background.meta.shape[4] == self.Input.meta.shape[4]
), "Shape of background values incompatible to shape of Input"
# do labeling in parallel over channels and time slices
pool = RequestPool()
start = np.asarray(roi.start, dtype=np.int)
stop = np.asarray(roi.stop, dtype=np.int)
for ti, t in enumerate(range(roi.start[0], roi.stop[0])):
start[0], stop[0] = t, t + 1
for ci, c in enumerate(range(roi.start[4], roi.stop[4])):
start[4], stop[4] = c, c + 1
newRoi = SubRegion(self.Output, start=tuple(start), stop=tuple(stop))
resView = result[ti, ..., ci].withAxes(*"xyz")
req = Request(partial(self._label3d, newRoi, bg[c, t], resView))
pool.add(req)
logger.debug("{}: Computing connected components for ROI {} ...".format(self.name, roi))
pool.wait()
pool.clean()
logger.debug("{}: Connected components computed.".format(self.name))
def test(self, position5d_start, position5d_stop):
print "test"
roi = SubRegion(self.op.Density, position5d_start,
position5d_stop)
key = roi.toSlice()
key = tuple(k for k in key if k != slice(0,0, None))
newKey = []
for k in key:
if k != slice(0,0,None):
if k.stop < k.start:
k = slice(k.stop, k.start)
newKey.append(k)
newKey = tuple(newKey)
try:
density = numpy.sum(self.op.Density[newKey].wait()) / 255
strdensity = "{0:.2f}".format(density)
self._labelControlUi.CountText.setText(strdensity)
except:
pass
def test2(self, position5d_start, position5d_stop):
print "test2"
roi = SubRegion(self.op.LabelInputs, position5d_start,
position5d_stop)
key = roi.toSlice()
#key = tuple(k for k in key if k != slice(0,0, None))
newKey = []
for k in key:
if k.stop < k.start:
k = slice(k.stop, k.start)
newKey.append(k)
newKey = tuple(newKey)
self.boxes[self.activeBox] = newKey
#self.op.BoxLabelImages[newKey] = self.activeBox + 2
#self.op.BoxLabelImages
labelShape = tuple([position5d_stop[i] + 1 - position5d_start[i] for i in range(5)])
labels = numpy.ones((labelShape), dtype = numpy.uint8) * (self.activeBox + 3)
self.boxlabelsrc.put(newKey, labels)
def execute(self, slot, subindex, roi, result):
sigma = 3.0
roi_with_halo, result_roi = enlargeRoiForHalo(
roi.start, roi.stop, self.Input.meta.shape, sigma, return_result_roi=True
)
start, stop = roi_with_halo
newroi = SubRegion(self.Input, start=start, stop=stop)
data = self.Input.get(newroi).wait()
time.sleep(self.delay)
result[:] = data[roiToSlice(*result_roi)]
def propagateDirty(self, slot, subindex, roi):
key = roi.toSlice()
if slot == self.inputs["Input"] and self._forward_dirty:
if not self._fixed:
self.outputs["Output"].setDirty(key)
elif self._blockShape is not None:
# Find the block key
roi = SubRegion(slot, pslice=key)
start, stop = roi.start, roi.stop
blockStart = (start / self._blockShape)
blockStop = (stop * 1.0 / self._blockShape).ceil()
with self._lock:
# check whether the dirty region encompasses the whole cache
if (blockStart == 0).all() and (blockStop
== self._dirtyShape).all():
self._fixed_all_dirty = True
# shortcut, if everything is dirty already, dont loop over the blocks
if self._fixed_all_dirty is False:
innerBlocks = self._get_block_numbers(
blockStart, blockStop)
for block_index in innerBlocks.flat:
self._fixed_dirty_blocks.add(block_index)
if slot == self.fixAtCurrent:
self._fixed = self.fixAtCurrent.value
if not self._fixed:
# We've become unfixed.
# Take the superset of all the blocks that became dirty in the meantime and notify our output
dirtystart, dirtystop = (None, None)
with self._lock:
if self._fixed_all_dirty is True:
dirtystop = self.Output.meta.shape
dirtystart = [0] * len(self.Output.meta.shape)
elif len(self._fixed_dirty_blocks) > 0:
dirtystart = self.Output.meta.shape
dirtystop = [0] * len(self.Output.meta.shape)
for block_index in self._fixed_dirty_blocks:
offset = self._blockShape * self._get_block_multi_index(
block_index)
bigstart = offset
bigstop = numpy.minimum(offset + self._blockShape,
self.Output.meta.shape)
dirtystart = numpy.minimum(bigstart, dirtystart)
dirtystop = numpy.maximum(bigstop, dirtystop)
self._fixed_dirty_blocks = set()
# reset all dirty state to false
self._fixed_all_dirty = False
if dirtystart is not None:
self.Output.setDirty(dirtystart, dirtystop)
def test(self, position5d_start, position5d_stop):
from lazyflow.rtype import SubRegion
import numpy
from sitecustomize import debug_trace
roi = SubRegion(self.op.Density, position5d_start,
position5d_stop)
key = roi.toSlice()
key = tuple(k for k in key if k != slice(0,0, None))
newKey = []
for k in key:
if k != slice(0,0,None):
if k.stop < k.start:
k = slice(k.stop, k.start)
newKey.append(k)
newKey = tuple(newKey)
try:
density = numpy.sum(self.op.Density[newKey].wait())
self._labelControlUi.CountText.setText(str(density))
except:
debug_trace()
def testPassThroughAndFixed(self):
self.op.fixAtCurrent.setValue(False)
# we want the last operator to throw an exception when its
# propagateDirty is called, so we can catch it below
def foo():
raise PropagateDirtyCalled()
self.call.function = foo
with self.assertRaises(PropagateDirtyCalled):
self.piper.Input.setValue(self.vol)
roi = self.call.roi
assert_array_equal(roi.start, (0, 0, 0))
assert_array_equal(roi.stop, self.vol.shape)
with self.assertRaises(PropagateDirtyCalled):
self.piper.Input.setDirty(slice(None))
roi = self.call.roi
assert_array_equal(roi.start, (0, 0, 0))
assert_array_equal(roi.stop, self.vol.shape)
self.op.fixAtCurrent.setValue(True)
try:
roi1 = SubRegion(self.piper.Input, (0, 10, 20), (15, 15, 35))
self.piper.Input.setDirty(roi1)
roi2 = SubRegion(self.piper.Input, (15, 9, 1), (20, 14, 20))
self.piper.Input.setDirty(roi2)
except PropagateDirtyCalled:
raise AssertionError("dirtyness is not fixed")
with self.assertRaises(PropagateDirtyCalled):
self.op.fixAtCurrent.setValue(False)
roi = self.call.roi
assert_array_equal(roi.start, (0, 9, 1))
assert_array_equal(roi.stop, (20, 15, 35))
def testFullBlocks(self):
op = OpSplitRequestsBlockwise(True, graph=self.g)
op.Input.connect(self.piper.Output)
op.BlockShape.setValue((10, 15, 20))
op.Output[0:20, 30:45, 10:30].wait()
slot = self.piper.Output
expected = [SubRegion(slot, (0, 30, 0), (10, 45, 20)),
SubRegion(slot, (0, 30, 20), (10, 45, 40)),
SubRegion(slot, (10, 30, 0), (20, 45, 20)),
SubRegion(slot, (10, 30, 20), (20, 45, 40))]
for roi in expected:
filtered = filter(lambda x: x == roi, self.piper.requests)
assert len(filtered) == 1, "missing roi {}".format(roi)
self.piper.requests = []
op.Output[5:14, 32:44, 17:21].wait()
for roi in expected:
filtered = filter(lambda x: x == roi, self.piper.requests)
assert len(filtered) == 1, "missing roi {}".format(roi)
def _calculateFeatures(self, interactive=True):
mainOperator = self.topLevelOperatorView
mainOperator.ObjectCenterImage.setDirty(
SubRegion(mainOperator.ObjectCenterImage))
current_t = self.editor.posModel.time
def _handle_all_finished(*args):
self._lock.acquire()
self.applet.progressSignal(100)
self.topLevelOperatorView._opRegFeats.fixed = True
feats = self.topLevelOperatorView.RegionFeatures[0].wait()
nfeatures = 0
nchannels = 0
try:
for pname, pfeats in feats[0].items():
if pname != default_features_key:
for featname, feat in pfeats.items():
nchannels += feat.shape[1]
nfeatures += 1
if interactive:
self._drawer.featuresSelected.setText(
"{} features computed, {} channels in total".format(
nfeatures, nchannels))
logger.info('Object Extraction: done.')
success = True
except AttributeError:
if interactive:
self._drawer.featuresSelected.setText(
"Feature computation failed (most likely due to memory issues)"
)
logger.error('Object Extraction: failed.')
success = False
self.applet.appletStateUpdateRequested()
self._lock.release()
self.applet.progressSignal(0)
self.applet.progressSignal(-1)
reqs = []
self.already_done = 0
req = mainOperator.RegionFeatures([current_t])
req.submit()
req.notify_failed(self.handleFeatureComputationFailure)
req.notify_finished(_handle_all_finished)
reqs.append(req)
def _prepareTaskInfos(self, roiList):
# Divide up the workload into large pieces
logger.info("Dividing into {} node jobs.".format(len(roiList)))
taskInfos = collections.OrderedDict()
for roiIndex, roi in enumerate(roiList):
roi = (tuple(roi[0]), tuple(roi[1]))
taskInfo = OpClusterize.TaskInfo()
taskInfo.subregion = SubRegion(None, start=roi[0], stop=roi[1])
taskName = "J{:02}".format(roiIndex)
commandArgs = []
commandArgs.append("--option_config_file=" +
self.ConfigFilePath.value)
commandArgs.append("--project=" + self.ProjectFilePath.value)
commandArgs.append("--_node_work_=\"" +
Roi.dumps(taskInfo.subregion) + "\"")
commandArgs.append("--process_name={}".format(taskName))
commandArgs.append("--output_description_file={}".format(
self.OutputDatasetDescription.value))
for slot in self.SecondaryOutputDescriptions:
commandArgs.append(
"--secondary_output_description_file={}".format(
slot.value))
# Check the command format string: We need to know where to put our args...
commandFormat = self._config.command_format
assert commandFormat.find("{task_args}") != -1
# Output log directory might be a relative path (relative to config file)
absLogDir, relLogDir = getPathVariants(
self._config.output_log_directory,
os.path.split(self.ConfigFilePath.value)[0])
taskOutputLogFilename = taskName + ".log"
taskOutputLogPath = os.path.join(absLogDir, taskOutputLogFilename)
allArgs = " " + " ".join(commandArgs) + " "
taskInfo.taskName = taskName
taskInfo.command = commandFormat.format(
task_args=allArgs,
task_name=taskName,
task_output_file=taskOutputLogPath)
taskInfos[roi] = taskInfo
return taskInfos
def _setInSlotInput(self, slot, subindex, roi, new_pixels):
"""
Since this is a label array, inserting pixels has a special meaning:
We only overwrite the new non-zero pixels. In the new data, zeros mean "don't change".
So, here's what each pixel we're adding means:
0: don't change
1: change to 1
2: change to 2
...
N: change to N
magic_eraser_value: change to 0
"""
if isinstance(new_pixels, vigra.VigraArray):
new_pixels = new_pixels.view(numpy.ndarray)
# Extract the data to modify
original_data = self.Output.stype.allocateDestination(
SubRegion(self.Output, *roiFromShape(new_pixels.shape)))
self.execute(self.Output, (), roi, original_data)
# Reset the pixels we need to change (so we can use |= below)
original_data[new_pixels.nonzero()] = 0
# Update
original_data |= new_pixels
# Replace 'eraser' values with zeros.
cleaned_data = original_data.copy()
cleaned_data[original_data == self._eraser_magic_value] = 0
# Set in the cache (our superclass).
super(OpCompressedUserLabelArray,
self)._setInSlotInput(slot,
subindex,
roi,
cleaned_data,
store_zero_blocks=False)
# FIXME: Shouldn't this notification be triggered from within OpUnmanagedCompressedCache?
self.Output.setDirty(roi.start, roi.stop)
return cleaned_data # Internal use: Return the cleaned_data
def propagateDirty(self, slot, subindex, roi):
# the roi here is a list of time steps
if slot is self.RegionCenters:
roi = list(roi)
t = roi[0]
T = t + 1
a = t
for b in roi[1:]:
assert b - 1 == a, "List roi must be contiguous"
a = b
T += 1
time_index = self.BinaryImage.meta.axistags.index('t')
stop = numpy.asarray(self.BinaryImage.meta.shape, dtype=numpy.int)
start = numpy.zeros_like(stop)
stop[time_index] = T
start[time_index] = t
roi = SubRegion(self.Output, tuple(start), tuple(stop))
self.Output.setDirty(roi)
def _export():
self.applet.busy = True
self.applet.appletStateUpdateRequested.emit()
if hasattr(self.mainOperator, "RelabeledImage"):
labelImageSlot = self.mainOperator.RelabeledImage
else:
labelImageSlot = self.mainOperator.LabelImage
logger.info("Saving first label image...")
key = []
for idx, flag in enumerate(
axisTagsToString(labelImageSlot.meta.axistags)):
if flag is 't':
key.append(slice(0, labelImageSlot.meta.shape[idx])
) #slice(t_from,t_from+1))
elif flag is 'c':
key.append(slice(0, 1))
else:
key.append(slice(0, labelImageSlot.meta.shape[idx]))
try:
events = self.mainOperator.EventsVector.value
logger.info("Saving events...")
logger.info("Length of events " + str(len(events)))
num_files = float(len(events))
for i in sorted(events.keys()):
events_at = events[i]
i = int(i)
t = i
key[0] = slice(t, t + 1)
roi = SubRegion(labelImageSlot, key)
labelImage = labelImageSlot.get(roi).wait()
labelImage = labelImage[0, ..., 0]
write_events(events_at, str(directory), t, labelImage)
_handle_progress(i / num_files * 100)
except IOError as e:
self._criticalMessage("Cannot export the tracking results. Maybe these files already exist. "\
"Please delete them or choose a different directory.")
return
def propagateDirty(self, slot, subindex, roi):
# all input slots affect the (global) graph cut computation
if slot == self.Beta:
# beta value affects the whole volume
self.Output.setDirty(slice(None))
elif slot == self.Prediction:
# time-channel slices are pairwise independent
# determine t, c from input volume
t_ind = 0
c_ind = 4
t = (roi.start[t_ind], roi.stop[t_ind])
c = (roi.start[c_ind], roi.stop[c_ind])
# set output dirty
start = t[0:1] + (0,)*3 + c[0:1]
stop = t[1:2] + self.Output.meta.shape[1:4] + c[1:2]
roi = SubRegion(self.Output, start=start, stop=stop)
self.Output.setDirty(roi)
def propagateDirty(self, slot, subindex, roi):
super(OpObjectsSegment, self).propagateDirty(slot, subindex, roi)
if slot == self.LabelImage:
# time-channel slices are pairwise independent
# determine t, c from input volume
t_ind = 0
c_ind = 4
t = (roi.start[t_ind], roi.stop[t_ind])
c = (roi.start[c_ind], roi.stop[c_ind])
# set output dirty
start = t[0:1] + (0, ) * 3 + c[0:1]
stop = t[1:2] + self.Output.meta.shape[1:4] + c[1:2]
roi = SubRegion(self.Output, start=start, stop=stop)
self.Output.setDirty(roi)
elif slot == self.Margin:
# margin affects the whole volume
self.Output.setDirty(slice(None))
def testSetDirty(self):
g = Graph()
vol = np.zeros((200, 100, 10))
vol = vol.astype(np.uint8)
vol = vigra.taggedView(vol, axistags="zyx")
vol[:200, ...] = 1
vol[800:, ...] = 1
op = OpLazyCC(graph=g)
op.ChunkShape.setValue((100, 20, 5))
op.Input.setValue(vol)
opCheck = DirtyAssert(graph=g)
opCheck.Input.connect(op.Output)
out = op.Output[:100, :20, :5].wait()
roi = SubRegion(op.Input, start=(0, 0, 0), stop=(200, 100, 10))
with self.assertRaises(PropagateDirtyCalled):
op.Input.setDirty(roi)
def _calculateFeatures(self, interactive=True):
mainOperator = self.topLevelOperatorView
mainOperator.ObjectCenterImage.setDirty(SubRegion(mainOperator.ObjectCenterImage))
maxt = mainOperator.LabelImage.meta.shape[0]
def _handle_one_finished(*args):
self.already_done = self.already_done+1
self.applet.progressSignal.emit(int(self.already_done*100./maxt))
if self.already_done == maxt:
_handle_all_finished(*args)
def _handle_all_finished(*args):
self.applet.progressSignal.emit(100)
self.topLevelOperatorView._opRegFeats.fixed = True
feats = self.topLevelOperatorView.RegionFeatures[0].wait()
nfeatures = 0
nchannels = 0
for pname, pfeats in feats[0].iteritems():
if pname!='Default features':
for featname, feat in pfeats.iteritems():
nchannels += feat.shape[1]
nfeatures += 1
if interactive:
self._drawer.featuresSelected.setText("{} features computed, {} channels in total".format(nfeatures, nchannels))
logger.info('Object Extraction: done.')
self.applet.progressSignal.emit(0)
self.applet.progressSignal.emit(-1)
reqs = []
self.already_done = 0
for t in range(maxt):
req = mainOperator.RegionFeatures([t])
req.submit()
reqs.append(req)
for req in reqs:
req.notify_failed( self.handleFeatureComputationFailure )
req.notify_finished( _handle_one_finished )
def execute(self, slot, subindex, roi, result):
if slot == self.Dirty:
assert False # Shouldn't get to this line because the dirty output is given a value directly
if slot == self.OutputDataPath:
assert False # This slot is already set via setupOutputs
if slot == self.ExportResult:
# We can stop now if the output isn't dirty
if not self.Dirty.value:
result[0] = True
return
exportFormat = self.Format.value
# Export H5
if exportFormat == ExportFormat.H5:
pathComp = PathComponents(self.OutputDataPath.value)
# Ensure the directory exists
if not os.path.exists(pathComp.externalDirectory):
with self._createDirLock:
# Check again now that we have the lock.
if not os.path.exists(pathComp.externalDirectory):
os.makedirs(pathComp.externalDirectory)
# Open the file
try:
hdf5File = h5py.File(pathComp.externalPath)
except:
logger.error("Unable to open hdf5File: " +
pathComp.externalPath)
logger.error(traceback.format_exc())
result[0] = False
return
# Set up the write operator
opH5Writer = OpH5WriterBigDataset(parent=self)
opH5Writer.hdf5File.setValue(hdf5File)
opH5Writer.hdf5Path.setValue(pathComp.internalPath)
#opH5Writer.Image.connect( self.ImageToExport )
opH5Writer.Image.connect(self.ImageCache.Output)
print "computing predictions for the selected slices:"
self.ImageCache.fixAtCurrent.setValue(False)
#check readiness
for inp in self.ImageCache.inputs:
print inp, self.ImageCache.inputs[inp].ready()
print "input shape:", self.ImageCache.Input.meta.shape
print "output shape:", self.ImageCache.Output.meta.shape
selectedSlices = self.SelectedSlices.value
zaxis = self.ImageToExport.meta.axistags.index('z')
for isl, sl in enumerate(selectedSlices):
print "computing for slice ...", isl
start = [0] * len(self.ImageToExport.meta.shape)
start[zaxis] = sl
stop = list(self.ImageToExport.meta.shape)
stop[zaxis] = sl + 1
roi = SubRegion(self.ImageCache, start=start, stop=stop)
print roi
temp = self.ImageCache.Output[roi.toSlice()].wait()
#print temp
self.ImageCache.fixAtCurrent.setValue(True)
#tstart = [0]*len(self.ImageToExport.meta.shape)
#tstop = list(self.ImageToExport.meta.shape)
#troi = SubRegion(self.ImageCache, start=tstart, stop=tstop)
#tttemp = self.ImageCache.Output[troi.toSlice()].wait()
#print tttemp
# The H5 Writer provides it's own progress signal, so just connect ours to it.
opH5Writer.progressSignal.subscribe(self.progressSignal)
# Trigger the write
self.Dirty.setValue(not opH5Writer.WriteImage.value)
hdf5File.close()
opH5Writer.cleanUp()
# elif exportFormat == ExportFormat.Npy:
# assert False # TODO
# elif exportFormat == ExportFormat.Npy:
# assert False # TODO
else:
assert False, "Unknown export format"
result[0] = not self.Dirty.value
def execute(self, slot, subindex, slot_roi, target):
assert slot == self.Features or slot == self.Output
if slot == self.Features:
feature_slice = roiToSlice(slot_roi.start, slot_roi.stop)
index = subindex[0]
feature_slice = list(feature_slice)
# Translate channel slice of this feature to the channel slice of the output slot.
output_channel_offset = self.featureOutputChannels[index][0]
feature_slice[1] = slice(
output_channel_offset + feature_slice[1].start,
output_channel_offset + feature_slice[1].stop)
slot_roi = SubRegion(self.Output, pslice=feature_slice)
# Get output slot region for this channel
return self.execute(self.Output, (), slot_roi, target)
elif slot == self.Output:
# Correlation of variable 'families' representing reference frames:
# ______________________________
# | input/output frame | input/output shape given by slots
# | _________________________ |
# | | smooth frame | | pre-smoothing op needs halo around filter roi
# | | ____________________ | |
# | | |filter frame | | | filter needs halo around target roi
# | | | _______________ | | |
# | | | | target frame | | | | target is given by output_roi
# note: The 'full_' variable prefix refers to the full 5D shape (tczyx), without 'full_' variables mostly
# refer to the 3D space subregion (zyx)
full_output_slice = slot_roi.toSlice()
logger.debug(
f"OpPixelFeaturesPresmoothed: request {slot_roi.pprint()}")
assert (slot_roi.stop <= self.Output.meta.shape).all()
full_output_shape = self.Output.meta.shape
full_output_start, full_output_stop = sliceToRoi(
full_output_slice, full_output_shape)
assert len(full_output_shape) == 5
if all(self.ComputeIn2d.value
): # todo: check for this particular slice
axes2enlarge = (0, 1, 1)
else:
axes2enlarge = (1, 1, 1)
output_shape = full_output_shape[2:]
output_start = full_output_start[2:]
output_stop = full_output_stop[2:]
axistags = self.Output.meta.axistags
target = target.view(vigra.VigraArray)
target.axistags = copy.copy(axistags)
# filter roi in input frame
# sigma = 0.7, because the features receive a pre-smoothed array and don't need much of a neighborhood
input_filter_start, input_filter_stop = roi.enlargeRoiForHalo(
output_start,
output_stop,
output_shape,
0.7,
self.WINDOW_SIZE,
enlarge_axes=axes2enlarge)
# smooth roi in input frame
input_smooth_start, input_smooth_stop = roi.enlargeRoiForHalo(
input_filter_start,
input_filter_stop,
output_shape,
self.max_sigma,
self.WINDOW_SIZE,
enlarge_axes=axes2enlarge,
)
# target roi in filter frame
filter_target_start = roi.TinyVector(output_start -
input_filter_start)
filter_target_stop = roi.TinyVector(output_stop -
input_filter_start)
# filter roi in smooth frame
smooth_filter_start = roi.TinyVector(input_filter_start -
input_smooth_start)
smooth_filter_stop = roi.TinyVector(input_filter_stop -
input_smooth_start)
filter_target_slice = roi.roiToSlice(filter_target_start,
filter_target_stop)
input_smooth_slice = roi.roiToSlice(input_smooth_start,
input_smooth_stop)
# pre-smooth for all requested time slices and all channels
full_input_smooth_slice = (full_output_slice[0], slice(None),
*input_smooth_slice)
req = self.Input[full_input_smooth_slice]
source = req.wait()
req.clean()
req.destination = None
if source.dtype != numpy.float32:
sourceF = source.astype(numpy.float32)
try:
source.resize((1, ), refcheck=False)
except Exception:
pass
del source
source = sourceF
sourceV = source.view(vigra.VigraArray)
sourceV.axistags = copy.copy(self.Input.meta.axistags)
dimCol = len(self.scales)
dimRow = self.matrix.shape[0]
presmoothed_source = [None] * dimCol
source_smooth_shape = tuple(smooth_filter_stop -
smooth_filter_start)
full_source_smooth_shape = (
full_output_stop[0] - full_output_start[0],
self.Input.meta.shape[1],
) + source_smooth_shape
try:
for j in range(dimCol):
for i in range(dimRow):
if self.matrix[i, j]:
# There is at least one filter op with this scale
break
else:
# There is no filter op at this scale
continue
if self.scales[j] > 1.0:
tempSigma = math.sqrt(self.scales[j]**2 - 1.0)
else:
tempSigma = self.scales[j]
presmoothed_source[j] = numpy.ndarray(
full_source_smooth_shape, numpy.float32)
droi = (
(0, *tuple(smooth_filter_start._asint())),
(sourceV.shape[1],
*tuple(smooth_filter_stop._asint())),
)
for i, vsa in enumerate(sourceV.timeIter()):
presmoothed_source[j][
i, ...] = self._computeGaussianSmoothing(
vsa,
tempSigma,
droi,
in2d=self.ComputeIn2d.value[j])
except RuntimeError as e:
if "kernel longer than line" in str(e):
raise RuntimeError(
"Feature computation error:\nYour image is too small to apply a filter with "
f"sigma={self.scales[j]:.1f}. Please select features with smaller sigmas."
)
else:
raise e
del sourceV
try:
source.resize((1, ), refcheck=False)
except ValueError:
# Sometimes this fails, but that's okay.
logger.debug("Failed to free array memory.")
del source
cnt = 0
written = 0
closures = []
# connect individual operators
for i in range(dimRow):
for j in range(dimCol):
if self.matrix[i, j]:
oslot = self.featureOps[i][j].Output
req = None
slices = oslot.meta.shape[1]
if (cnt + slices >= slot_roi.start[1]
and slot_roi.start[1] - cnt < slices
and slot_roi.start[1] + written <
slot_roi.stop[1]):
begin = 0
if cnt < slot_roi.start[1]:
begin = slot_roi.start[1] - cnt
end = slices
if cnt + end > slot_roi.stop[1]:
end = slot_roi.stop[1] - cnt
# feature slice in output frame
feature_slice = (slice(None),
slice(
written, written + end -
begin)) + (slice(None), ) * 3
subtarget = target[feature_slice]
# readjust the roi for the new source array
full_filter_target_slice = [
full_output_slice[0],
slice(begin, end), *filter_target_slice
]
filter_target_roi = SubRegion(
oslot, pslice=full_filter_target_slice)
closure = partial(
oslot.operator.execute,
oslot,
(),
filter_target_roi,
subtarget,
sourceArray=presmoothed_source[j],
)
closures.append(closure)
written += end - begin
cnt += slices
pool = RequestPool()
for c in closures:
pool.request(c)
pool.wait()
pool.clean()
for i in range(len(presmoothed_source)):
if presmoothed_source[i] is not None:
try:
presmoothed_source[i].resize((1, ))
except Exception:
presmoothed_source[i] = None
def execute(self, slot, subindex, roi, result):
if slot == self.Dirty:
assert False # Shouldn't get to this line because the dirty output is given a value directly
if slot == self.OutputDataPath:
assert False # This slot is already set via setupOutputs
if slot == self.ExportResult:
# We can stop now if the output isn't dirty
if not self.Dirty.value:
result[0] = True
return
exportFormat = self.Format.value
# Export H5
if exportFormat == ExportFormat.H5:
pathComp = PathComponents(self.OutputDataPath.value)
# Ensure the directory exists
if not os.path.exists(pathComp.externalDirectory):
with self._createDirLock:
# Check again now that we have the lock.
if not os.path.exists(pathComp.externalDirectory):
os.makedirs(pathComp.externalDirectory)
# Open the file
try:
hdf5File = h5py.File(pathComp.externalPath)
except:
logger.error("Unable to open hdf5File: " + pathComp.externalPath)
logger.error( traceback.format_exc() )
result[0] = False
return
# Set up the write operator
opH5Writer = OpH5WriterBigDataset(parent=self)
opH5Writer.hdf5File.setValue( hdf5File )
opH5Writer.hdf5Path.setValue( pathComp.internalPath )
#opH5Writer.Image.connect( self.ImageToExport )
opH5Writer.Image.connect(self.ImageCache.Output)
print "computing predictions for the selected slices:"
self.ImageCache.fixAtCurrent.setValue(False)
#check readiness
for inp in self.ImageCache.inputs:
print inp, self.ImageCache.inputs[inp].ready()
print "input shape:", self.ImageCache.Input.meta.shape
print "output shape:", self.ImageCache.Output.meta.shape
selectedSlices = self.SelectedSlices.value
zaxis = self.ImageToExport.meta.axistags.index('z')
for isl, sl in enumerate(selectedSlices):
print "computing for slice ...", isl
start = [0]*len(self.ImageToExport.meta.shape)
start[zaxis]=sl
stop = list(self.ImageToExport.meta.shape)
stop[zaxis]=sl+1
roi = SubRegion(self.ImageCache, start=start, stop=stop)
print roi
temp = self.ImageCache.Output[roi.toSlice()].wait()
#print temp
self.ImageCache.fixAtCurrent.setValue(True)
#tstart = [0]*len(self.ImageToExport.meta.shape)
#tstop = list(self.ImageToExport.meta.shape)
#troi = SubRegion(self.ImageCache, start=tstart, stop=tstop)
#tttemp = self.ImageCache.Output[troi.toSlice()].wait()
#print tttemp
# The H5 Writer provides it's own progress signal, so just connect ours to it.
opH5Writer.progressSignal.subscribe( self.progressSignal )
# Trigger the write
self.Dirty.setValue( not opH5Writer.WriteImage.value )
hdf5File.close()
opH5Writer.cleanUp()
# elif exportFormat == ExportFormat.Npy:
# assert False # TODO
# elif exportFormat == ExportFormat.Npy:
# assert False # TODO
else:
assert False, "Unknown export format"
result[0] = not self.Dirty.value
def execute(self, slot, subindex, slot_roi, target):
assert slot == self.Features or slot == self.Output
if slot == self.Features:
feature_slice = roiToSlice(slot_roi.start, slot_roi.stop)
index = subindex[0]
feature_slice = list(feature_slice)
# Translate channel slice of this feature to the channel slice of the output slot.
output_channel_offset = self.featureOutputChannels[index][0]
feature_slice[1] = slice(
output_channel_offset + feature_slice[1].start, output_channel_offset + feature_slice[1].stop
)
slot_roi = SubRegion(self.Output, pslice=feature_slice)
# Get output slot region for this channel
return self.execute(self.Output, (), slot_roi, target)
elif slot == self.Output:
# Correlation of variable 'families' representing reference frames:
# ______________________________
# | input/output frame | input/output shape given by slots
# | _________________________ |
# | | smooth frame | | pre-smoothing op needs halo around filter roi
# | | ____________________ | |
# | | |filter frame | | | filter needs halo around target roi
# | | | _______________ | | |
# | | | | target frame | | | | target is given by output_roi
# note: The 'full_' variable prefix refers to the full 5D shape (tczyx), without 'full_' variables mostly
# refer to the 3D space subregion (zyx)
full_output_slice = slot_roi.toSlice()
logger.debug(f"OpPixelFeaturesPresmoothed: request {slot_roi.pprint()}")
assert (slot_roi.stop <= self.Output.meta.shape).all()
full_output_shape = self.Output.meta.shape
full_output_start, full_output_stop = sliceToRoi(full_output_slice, full_output_shape)
assert len(full_output_shape) == 5
if all(self.ComputeIn2d.value): # todo: check for this particular slice
axes2enlarge = (0, 1, 1)
else:
axes2enlarge = (1, 1, 1)
output_shape = full_output_shape[2:]
output_start = full_output_start[2:]
output_stop = full_output_stop[2:]
axistags = self.Output.meta.axistags
target = target.view(vigra.VigraArray)
target.axistags = copy.copy(axistags)
# filter roi in input frame
# sigma = 0.7, because the features receive a pre-smoothed array and don't need much of a neighborhood
input_filter_start, input_filter_stop = roi.enlargeRoiForHalo(
output_start, output_stop, output_shape, 0.7, self.WINDOW_SIZE, enlarge_axes=axes2enlarge
)
# smooth roi in input frame
input_smooth_start, input_smooth_stop = roi.enlargeRoiForHalo(
input_filter_start,
input_filter_stop,
output_shape,
self.max_sigma,
self.WINDOW_SIZE,
enlarge_axes=axes2enlarge,
)
# target roi in filter frame
filter_target_start = roi.TinyVector(output_start - input_filter_start)
filter_target_stop = roi.TinyVector(output_stop - input_filter_start)
# filter roi in smooth frame
smooth_filter_start = roi.TinyVector(input_filter_start - input_smooth_start)
smooth_filter_stop = roi.TinyVector(input_filter_stop - input_smooth_start)
filter_target_slice = roi.roiToSlice(filter_target_start, filter_target_stop)
input_smooth_slice = roi.roiToSlice(input_smooth_start, input_smooth_stop)
# pre-smooth for all requested time slices and all channels
full_input_smooth_slice = (full_output_slice[0], slice(None), *input_smooth_slice)
req = self.Input[full_input_smooth_slice]
source = req.wait()
req.clean()
req.destination = None
if source.dtype != numpy.float32:
sourceF = source.astype(numpy.float32)
try:
source.resize((1,), refcheck=False)
except Exception:
pass
del source
source = sourceF
sourceV = source.view(vigra.VigraArray)
sourceV.axistags = copy.copy(self.Input.meta.axistags)
dimCol = len(self.scales)
dimRow = self.matrix.shape[0]
presmoothed_source = [None] * dimCol
source_smooth_shape = tuple(smooth_filter_stop - smooth_filter_start)
full_source_smooth_shape = (
full_output_stop[0] - full_output_start[0],
self.Input.meta.shape[1],
) + source_smooth_shape
try:
for j in range(dimCol):
for i in range(dimRow):
if self.matrix[i, j]:
# There is at least one filter op with this scale
break
else:
# There is no filter op at this scale
continue
if self.scales[j] > 1.0:
tempSigma = math.sqrt(self.scales[j] ** 2 - 1.0)
else:
tempSigma = self.scales[j]
presmoothed_source[j] = numpy.ndarray(full_source_smooth_shape, numpy.float32)
droi = (
(0, *tuple(smooth_filter_start._asint())),
(sourceV.shape[1], *tuple(smooth_filter_stop._asint())),
)
for i, vsa in enumerate(sourceV.timeIter()):
presmoothed_source[j][i, ...] = self._computeGaussianSmoothing(
vsa, tempSigma, droi, in2d=self.ComputeIn2d.value[j]
)
except RuntimeError as e:
if "kernel longer than line" in str(e):
raise RuntimeError(
"Feature computation error:\nYour image is too small to apply a filter with "
f"sigma={self.scales[j]:.1f}. Please select features with smaller sigmas."
)
else:
raise e
del sourceV
try:
source.resize((1,), refcheck=False)
except ValueError:
# Sometimes this fails, but that's okay.
logger.debug("Failed to free array memory.")
del source
cnt = 0
written = 0
closures = []
# connect individual operators
for i in range(dimRow):
for j in range(dimCol):
if self.matrix[i, j]:
oslot = self.featureOps[i][j].Output
req = None
slices = oslot.meta.shape[1]
if (
cnt + slices >= slot_roi.start[1]
and slot_roi.start[1] - cnt < slices
and slot_roi.start[1] + written < slot_roi.stop[1]
):
begin = 0
if cnt < slot_roi.start[1]:
begin = slot_roi.start[1] - cnt
end = slices
if cnt + end > slot_roi.stop[1]:
end = slot_roi.stop[1] - cnt
# feature slice in output frame
feature_slice = (slice(None), slice(written, written + end - begin)) + (slice(None),) * 3
subtarget = target[feature_slice]
# readjust the roi for the new source array
full_filter_target_slice = [full_output_slice[0], slice(begin, end), *filter_target_slice]
filter_target_roi = SubRegion(oslot, pslice=full_filter_target_slice)
closure = partial(
oslot.operator.execute,
oslot,
(),
filter_target_roi,
subtarget,
sourceArray=presmoothed_source[j],
)
closures.append(closure)
written += end - begin
cnt += slices
pool = RequestPool()
for c in closures:
pool.request(c)
pool.wait()
pool.clean()
for i in range(len(presmoothed_source)):
if presmoothed_source[i] is not None:
try:
presmoothed_source[i].resize((1,))
except Exception:
presmoothed_source[i] = None
def _setInSlotInput(self, slot, subindex, roi, new_pixels):
"""
Since this is a label array, inserting pixels has a special meaning:
We only overwrite the new non-zero pixels. In the new data, zeros mean "don't change".
So, here's what each pixel we're adding means:
0: don't change
1: change to 1
2: change to 2
...
N: change to N
eraser_magic_value: change to 0
"""
if isinstance(new_pixels, vigra.VigraArray):
new_pixels = new_pixels.view(numpy.ndarray)
# Get logical blocking.
block_rois = getIntersectingRois(self.Output.meta.shape, self._blockshape, (roi.start, roi.stop))
# Convert to tuples
block_rois = [(tuple(start), tuple(stop)) for start, stop in block_rois]
max_label = 0
for block_roi in block_rois:
roi_within_data = numpy.array(block_roi) - roi.start
new_block_pixels = new_pixels[roiToSlice(*roi_within_data)]
# Shortcut: Nothing to change if this block is all zeros.
if not new_block_pixels.any():
continue
block_slot_roi = SubRegion(self.Output, *block_roi)
# Extract the data to modify
original_block_data = self.Output.stype.allocateDestination(block_slot_roi)
self.execute(self.Output, (), block_slot_roi, original_block_data)
# Reset the pixels we need to change (so we can use |= below)
original_block_data[new_block_pixels.nonzero()] = 0
# Update
original_block_data |= new_block_pixels
# Replace 'eraser' values with zeros.
cleaned_block_data = original_block_data.copy()
cleaned_block_data[original_block_data == self._eraser_magic_value] = 0
# Set in the cache (our superclass).
super(OpCompressedUserLabelArray, self)._setInSlotInput(
slot, subindex, block_slot_roi, cleaned_block_data, store_zero_blocks=False
)
max_label = max(max_label, cleaned_block_data.max())
# We could wait to send out one big dirty notification (instead of one per block),
# But that might result in a lot of unecessarily dirty pixels in cases when the
# new_pixels were mostly empty (such as when importing labels from disk).
# That's bad for downstream operators like OpFeatureMatrixCache
# So instead, we only send notifications for the blocks that were touched.
# During labeling, it makes no difference.
# During project import, this is slightly worse.
# But during label import from disk, this is very important.a
# FIXME: Shouldn't this notification be triggered from within OpUnmanagedCompressedCache?
self.Output.setDirty(*block_roi)
return max_label # Internal use: Return max label
def _export():
self.applet.busy = True
self.applet.appletStateUpdateRequested.emit()
t_from = None
# determine from_time (it could has been changed in the GUI meanwhile)
for t_from, label2color_at in enumerate(
self.mainOperator.label2color):
if len(label2color_at) == 0:
continue
else:
break
if t_from is None:
self._criticalMessage("There is nothing to export.")
return
t_from = int(t_from)
logger.info("Saving first label image...")
key = []
for idx, flag in enumerate(
axisTagsToString(
self.mainOperator.LabelImage.meta.axistags)):
if flag is 't':
key.append(slice(t_from, t_from + 1))
elif flag is 'c':
key.append(slice(0, 1))
else:
key.append(
slice(0, self.mainOperator.LabelImage.meta.shape[idx]))
roi = SubRegion(self.mainOperator.LabelImage, key)
labelImage = self.mainOperator.LabelImage.get(roi).wait()
labelImage = labelImage[0, ..., 0]
try:
# write_events([], str(directory), t_from, labelImage)
events = self.mainOperator.EventsVector.value
logger.info("Saving events...")
logger.info("Length of events " + str(len(events)))
num_files = float(len(events))
for i in sorted(events.keys()):
events_at = events[i]
i = int(i)
t = t_from + i
key[0] = slice(t, t + 1)
roi = SubRegion(self.mainOperator.LabelImage, key)
labelImage = self.mainOperator.LabelImage.get(roi).wait()
labelImage = labelImage[0, ..., 0]
if self.withMergers:
write_events(events_at, str(directory), t, labelImage,
self.mainOperator.mergers)
else:
write_events(events_at, str(directory), t, labelImage)
_handle_progress(i / num_files * 100)
except IOError as e:
self._criticalMessage("Cannot export the tracking results. Maybe these files already exist. "\
"Please delete them or choose a different directory.")
return
def execute(self, slot, subindex, rroi, result):
assert slot == self.Features or slot == self.Output
if slot == self.Features:
key = roiToSlice(rroi.start, rroi.stop)
index = subindex[0]
subslot = self.Features[index]
key = list(key)
channelIndex = self.Input.meta.axistags.index('c')
# Translate channel slice to the correct location for the output slot.
key[channelIndex] = slice(self.featureOutputChannels[index][0] + key[channelIndex].start,
self.featureOutputChannels[index][0] + key[channelIndex].stop)
rroi = SubRegion(subslot, pslice=key)
# Get output slot region for this channel
return self.execute(self.Output, (), rroi, result)
elif slot == self.outputs["Output"]:
key = rroi.toSlice()
cnt = 0
written = 0
assert (rroi.stop<=self.outputs["Output"].meta.shape).all()
flag = 'c'
channelAxis=self.inputs["Input"].meta.axistags.index('c')
axisindex = channelAxis
oldkey = list(key)
oldkey.pop(axisindex)
inShape = self.inputs["Input"].meta.shape
shape = self.outputs["Output"].meta.shape
axistags = self.inputs["Input"].meta.axistags
result = result.view(vigra.VigraArray)
result.axistags = copy.copy(axistags)
hasTimeAxis = self.inputs["Input"].meta.axistags.axisTypeCount(vigra.AxisType.Time)
timeAxis=self.inputs["Input"].meta.axistags.index('t')
subkey = popFlagsFromTheKey(key,axistags,'c')
subshape=popFlagsFromTheKey(shape,axistags,'c')
at2 = copy.copy(axistags)
at2.dropChannelAxis()
subshape=popFlagsFromTheKey(subshape,at2,'t')
subkey = popFlagsFromTheKey(subkey,at2,'t')
oldstart, oldstop = roi.sliceToRoi(key, shape)
start, stop = roi.sliceToRoi(subkey,subkey)
maxSigma = max(0.7,self.maxSigma)
# The region of the smoothed image we need to give to the feature filter (in terms of INPUT coordinates)
vigOpSourceStart, vigOpSourceStop = roi.extendSlice(start, stop, subshape, 0.7, window = 2)
# The region of the input that we need to give to the smoothing operator (in terms of INPUT coordinates)
newStart, newStop = roi.extendSlice(vigOpSourceStart, vigOpSourceStop, subshape, maxSigma, window = 3.5)
# Translate coordinates (now in terms of smoothed image coordinates)
vigOpSourceStart = roi.TinyVector(vigOpSourceStart - newStart)
vigOpSourceStop = roi.TinyVector(vigOpSourceStop - newStart)
readKey = roi.roiToSlice(newStart, newStop)
writeNewStart = start - newStart
writeNewStop = writeNewStart + stop - start
treadKey=list(readKey)
if hasTimeAxis:
if timeAxis < channelAxis:
treadKey.insert(timeAxis, key[timeAxis])
else:
treadKey.insert(timeAxis-1, key[timeAxis])
if self.inputs["Input"].meta.axistags.axisTypeCount(vigra.AxisType.Channels) == 0:
treadKey = popFlagsFromTheKey(treadKey,axistags,'c')
else:
treadKey.insert(channelAxis, slice(None,None,None))
treadKey=tuple(treadKey)
req = self.inputs["Input"][treadKey].allocate()
sourceArray = req.wait()
req.result = None
req.destination = None
if sourceArray.dtype != numpy.float32:
sourceArrayF = sourceArray.astype(numpy.float32)
sourceArray.resize((1,), refcheck = False)
del sourceArray
sourceArray = sourceArrayF
sourceArrayV = sourceArray.view(vigra.VigraArray)
sourceArrayV.axistags = copy.copy(axistags)
dimCol = len(self.scales)
dimRow = self.matrix.shape[0]
sourceArraysForSigmas = [None]*dimCol
#connect individual operators
for j in range(dimCol):
hasScale = False
for i in range(dimRow):
if self.matrix[i,j]:
hasScale = True
if not hasScale:
continue
destSigma = 1.0
if self.scales[j] > destSigma:
tempSigma = math.sqrt(self.scales[j]**2 - destSigma**2)
else:
destSigma = 0.0
tempSigma = self.scales[j]
vigOpSourceShape = list(vigOpSourceStop - vigOpSourceStart)
if hasTimeAxis:
if timeAxis < channelAxis:
vigOpSourceShape.insert(timeAxis, ( oldstop - oldstart)[timeAxis])
else:
vigOpSourceShape.insert(timeAxis-1, ( oldstop - oldstart)[timeAxis])
vigOpSourceShape.insert(channelAxis, inShape[channelAxis])
sourceArraysForSigmas[j] = numpy.ndarray(tuple(vigOpSourceShape),numpy.float32)
for i,vsa in enumerate(sourceArrayV.timeIter()):
droi = (tuple(vigOpSourceStart._asint()), tuple(vigOpSourceStop._asint()))
tmp_key = getAllExceptAxis(len(sourceArraysForSigmas[j].shape),timeAxis, i)
sourceArraysForSigmas[j][tmp_key] = vigra.filters.gaussianSmoothing(vsa,tempSigma, roi = droi, window_size = 3.5 )
else:
droi = (tuple(vigOpSourceStart._asint()), tuple(vigOpSourceStop._asint()))
#print droi, sourceArray.shape, tempSigma,self.scales[j]
sourceArraysForSigmas[j] = vigra.filters.gaussianSmoothing(sourceArrayV, sigma = tempSigma, roi = droi, window_size = 3.5)
#sourceArrayForSigma = sourceArrayForSigma.view(numpy.ndarray)
del sourceArrayV
try:
sourceArray.resize((1,), refcheck = False)
except ValueError:
# Sometimes this fails, but that's okay.
logger.debug("Failed to free array memory.")
del sourceArray
closures = []
#connect individual operators
for i in range(dimRow):
for j in range(dimCol):
val=self.matrix[i,j]
if val:
vop= self.featureOps[i][j]
oslot = vop.outputs["Output"]
req = None
inTagKeys = [ax.key for ax in oslot.meta.axistags]
if flag in inTagKeys:
slices = oslot.meta.shape[axisindex]
if cnt + slices >= rroi.start[axisindex] and rroi.start[axisindex]-cnt<slices and rroi.start[axisindex]+written<rroi.stop[axisindex]:
begin = 0
if cnt < rroi.start[axisindex]:
begin = rroi.start[axisindex] - cnt
end = slices
if cnt + end > rroi.stop[axisindex]:
end -= cnt + end - rroi.stop[axisindex]
key_ = copy.copy(oldkey)
key_.insert(axisindex, slice(begin, end, None))
reskey = [slice(None, None, None) for x in range(len(result.shape))]
reskey[axisindex] = slice(written, written+end-begin, None)
destArea = result[tuple(reskey)]
roi_ = SubRegion(self.Input, pslice=key_)
closure = partial(oslot.operator.execute, oslot, (), roi_, destArea, sourceArray = sourceArraysForSigmas[j])
closures.append(closure)
written += end - begin
cnt += slices
else:
if cnt>=rroi.start[axisindex] and rroi.start[axisindex] + written < rroi.stop[axisindex]:
reskey = copy.copy(oldkey)
reskey.insert(axisindex, written)
#print "key: ", key, "reskey: ", reskey, "oldkey: ", oldkey
#print "result: ", result.shape, "inslot:", inSlot.shape
destArea = result[tuple(reskey)]
logger.debug(oldkey, destArea.shape, sourceArraysForSigmas[j].shape)
oldroi = SubRegion(self.Input, pslice=oldkey)
closure = partial(oslot.operator.execute, oslot, (), oldroi, destArea, sourceArray = sourceArraysForSigmas[j])
closures.append(closure)
written += 1
cnt += 1
pool = Pool()
for c in closures:
r = pool.request(c)
pool.wait()
pool.clean()
for i in range(len(sourceArraysForSigmas)):
if sourceArraysForSigmas[i] is not None:
try:
sourceArraysForSigmas[i].resize((1,))
except:
sourceArraysForSigmas[i] = None
def execute(self, slot, subindex, roi, result):
assert (roi.start >= 0).all(), \
"Requested roi is out-of-bounds: [{}, {}]".format( roi.start, roi.stop )
assert (roi.stop <= self.Input.meta.shape).all(), \
"Requested roi is out-of-bounds: [{}, {}] (exceeds shape {})"\
.format( roi.start, roi.stop, self.Input.meta.shape )
if not self._configured:
# this happens when the operator is not yet fully configured due to fixAtCurrent == True
result[:] = 0
return
t = time.time()
start, stop = roi.start, roi.stop
blockStart = (start / self._blockShape)
blockStop = (stop * 1.0 / self._blockShape).ceil()
innerBlocks = self._get_block_numbers(blockStart, blockStop)
pool = RequestPool()
for block_index in innerBlocks.flat:
#which part of the original key does this block fill?
block_multi_index = self._get_block_multi_index(block_index)
offset = self._blockShape * block_multi_index
bigstart = numpy.maximum(offset, start)
bigstop = numpy.minimum(offset + self._blockShape, stop)
smallstart = bigstart - offset
smallstop = bigstop - offset
bigkey = roiToSlice(bigstart - start, bigstop - start)
with self._lock:
if not self._fixed:
if not self._cache_list.has_key(block_index):
self._opSub_list[block_index] = generic.OpSubRegion(
parent=self)
self._opSub_list[block_index].inputs["Input"].connect(
self.inputs["Input"])
tstart = self._blockShape * block_multi_index
tstop = numpy.minimum((block_multi_index + numpy.ones(
block_multi_index.shape, numpy.uint8)) *
self._blockShape, self.shape)
self._opSub_list[block_index].Roi.setValue(
(tuple(tstart), tuple(tstop)))
self._cache_list[block_index] = OpArrayCache(
parent=self)
self._cache_list[block_index].inputs["Input"].connect(
self._opSub_list[block_index].outputs["Output"])
self._cache_list[block_index].inputs[
"fixAtCurrent"].connect(self.fixAtCurrent)
self._cache_list[block_index].inputs[
"blockShape"].setValue(
self.inputs["innerBlockShape"].value)
# we dont register a callback for dirtyness, since we already forward the signal
# Forward value changed notifications to our own output.
self._cache_list[
block_index].Output.notifyValueChanged(
self.Output._sig_value_changed)
if self._cache_list.has_key(block_index):
op = self._cache_list[block_index]
#req = self._cache_list[block_index].outputs["Output"][smallkey].writeInto(result[bigkey])
smallroi = SubRegion(op.outputs["Output"],
start=smallstart,
stop=smallstop)
req = op.Output(smallroi.start, smallroi.stop)
req.writeInto(result[bigkey])
pool.add(req)
#op.execute(op.outputs["Output"], (), smallroi, result[bigkey])
####op.getOutSlot(op.outputs["Output"],smallkey,result[bigkey])
#requests.append(req)
else:
#When this block has never been in the cache and the current
#value is fixed (fixAtCurrent=True), return 0 values
#This prevents random noise appearing in such cases.
result[bigkey] = 0
with self._lock:
# Since a downstream operator has expressed an interest in this block,
# mark it to be signaled as dirty when we become unfixed.
# Otherwise, downstream operators won't know when there's valid data in this block.
self._fixed_dirty_blocks.add(block_index)
pool.wait()
self.logger.debug("read %r took %f msec." % (roi.pprint(), 1000.0 *
(time.time() - t)))
