def _store_block_data(self, block_roi, block_data):
"""
Copy block_data and store it into the cache.
The block_lock is not obtained here, so lock it before you call this.
"""
with self._lock:
if self.CompressionEnabled.value and numpy.dtype(
block_data.dtype) in [
numpy.dtype(numpy.uint8),
numpy.dtype(numpy.uint32),
numpy.dtype(numpy.float32)
]:
compressed_block = vigra.ChunkedArrayCompressed(
block_data.shape, vigra.Compression.LZ4, block_data.dtype)
compressed_block[:] = block_data
block_storage_data = compressed_block
else:
block_storage_data = block_data.copy()
# Store the data.
# First double-check that the block wasn't removed from the
# cache while we were requesting it.
# (Could have happened via propagateDirty() or eventually the arrayCacheMemoryMgr)
if block_roi in self._block_locks:
self._block_data[block_roi] = block_storage_data
self._last_access_times[block_roi] = time.time()
def save_debug_image(name, image, out_debug_image_dict):
"""
If output_debug_image_dict isn't None, save the
given image in the dict as a compressed array.
"""
if out_debug_image_dict is None:
return
if hasattr(image, 'axistags'):
axistags = image.axistags
else:
axistags = None
out_debug_image_dict[name] = vigra.ChunkedArrayCompressed(
image.shape, dtype=image.dtype, axistags=axistags)
out_debug_image_dict[name][:] = image
def _setDefaultInternals(self):
# chunk array shape calculation
shape = self._Input.meta.shape
if self.ChunkShape.ready():
chunkShape = (1, ) + self.ChunkShape.value + (1, )
elif self._Input.meta.ideal_blockshape is not None and\
np.prod(self._Input.meta.ideal_blockshape) > 0:
chunkShape = self._Input.meta.ideal_blockshape
else:
chunkShape = self._automaticChunkShape(self._Input.meta.shape)
assert len(shape) == len(chunkShape),\
"Encountered an invalid chunkShape"
chunkShape = np.minimum(shape, chunkShape)
f = lambda i: shape[i] // chunkShape[i] + (1 if shape[i] % chunkShape[
i] else 0)
self._chunkArrayShape = tuple(map(f, range(len(shape))))
self._chunkShape = np.asarray(chunkShape, dtype=np.int)
self._shape = shape
# determine the background values
self._background = np.zeros((shape[0], shape[4]),
dtype=self.Input.meta.dtype)
if self.Background.ready():
bg = self.Background[...].wait()
bg = vigra.taggedView(bg, axistags="txyzc").withAxes('t', 'c')
# we might have an old value set for the background value
# ignore it until it is configured correctly, or execute is called
if bg.size > 1 and \
(shape[0] != bg.shape[0] or shape[4] != bg.shape[1]):
self._background_valid = False
else:
self._background_valid = True
self._background[:] = bg
else:
self._background_valid = True
# manager object
self._manager = _LabelManager()
### local labels ###
# cache for local labels
# adjust cache chunk shape to our chunk shape
cs = tuple(map(_get_next_power, self._chunkShape))
logger.debug("Creating cache with chunk shape {}".format(cs))
self._cache = vigra.ChunkedArrayCompressed(shape,
dtype=_LABEL_TYPE,
chunk_shape=cs)
### global indices ###
# offset (global labels - local labels) per chunk
self._globalLabelOffset = np.ones(self._chunkArrayShape,
dtype=_LABEL_TYPE)
# keep track of number of indices in chunk (-1 == not labeled yet)
self._numIndices = -np.ones(self._chunkArrayShape, dtype=np.int32)
# union find data structure, tells us for every global index to which
# label it belongs
self._uf = UnionFindArray(_LABEL_TYPE(1))
### global labels ###
# keep track of assigned global labels
gen = partial(InfiniteLabelIterator, 1, dtype=_LABEL_TYPE)
self._labelIterators = defaultdict(gen)
self._globalToFinal = defaultdict(dict)
self._isFinal = np.zeros(self._chunkArrayShape, dtype=np.bool)
### algorithmic ###
# keep track of merged regions
self._mergeMap = defaultdict(list)
# locks that keep threads from changing a specific chunk
self._chunk_locks = defaultdict(HardLock)
def execute(self, slot, subindex, roi, result):
with self._lock:
# Does this roi happen to fit ENTIRELY within an existing stored block?
outer_rois = containing_rois(self._block_data.keys(),
(roi.start, roi.stop))
if len(outer_rois) > 0:
# Use the first one we found
block_roi = self._standardize_roi(*outer_rois[0])
block_relative_roi = numpy.array(
(roi.start, roi.stop)) - block_roi[0]
self.Output.stype.copy_data(
result, self._block_data[block_roi][roiToSlice(
*block_relative_roi)])
return
# Standardize roi for usage as dict key
block_roi = self._standardize_roi(roi.start, roi.stop)
# Get lock for this block (create first if necessary)
with self._lock:
if block_roi not in self._block_locks:
self._block_locks[block_roi] = RequestLock()
block_lock = self._block_locks[block_roi]
# Handle identical simultaneous requests
with block_lock:
try:
# Extra [:] here is in case we are decompressing from a chunkedarray
self.Output.stype.copy_data(result,
self._block_data[block_roi][:])
return
except KeyError: # Not yet stored: Request it now.
# We attach a special attribute to the array to allow the upstream operator
# to optionally tell us not to bother caching the data.
self.Input(roi.start, roi.stop).writeInto(result).block()
if self.Input.meta.dontcache:
# The upstream operator says not to bother caching the data.
# (For example, see OpCacheFixer.)
return
if self.CompressionEnabled.value and numpy.dtype(
result.dtype) in [
numpy.dtype(numpy.uint8),
numpy.dtype(numpy.uint32),
numpy.dtype(numpy.float32)
]:
compressed_block = vigra.ChunkedArrayCompressed(
result.shape, vigra.Compression.LZ4, result.dtype)
compressed_block[:] = result
block_storage_data = compressed_block
else:
block_storage_data = result.copy()
with self._lock:
# Store the data.
# First double-check that the block wasn't removed from the
# cache while we were requesting it.
# (Could have happened via propagateDirty() or eventually the arrayCacheMemoryMgr)
if block_roi in self._block_locks:
self._block_data[block_roi] = block_storage_data
self._last_access_times[block_roi] = time.time()
