def run(self):
with h5py.File(self._filepath, "r") as h5_file:
(internal_path, slicing) = self._request_queue_recv.recv()
# 'None' means stop the process.
while internal_path is not None:
try:
if METHOD == "shared-array":
read_roi = slice_to_roi(slicing, h5_file[internal_path].shape)
read_roi = numpy.array(read_roi)
read_shape = read_roi[1] - read_roi[0]
num_bytes = h5_file[internal_path].dtype.itemsize * bigintprod(read_shape)
assert num_bytes <= self.available_bytes, "I don't yet support really big slicings"
read_array = numpy.frombuffer(self.transfer_buffer, dtype=numpy.uint8, count=num_bytes)
read_array.setflags(write=True)
read_array = read_array.view(h5_file[internal_path].dtype).reshape(read_shape)
h5_file[internal_path].read_direct(read_array, slicing)
if METHOD == "pipe-bytes" or METHOD == "pipe-array":
read_array = h5_file[internal_path][slicing]
except Exception as ex:
self._result_queue_send.send(ex)
raise
else:
self._result_queue_send.send((read_array.shape, read_array.dtype))
if METHOD == "pipe-array":
self._result_queue_send.send(read_array)
if METHOD == "pipe-bytes":
self._result_queue_send.send_bytes(numpy.getbuffer(read_array))
# Wait for the next request
(internal_path, slicing) = self._request_queue_recv.recv()
def _handleCompletedRequest(self, roi, result):
try:
if self._allowParallelResults:
# Signal the user with the result before the critical section
self.resultSignal(roi, result)
except Exception:
# Always notify.
with self._condition:
self._failure_excinfo = sys.exc_info()
self._condition.notify()
raise
with self._condition:
try:
if not self._allowParallelResults:
# Signal here, inside the critical section.
self.resultSignal(roi, result)
# Report progress (if possible)
if self._totalVolume is not None:
self._processedVolume += bigintprod(numpy.subtract(roi[1], roi[0]))
progress = 100 * self._processedVolume // self._totalVolume
self.progressSignal(progress)
logger.debug("Request completed for roi: {}".format(roi))
self._completed_count += 1
finally:
# Always notify in this finally section,
# even if the client result/progress handler raised.
self._condition.notify()
def _calculate_probabilities(self, roi):
classifier = self.Classifier.value
assert isinstance(
classifier, LazyflowVectorwiseClassifierABC
), f"Classifier {classifier} must be sublcass of {LazyflowVectorwiseClassifierABC}"
key = roi.toSlice()
newKey = key[:-1]
newKey += (slice(0, self.Image.meta.shape[-1], None),)
with Timer() as features_timer:
input_data = self.Image[newKey].wait()
input_data = numpy.asarray(input_data, numpy.float32)
shape = input_data.shape
prod = bigintprod(shape[:-1])
features = input_data.reshape((prod, shape[-1]))
with Timer() as prediction_timer:
probabilities = classifier.predict_probabilities(features)
logger.debug(
f"Features took {features_timer.seconds()} seconds."
f" Prediction took {prediction_timer.seconds()} seconds. {roi}"
)
probabilities.shape = shape[:-1] + (probabilities.shape[-1],)
return probabilities
def setupOutputs(self):
# Read the dataset meta-info from the HDF5 dataset
self._h5N5File = self.H5N5File.value
internalPath = self.InternalPath.value
if internalPath not in self._h5N5File:
raise OpStreamingH5N5Reader.DatasetReadError(internalPath)
dataset = self._h5N5File[internalPath]
try:
# Read the axistags property without actually importing the data
# Throws KeyError if 'axistags' can't be found
axistagsJson = self._h5N5File[internalPath].attrs["axistags"]
axistags = vigra.AxisTags.fromJSON(axistagsJson)
axisorder = "".join(tag.key for tag in axistags)
if "?" in axisorder:
raise KeyError("?")
except KeyError:
# No axistags found.
if "axes" in dataset.attrs:
axisorder = "".join(dataset.attrs["axes"][::-1]).lower()
else:
axisorder = get_default_axisordering(dataset.shape)
axistags = vigra.defaultAxistags(str(axisorder))
assert len(axistags) == len(
dataset.shape
), f"Mismatch between shape {dataset.shape} and axisorder {axisorder}"
# Configure our slot meta-info
self.OutputImage.meta.dtype = dataset.dtype.type
self.OutputImage.meta.shape = dataset.shape
self.OutputImage.meta.axistags = axistags
# If the dataset specifies a datarange, add it to the slot metadata
if "drange" in self._h5N5File[internalPath].attrs:
self.OutputImage.meta.drange = tuple(
self._h5N5File[internalPath].attrs["drange"])
# Same for display_mode
if "display_mode" in self._h5N5File[internalPath].attrs:
self.OutputImage.meta.display_mode = str(
self._h5N5File[internalPath].attrs["display_mode"])
total_volume = bigintprod(self._h5N5File[internalPath].shape)
chunks = self._h5N5File[internalPath].chunks
if not chunks and total_volume > 1e8:
self.OutputImage.meta.inefficient_format = True
logger.warning(
f"This dataset ({self._h5N5File.filename}{internalPath}) is NOT chunked. "
f"Performance for 3D access patterns will be bad!")
if chunks:
self.OutputImage.meta.ideal_blockshape = chunks
def export_from_tiled_volume(tiles_description_json_path, roi,
output_hdf5_path, output_dataset_name):
"""
Export a cutout volume from a TiledVolume into an hdf5 dataset.
Args:
tiles_description_json_path: path to the TiledVolume's json description file.
roi: The (start, stop) corners of the cutout region to export. (Must be tuple-of-tuples.)
output_hdf5_path: The HDF5 file to export to.
output_dataset_name: The name of the HDF5 dataset to write. Will be deleted first if necessary.
"""
if not os.path.exists(tiles_description_json_path):
raise Exception("Description file does not exist: " +
tiles_description_json_path)
start, stop = numpy.array(roi)
shape = tuple(stop - start)
tiled_volume = TiledVolume(tiles_description_json_path)
with Timer() as timer:
result_array = numpy.ndarray(shape, tiled_volume.description.dtype)
logger.info("Reading cutout volume of shape: {}".format(shape))
tiled_volume.read((start, stop), result_out=result_array)
logger.info("Writing data to: {}/{}".format(output_hdf5_path,
output_dataset_name))
with h5py.File(output_hdf5_path, "a") as output_h5_file:
if output_dataset_name in output_h5_file:
del output_h5_file[output_dataset_name]
dset = output_h5_file.create_dataset(output_dataset_name,
shape,
result_array.dtype,
chunks=True,
data=result_array)
try:
import vigra
except ImportError:
pass
else:
# Attach axistags to the exported dataset, so ilastik
# automatically interprets the volume correctly.
output_axes = tiled_volume.description.output_axes
dset.attrs["axistags"] = vigra.defaultAxistags(
output_axes).toJSON()
logger.info("Exported {:.1e} pixels in {:.1f} seconds.".format(
bigintprod(shape), timer.seconds()))
def execute(self, slot, subindex, roi, result):
with self._lock:
if self.cache is None:
shape = self.Input.meta.shape
# self.blockshape has None in the last dimension to indicate that it should not be
# handled block-wise. None is replaced with the image shape in the respective axis.
fullBlockShape = []
for u, v in zip(self.blockShape.value, shape):
if u is not None:
fullBlockShape.append(u)
else:
fullBlockShape.append(v)
fullBlockShape = numpy.array(fullBlockShape,
dtype=numpy.float64)
# data = self.inputs["Input"][:].wait()
# split up requests into blocks
numBlocks = numpy.ceil(shape / fullBlockShape).astype("int")
blockCache = numpy.ndarray(shape=bigintprod(numBlocks),
dtype=self.Output.meta.dtype)
pool = RequestPool()
# blocks holds the different roi keys for each of the blocks
blocks = itertools.product(
*[list(range(i)) for i in numBlocks])
blockKeys = []
for b in blocks:
start = b * fullBlockShape
stop = b * fullBlockShape + fullBlockShape
stop = numpy.min(numpy.vstack((stop, shape)), axis=0)
blockKey = roiToSlice(start, stop)
blockKeys.append(blockKey)
fun = self.inputs["Function"].value
def predict_block(i):
data = self.Input[blockKeys[i]].wait()
blockCache[i] = fun(data)
for i, f in enumerate(blockCache):
req = pool.request(partial(predict_block, i))
pool.wait()
pool.clean()
self.cache = [fun(blockCache)]
return self.cache
def get_intersecting_blocks(blockshape, roi, shape):
"""Find block indices for given roi
Wraps around lazyflow.roi.getIntersectingBlocks
Idea is that only the required blocks are allocated using
Everything in 'czyx' - order
Args:
blockshape (iterable): block shape
roi (tuple): (start, stop), inclusive start of block, exclusive end
Difference is that it returns a dictionary consisting of
* 'array_of_blocks' the array of blocks
* 'subimage_shape' shape of the subimage
* 'block_offsets' array of offset for each of the blocks
* 'subimage_roi' the roi in the sub_image
"""
blocks = lazyflow.roi.getIntersectingBlocks(blockshape,
roi,
asarray=True)
num_indexes = bigintprod(blocks.shape[0:-1])
axiscount = blocks.shape[-1]
blocks_array = numpy.reshape(blocks, (num_indexes, axiscount))
block_aligned_subimage_start = blocks_array.min(axis=0)
block_aligned_subimage_end = blocks_array.max(axis=0)
assert (block_aligned_subimage_start == blocks_array).all(
axis=1).any(), "roi does not seem to be block aligned"
assert (block_aligned_subimage_end == blocks_array).all(
axis=1).any(), "roi does not seem to be block aligned"
# get the real end of the image:
block_aligned_subimage_end += blockshape
# take care of image border
block_aligned_subimage_end = numpy.min(
[block_aligned_subimage_end, shape], axis=0)
subimage_shape = block_aligned_subimage_end - block_aligned_subimage_start
block_offsets = blocks_array - block_aligned_subimage_start
subimage_start = roi[0] - block_aligned_subimage_start
subimage_roi = ((subimage_start), (subimage_start + (roi[1] - roi[0])))
return blocks_array, block_offsets, subimage_roi, subimage_shape
def _ensureCached(self, entire_block_roi):
"""
Ensure that the cache file for the given block is up-to-date.
(Refresh it if it's dirty.)
"""
block_start = tuple(entire_block_roi[0])
block_file = self._getCacheFile(entire_block_roi)
if block_start in self._dirtyBlocks:
updated_cache = False
with self._blockLocks[block_start]:
# Check AGAIN now that we have the lock.
# (Avoid doing this twice in parallel requests.)
if block_start in self._dirtyBlocks:
# Can't write directly into the hdf5 dataset because
# h5py.dataset.__getitem__ creates a copy, not a view.
# We must use a temporary numpy array to hold the data.
data = self.Input(*entire_block_roi).wait()
block_file["data"][...] = data
if self.Output.meta.has_mask:
block_file["mask"][...] = data.mask
block_file["fill_value"][...] = data.fill_value
if logger.isEnabledFor(logging.DEBUG):
uncompressed_size = bigintprod(
data.shape) * self._getDtypeBytes(data.dtype)
storage_size = block_file["data"].id.get_storage_size()
if "mask" in block_file:
storage_size += block_file[
"mask"].id.get_storage_size()
if "fill_value" in block_file:
storage_size += block_file[
"fill_value"].id.get_storage_size()
logger.debug(
"Storage for block: {} is {}. ({}% of original)".
format(block_start, storage_size,
100 * storage_size / uncompressed_size))
with self._lock:
self._dirtyBlocks.remove(block_start)
updated_cache = True
if updated_cache:
# Now that the lock is released, signal that the cache was updated.
self.Output._sig_value_changed()
self.OutputHdf5._sig_value_changed()
self.CleanBlocks._sig_value_changed()
def execute(self, slot, subindex, roi, result):
if bigintprod(roi.stop - roi.start) > 1e9:
logger.error(
"Requesting a very large volume from DVID: {}\nIs that really what you meant to do?"
.format(roi))
# TODO: Modify accessor implementation to accept a pre-allocated array.
# FIXME: Disabled throttling for now. Need a better heuristic or explicit setting.
# # For "heavy" requests, we'll use the throttled accessor
# HEAVY_REQ_SIZE = 256*256*10
# if bigintprod(result.shape) > HEAVY_REQ_SIZE:
# accessor = self._throttled_accessor
# else:
# accessor = self._default_accessor
accessor = self._default_accessor # FIXME (see above)
result[:] = accessor.get_ndarray(roi.start, roi.stop)
return result
def read_subvolume(self, internal_path, slicing):
self._request_queue_send.send((internal_path, slicing))
response_info = self._result_queue_recv.recv()
if isinstance(response_info, Exception):
raise response_info
shape, dtype = response_info
if METHOD == "pipe-array":
result = self._result_queue_recv.recv()
if METHOD == "pipe-bytes":
raw_buffer = self._result_queue_recv.recv_bytes()
result = numpy.frombuffer(raw_buffer, dtype=dtype).reshape(shape)
result.setflags(write=True)
if METHOD == "shared-array":
result = numpy.frombuffer(self.transfer_buffer, dtype=dtype, count=bigintprod(shape)).copy()
result = result.reshape(shape)
return result
def predict_probabilities_pixelwise(self, X, roi, axistags=None):
"""
For each pixel in the given feature_image, predict the probabilities that the
pixel belongs to each label class the classifier was trained with.
X: An ND image. Last axis must be channel.
roi: The region of interest (start, stop) within feature_image to predict (e.g. without the halo region)
Note: roi parameter should not include channel.
For example, a valid roi for a zyxc image could be ((0,0,0), (10,20,30))
axistags: Optional. A vigra.AxisTags object describing the feature_image.
Returns: A multi-channel image (each channel corresponds to a different label class).
The result image size is determined by the roi parameter.
"""
logger.debug("predicting PIXELWISE vigra RF")
# This classifier doesn't benefit from any context around the input,
# so just strip it off and only use the given roi.
assert len(roi[0]) == len(roi[1]) == X.ndim - 1
X = X[roi_to_slice(*roi)]
# reshape the image into a 2D feature matrix
matrix_shape = (bigintprod(X.shape[:-1]), X.shape[-1])
feature_matrix = numpy.reshape(X, matrix_shape)
# Run classifier
probabilities = self._vigra_rf.predictProbabilities(
feature_matrix.view(numpy.ndarray))
# Reshape into an image.
# Choose the prediction image shape carefully:
#
# Most classifiers omit a channel entirely if there are no labels given for a particular class,
# So the number of prediction channels we got is the same as the number of known_classes
# But if the classifier attempts to "help us out" by including channels for "missing" labels,
# then we want to just return the whole thing.
num_probability_channels = max(len(self.known_classes),
probabilities.shape[-1])
prediction_shape = X.shape[:-1] + (num_probability_channels, )
return numpy.reshape(probabilities, prediction_shape)
def execute(self, slot, subindex, roi, result):
input_data = self.Input(roi.start, roi.stop).wait()
assert slot == self.Output
n_segments = self.NumSegments.value
if n_segments == 0:
# If the number of supervoxels was not given, use a default proportional to the number of voxels
n_segments = numpy.int(bigintprod(input_data.shape) / 2500)
logger.debug(
"calling skimage.segmentation.slic with {}".format(
dict(
n_segments=n_segments,
compactness=self.Compactness.value,
max_iter=self.MaxIter.value,
multichannel=True,
enforce_connectivity=True,
convert2lab=False,
)
)
)
slic_sp = skimage.segmentation.slic(
input_data,
n_segments=n_segments,
compactness=self.Compactness.value,
max_iter=self.MaxIter.value,
multichannel=True,
enforce_connectivity=True,
convert2lab=False,
) # Use with caution.
# This would cause slic() to have special behavior for 3-channel data,
# in which case we better really be dealing with RGB channels
# (not, say 3 unrelated image features).
# slic_sp has no channel axis, so insert that axis before copying to 'result'
result[:] = slic_sp[..., None]
# import IPython; IPython.embed()
return result
def remove_wrongly_sized_connected_components(a, min_size, max_size=None, in_place=False, bin_out=False):
original_dtype = a.dtype
if not in_place:
a = a.copy()
if min_size == 0 and (max_size is None or max_size > bigintprod(a.shape)): # shortcut for efficiency
if bin_out:
numpy.place(a, a, 1)
return a
component_sizes = vigra_bincount(a)
bad_sizes = component_sizes < min_size
if max_size is not None:
numpy.logical_or(bad_sizes, component_sizes > max_size, out=bad_sizes)
del component_sizes
bad_locations = bad_sizes[a]
a[bad_locations] = 0
del bad_locations
if bin_out:
# Replace non-zero values with 1
numpy.place(a, a, 1)
return numpy.asarray(a, dtype=original_dtype)
def __init__(self, slot: Slot, batchsize: int, iterate_axes: str):
"""
Args:
slot: slot to request data from
batchsize: maximum number of requests to launch in parallel
"""
self._slot = slot
self._batchsize = batchsize
self._q: Queue[Tuple[ROI_TUPLE, numpy.ndarray]] = Queue()
self._items: Dict[int, numpy.ndarray] = {}
self._index = 0
self._roi_iter = _RoiIter(self._slot, iterate_axes=iterate_axes)
self._max = len(self._roi_iter)
self._roi_request_batch = RoiRequestBatch(
outputSlot=self._slot,
roiIterator=iter(self._roi_iter),
totalVolume=bigintprod(self._slot.meta.shape),
batchSize=self._batchsize,
allowParallelResults=False,
)
self._roi_request_batch.resultSignal.subscribe(self._put)
def normalized_surface_area(shape):
pairs = numpy.array(list(combinations(shape, 2)))
surface_area = 2 * (pairs[:, 0] * pairs[:, 1]).sum()
volume = bigintprod(shape)
return surface_area / volume
def execute(self, slot, subindex, roi, result):
classifier = self.Classifier.value
# Training operator may return 'None' if there was no data to train with
skip_prediction = classifier is None
# Shortcut: If the mask is totally zero, skip this request entirely
if not skip_prediction and self.PredictionMask.ready():
mask_roi = numpy.array((roi.start, roi.stop))
mask_roi[:, -1:] = [[0], [1]]
start, stop = list(map(tuple, mask_roi))
mask = self.PredictionMask(start, stop).wait()
skip_prediction = not numpy.any(mask)
del mask
if skip_prediction:
result[:] = 0.0
return result
assert issubclass(
type(classifier), LazyflowVectorwiseClassifierABC
), "Classifier is of type {}, which does not satisfy the LazyflowVectorwiseClassifierABC interface.".format(
type(classifier)
)
key = roi.toSlice()
newKey = key[:-1]
newKey += (slice(0, self.Image.meta.shape[-1], None),)
with Timer() as features_timer:
input_data = self.Image[newKey].wait()
input_data = numpy.asarray(input_data, numpy.float32)
shape = input_data.shape
prod = bigintprod(shape[:-1])
features = input_data.reshape((prod, shape[-1]))
features = self.SupervoxelFeatures.value
# print("features before prediction {}".format(features))
# features = get_supervoxel_features(features, self.SupervoxelSegmentation.value)
# import ipdb; ipdb.set_trace()
with Timer() as prediction_timer:
probabilities = classifier.predict_probabilities(features)
# import ipdb; ipdb.set_trace()
probabilities = slic_to_mask(self.SupervoxelSegmentation.value, probabilities).reshape(
-1, probabilities.shape[-1]
)
logger.debug(
"Features took {} seconds, Prediction took {} seconds for roi: {} : {}".format(
features_timer.seconds(), prediction_timer.seconds(), roi.start, roi.stop
)
)
assert probabilities.shape[1] <= self.PMaps.meta.shape[-1], (
"Error: Somehow the classifier has more label classes than expected:"
" Got {} classes, expected {} classes".format(probabilities.shape[1], self.PMaps.meta.shape[-1])
)
# We're expecting a channel for each label class.
# If we didn't provide at least one sample for each label,
# we may get back fewer channels.
if probabilities.shape[1] < self.PMaps.meta.shape[-1]:
# Copy to an array of the correct shape
# This is slow, but it's an unusual case
assert probabilities.shape[-1] == len(classifier.known_classes)
full_probabilities = numpy.zeros(
probabilities.shape[:-1] + (self.PMaps.meta.shape[-1],), dtype=numpy.float32
)
for i, label in enumerate(classifier.known_classes):
full_probabilities[:, label - 1] = probabilities[:, i]
probabilities = full_probabilities
# Reshape to image
probabilities.shape = shape[:-1] + (self.PMaps.meta.shape[-1],)
# Copy only the prediction channels the client requested.
result[...] = probabilities[..., roi.start[-1] : roi.stop[-1]]
return result
def _chooseChunkshape(self, blockshape):
"""
Choose an optimal chunkshape for our blockshape and Input shape.
We assume access patterns to vary more in space than in time or channel
and choose the inner chunk shape to be about 1MiB slices of t and c.
Furthermore, we use the function
lazyflow.utility.chunkHelpers.chooseChunkShape()
to preserve the aspect ratio of the input (at least approximately).
"""
if blockshape is None:
return None
def isConsistent(idealshape):
"""
check if ideal block shape and given block shape are consistent
shapes are consistent if, for each dimension,
* input is unready, or
* blockshape equals fullshape, or
* idealshape divides blockshape evenly
"""
if not self.Input.ready():
return True
fullshape = self.Input.meta.shape
z = list(zip(idealshape, blockshape, fullshape))
m = [
i_b_f[1] == i_b_f[2] or i_b_f[1] % i_b_f[0] == 0 for i_b_f in z
]
return all(m)
if not self._ignore_ideal_blockshape and self.Input.ready():
# take the ideal chunk shape, but check if sane
ideal = self.Input.meta.ideal_blockshape
if ideal is not None:
if len(ideal) == len(blockshape):
ideal = numpy.asarray(ideal, dtype=numpy.int)
for i, d in enumerate(ideal):
if d == 0:
ideal[i] = blockshape[i]
if not isConsistent(ideal):
logger.warning(
"{}: BlockShape and ideal_blockshape are "
"inconsistent {} vs {}".format(
self.name, blockshape, ideal))
else:
return tuple(ideal)
else:
logger.warning(
"{}: Encountered meta.ideal_blockshape that does not fit the data"
.format(self.name))
# we need to figure out an ideal chunk shape on our own
# Start with a copy of blockshape
axes = list(self.Output.meta.getTaggedShape().keys())
taggedBlockShape = collections.OrderedDict(
list(zip(axes, self._blockshape)))
dtypeBytes = self._getDtypeBytes(self.Output.meta.dtype)
desiredSpace = 1024**2 / float(dtypeBytes)
if bigintprod(blockshape) <= desiredSpace:
return blockshape
# set t and c to 1
for key in "tc":
if key in taggedBlockShape:
taggedBlockShape[key] = 1
logger.debug("desired space: {}".format(desiredSpace))
# extract only the spatial shape
spatialKeys = [k for k in list(taggedBlockShape.keys()) if k in "xyz"]
spatialShape = [taggedBlockShape[k] for k in spatialKeys]
newSpatialShape = chooseChunkShape(spatialShape, desiredSpace)
for k, v in zip(spatialKeys, newSpatialShape):
taggedBlockShape[k] = v
chunkShape = tuple(taggedBlockShape.values())
logger.debug("Using chunk shape: {}".format(chunkShape))
return chunkShape
def __len__(self) -> int:
return bigintprod(self._iterate_shape)
def _determine_blockshape(self, outputSlot):
"""
Choose a blockshape using the slot metadata (if available) or an arbitrary guess otherwise.
"""
input_shape = outputSlot.meta.shape
ideal_blockshape = outputSlot.meta.ideal_blockshape
ram_usage_per_requested_pixel = outputSlot.meta.ram_usage_per_requested_pixel
max_blockshape = outputSlot.meta.max_blockshape or input_shape
num_channels = 1
tagged_shape = outputSlot.meta.getTaggedShape()
available_ram = Memory.getAvailableRamComputation()
# Generally, we don't want to split requests across channels.
if "c" in list(tagged_shape.keys()):
num_channels = tagged_shape["c"]
channel_index = list(tagged_shape.keys()).index("c")
input_shape = input_shape[:channel_index] + input_shape[
channel_index + 1:]
max_blockshape = max_blockshape[:channel_index] + max_blockshape[
channel_index + 1:]
if ideal_blockshape:
# Never enlarge 'ideal' in the channel dimension.
num_channels = ideal_blockshape[channel_index]
ideal_blockshape = ideal_blockshape[:
channel_index] + ideal_blockshape[
channel_index + 1:]
del tagged_shape["c"]
# Generally, we don't want to join time slices
if "t" in tagged_shape.keys():
blockshape_time_steps = 1
time_index = list(tagged_shape.keys()).index("t")
input_shape = input_shape[:time_index] + input_shape[time_index +
1:]
max_blockshape = max_blockshape[:time_index] + max_blockshape[
time_index + 1:]
if ideal_blockshape:
# Never enlarge 'ideal' in the time dimension.
blockshape_time_steps = ideal_blockshape[time_index]
ideal_blockshape = ideal_blockshape[:
time_index] + ideal_blockshape[
time_index + 1:]
available_ram /= blockshape_time_steps
del tagged_shape["t"]
if ram_usage_per_requested_pixel is None:
# Make a conservative guess: 2*(bytes for dtype) * (num channels) + (fudge factor=4)
ram_usage_per_requested_pixel = 2 * outputSlot.meta.dtype(
).nbytes * num_channels + 4
warnings.warn(
"Unknown per-pixel RAM requirement. Making a guess.")
# Safety factor (fudge factor): Double the estimated RAM usage per pixel
safety_factor = 2.0
logger.info(
"Estimated RAM usage per pixel is {} * safety factor ({})".format(
Memory.format(ram_usage_per_requested_pixel), safety_factor))
ram_usage_per_requested_pixel *= safety_factor
if ideal_blockshape is None:
blockshape = determineBlockShape(
input_shape,
(available_ram //
(self._num_threads * ram_usage_per_requested_pixel)))
blockshape = tuple(numpy.minimum(max_blockshape, blockshape))
warnings.warn("Chose an arbitrary request blockshape")
else:
logger.info("determining blockshape assuming available_ram is {}"
", split between {} threads".format(
Memory.format(available_ram), self._num_threads))
# By convention, ram_usage_per_requested_pixel refers to the ram used when requesting ALL channels of a 'pixel'
# Therefore, we do not include the channel dimension in the blockshapes here.
#
# Also, it rarely makes sense to request more than one time slice, so we omit that, too. (See above.)
blockshape = determine_optimal_request_blockshape(
max_blockshape, ideal_blockshape,
ram_usage_per_requested_pixel, self._num_threads,
available_ram)
# compute the RAM size of the block before adding back t anc c dimensions
fmt = Memory.format(ram_usage_per_requested_pixel *
bigintprod(blockshape))
# If we removed time and channel from consideration, add them back now before returning
if "t" in outputSlot.meta.getAxisKeys():
blockshape = blockshape[:time_index] + (
blockshape_time_steps, ) + blockshape[time_index:]
if "c" in outputSlot.meta.getAxisKeys():
blockshape = blockshape[:channel_index] + (
num_channels, ) + blockshape[channel_index:]
logger.info("Chose blockshape: {}".format(blockshape))
logger.info("Estimated RAM usage per block is {}".format(fmt))
return blockshape
def __init__(self,
outputSlot,
roi,
blockshape=None,
batchSize=None,
blockAlignment="absolute",
allowParallelResults=False):
"""
Constructor.
:param outputSlot: The slot to request data from.
:param roi: The roi `(start, stop)` of interest. Will be broken up and requested via smaller requests.
:param blockshape: The amount of data to request in each request. If omitted, a default blockshape is chosen by inspecting the metadata of the given slot.
:param batchSize: The maximum number of requests to launch in parallel. This should not be necessary if the blockshape is small enough that you won't run out of RAM.
:param blockAlignment: Determines how block the requests. Choices are 'absolute' or 'relative'.
:param allowParallelResults: If False, The resultSignal will not be called in parallel.
In that case, your handler function has no need for locks.
"""
self._outputSlot = outputSlot
self._bigRoi = roi
self._num_threads = max(1, Request.global_thread_pool.num_workers)
totalVolume = bigintprod(numpy.subtract(roi[1], roi[0]))
if batchSize is None:
batchSize = self._num_threads
if blockshape is None:
blockshape = self._determine_blockshape(outputSlot)
assert blockAlignment in ["relative", "absolute"]
if blockAlignment == "relative":
# Align the blocking with the start of the roi
offsetRoi = ([0] * len(roi[0]), numpy.subtract(roi[1], roi[0]))
block_starts = getIntersectingBlocks(blockshape, offsetRoi)
block_starts += roi[0] # Un-offset
# For now, simply iterate over the min blocks
# TODO: Auto-dialate block sizes based on CPU/RAM usage.
def roiGen():
block_iter = block_starts.__iter__()
while True:
try:
block_start = next(block_iter)
except StopIteration:
# As of Python 3.7, not allowed to let StopIteration exceptions escape a generator
# https://www.python.org/dev/peps/pep-0479
break
else:
# Use offset blocking
offset_block_start = block_start - self._bigRoi[0]
offset_data_shape = numpy.subtract(
self._bigRoi[1], self._bigRoi[0])
offset_block_bounds = getBlockBounds(
offset_data_shape, blockshape, offset_block_start)
# Un-offset
block_bounds = (
offset_block_bounds[0] + self._bigRoi[0],
offset_block_bounds[1] + self._bigRoi[0],
)
logger.debug("Requesting Roi: {}".format(block_bounds))
yield block_bounds
else:
# Absolute blocking.
# Blocks are simply relative to (0,0,0,...)
# But we still clip the requests to the overall roi bounds.
block_starts = getIntersectingBlocks(blockshape, roi)
def roiGen():
block_iter = block_starts.__iter__()
while True:
try:
block_start = next(block_iter)
except StopIteration:
# As of Python 3.7, not allowed to let StopIteration exceptions escape a generator
# https://www.python.org/dev/peps/pep-0479
break
else:
block_bounds = getBlockBounds(outputSlot.meta.shape,
blockshape, block_start)
block_intersecting_portion = getIntersection(
block_bounds, roi)
logger.debug("Requesting Roi: {}".format(block_bounds))
yield block_intersecting_portion
self._requestBatch = RoiRequestBatch(self._outputSlot, roiGen(),
totalVolume, batchSize,
allowParallelResults)
def determine_optimal_request_blockshape(max_blockshape, ideal_blockshape,
ram_usage_per_requested_pixel,
num_threads, available_ram):
"""
Choose a blockshape for requests subject to the following constraints:
- not larger than max_blockshape in any dimension
- not too large to run in parallel without exceeding available ram (according to num_threads and available_ram)
Within those constraints, choose the largest blockshape possible.
The blockshape will be chosen according to the following heuristics:
- If any dimensions in ideal_blockshape are 0, prefer to expand those first until max_blockshape is reached.
(The result is known as atomic_blockshape.)
- After that, attempt to expand the blockshape by incrementing a dimension according to its width in atomic_blockshape.
Note: For most use-cases, the ``ram_usage_per_requested_pixel`` parameter refers to the ram consumed when requesting ALL channels of an image.
Therefore, you probably want to omit the channel dimension from your max_blockshape and ideal_blockshape parameters.
>>> determine_optimal_request_blockshape( (1000,1000,100), (0,0,1), 4, 10, 1e6 )
(158, 158, 1)
>>> determine_optimal_request_blockshape( (1000,1000,100), (0,0,1), 4, 10, 1e9 )
(1000, 1000, 24)
"""
assert len(max_blockshape) == len(ideal_blockshape)
# Convert to numpy for convenience.
max_blockshape = numpy.asarray(max_blockshape)
ideal_blockshape = numpy.asarray(ideal_blockshape)
target_block_volume_bytes = available_ram // num_threads
target_block_volume_pixels = max(
4096, target_block_volume_bytes // ram_usage_per_requested_pixel)
# Replace 0's in the ideal_blockshape with the corresponding piece of max_blockshape
complete_ideal_blockshape = numpy.where(ideal_blockshape == 0,
max_blockshape, ideal_blockshape)
# Clip to max
clipped_ideal_blockshape = numpy.minimum(max_blockshape,
complete_ideal_blockshape)
atomic_blockshape = determineBlockShape(clipped_ideal_blockshape,
target_block_volume_pixels)
atomic_blockshape = numpy.asarray(atomic_blockshape)
if bigintprod(clipped_ideal_blockshape) >= target_block_volume_pixels:
# Target volume is too small for us to stack the atomic blockshape, anyway
return tuple(atomic_blockshape)
# Need to stack the ideal_blockshape to come up with something larger.
# Start with an isotropic block, clipped to the nearest multiple of the atomic_blockshape
blockshape = numpy.array(
determineBlockShape(clipped_ideal_blockshape,
target_block_volume_pixels))
blockshape -= blockshape % atomic_blockshape
while True:
# Find a dimension of atomic_blockshape that isn't already maxed out,
# And see if we have enough RAM to
candidate_blockshapes = []
for index in range(len(blockshape)):
# If we were to expand the blockshape in this dimension, would the block still fit in RAM?
candidate_blockshape = blockshape.copy()
candidate_blockshape[index] += clipped_ideal_blockshape[index]
if (candidate_blockshape <=
max_blockshape).all() and (bigintprod(candidate_blockshape)
< target_block_volume_pixels):
candidate_blockshapes.append(candidate_blockshape)
if len(candidate_blockshapes) == 0:
break
def normalized_surface_area(shape):
pairs = numpy.array(list(combinations(shape, 2)))
surface_area = 2 * (pairs[:, 0] * pairs[:, 1]).sum()
volume = bigintprod(shape)
return surface_area / volume
# Choose the best among the canidates
scores = list(map(normalized_surface_area, candidate_blockshapes))
(best_shape, best_score) = min(zip(candidate_blockshapes, scores),
key=lambda shape_score: shape_score[1])
blockshape = best_shape
return tuple(blockshape)
def execute(self, slot, subindex, roi, result):
dtypeBytes = self._getDtypeBytes()
totalBytes = dtypeBytes * bigintprod(self.Input.meta.shape)
totalMB = old_div(totalBytes, (1000 * 1000))
logger.info(
"Clusterizing computation of {} MB dataset, outputting according to {}".format(
totalMB, self.OutputDatasetDescription.value
)
)
configFilePath = self.ConfigFilePath.value
self._config = parseClusterConfigFile(configFilePath)
# Create the destination file if necessary
blockwiseFileset, taskInfos = self._prepareDestination()
try:
# Figure out which work doesn't need to be recomputed (if any)
unneeded_rois = []
for roi in list(taskInfos.keys()):
if blockwiseFileset.getBlockStatus(
roi[0]
) == BlockwiseFileset.BLOCK_AVAILABLE or blockwiseFileset.isBlockLocked(
roi[0]
): # We don't attempt to process currently locked blocks.
unneeded_rois.append(roi)
# Remove any tasks that we don't need to compute (they were finished in a previous run)
for roi in unneeded_rois:
logger.info("No need to run task: {} for roi: {}".format(taskInfos[roi].taskName, roi))
del taskInfos[roi]
absWorkDir, _ = getPathVariants(self._config.server_working_directory, os.path.split(configFilePath)[0])
if self._config.task_launch_server == "localhost":
def localCommand(cmd):
cwd = os.getcwd()
os.chdir(absWorkDir)
subprocess.call(cmd, shell=True)
os.chdir(cwd)
launchFunc = localCommand
else:
# We use fabric for executing remote tasks
# Import it here because it isn't required that the nodes can use it.
import fabric.api as fab
@fab.hosts(self._config.task_launch_server)
def remoteCommand(cmd):
with fab.cd(absWorkDir):
fab.run(cmd)
launchFunc = functools.partial(fab.execute, remoteCommand)
# Spawn each task
for taskInfo in list(taskInfos.values()):
logger.info("Launching node task: " + taskInfo.command)
launchFunc(taskInfo.command)
# Return immediately. We do not attempt to monitor the task progress.
result[0] = True
return result
finally:
blockwiseFileset.close()
def test_bigintprod(nums, result):
assert bigintprod(nums) == result
def predict_probabilities_pixelwise(self, X, roi, axistags=None):
logger.debug("predicting PIXELWISE vigra RF")
# This classifier doesn't benefit from any context around the input, (does it?)
# so just strip it off and only use the given roi.
assert len(roi[0]) == len(roi[1]) == X.ndim - 1
X = X[roi_to_slice(*roi)]
FRAME_SPAN = 10 # Number of frames to wait until the mask is recalculated
DILATION_RADIUS = 50 # In pixels
BACKGROUND_LABEL = 1
# Allocate memory for probability volume and mask
prob_vol = numpy.zeros((X.shape[:-1] + (len(self._known_labels),)), dtype=numpy.float32)
mask = numpy.ones(bigintprod(X.shape[1:-1]), dtype=numpy.bool)
frm_cnt = 0
for X_t in X:
if frm_cnt % FRAME_SPAN == 0:
mask = numpy.ones(bigintprod(X.shape[1:-1]), dtype=numpy.bool)
prob_mat = numpy.zeros((bigintprod(X.shape[1:-1]), len(self._known_labels)), dtype=numpy.float32)
# Reshape the image into a 2D feature matrix
mat_shape = (bigintprod(X_t.shape[:-1]), X_t.shape[-1])
feature_mat = numpy.reshape(X_t, mat_shape)
# Mask the feature matrix
feature_mat_masked = feature_mat[mask == 1, :]
# Run classifier
prob_mat_masked = self._vigra_rf.predictProbabilities(feature_mat_masked.view(numpy.ndarray))
prob_mat[mask == 1, :] = prob_mat_masked
prob_mat[mask == 0, 0] = 1.0 # Fill background
prob_img = prob_mat.reshape((1,) + X_t.shape[:-1] + (prob_mat.shape[-1],))
# Recalculate the mask every 20 frames
if frm_cnt % FRAME_SPAN == 0:
predicted_labels = numpy.argmax(prob_img[0], axis=-1) + 1
prob_slice = (predicted_labels != BACKGROUND_LABEL).astype(numpy.bool)
kernel = numpy.ones((DILATION_RADIUS * 2 + 1), dtype=bool)
with Timer() as morpho_timer:
prob_slice_dilated = scipy.ndimage.morphology.binary_dilation(prob_slice, kernel[None, :])
prob_slice_dilated = scipy.ndimage.morphology.binary_dilation(prob_slice_dilated, kernel[:, None])
logger.debug("[PROF] Morphology took {} ".format(morpho_timer.seconds()))
mask = prob_slice_dilated.reshape(bigintprod(prob_slice_dilated.shape))
# vigra.impex.writeHDF5(prob_slice_dilated, 'mask.h5', 'data')
prob_vol[frm_cnt, :, :, :] = prob_img
frm_cnt = frm_cnt + 1
# Reshape into an image.
# Choose the prediction image shape carefully:
#
# Most classifiers omit a channel entirely if there are no labels given for a particular class,
# So the number of prediction channels we got is the same as the number of known_classes
# But if the classifier attempts to "help us out" by including channels for "missing" labels,
# then we want to just return the whole thing.
num_probability_channels = max(len(self.known_classes), prob_vol.shape[-1])
prediction_shape = X.shape[:-1] + (num_probability_channels,)
return numpy.reshape(prob_vol, prediction_shape)
def getIntersectingBlocks(blockshape, roi, asarray=False):
"""
Returns the start coordinate of each block that the given roi intersects.
By default, returned as an array of shape (N,M) (N indexes with M coordinates each).
If asarray=True, then the blocks are returned as an array of shape (D1,D2,D3,...DN,M)
such that coordinates of spatially adjacent blocks are returned in adjacent entries of the array.
(SEE ALSO: ``lazyflow.utility.blockwise_view``)
For example:
>>> block_starts = getIntersectingBlocks( (10, 20), [(15, 25),(23, 40)] )
>>> block_starts.shape
(2, 2)
>>> print(block_starts)
[[10 20]
[20 20]]
>>> block_starts = getIntersectingBlocks( (10, 20), [(15, 25),(23, 41)] )
>>> block_starts.shape
(4, 2)
>>> print(block_starts)
[[10 20]
[10 40]
[20 20]
[20 40]]
Now the same two examples, with asarray=True. Note the shape of the result.
>>> block_start_matrix = getIntersectingBlocks( (10, 20), [(15, 25),(23, 40)], asarray=True )
>>> block_start_matrix.shape
(2, 1, 2)
>>> print(block_start_matrix)
[[[10 20]]
<BLANKLINE>
[[20 20]]]
>>> block_start_matrix = getIntersectingBlocks( (10, 20), [(15, 25),(23, 41)], asarray=True )
>>> block_start_matrix.shape
(2, 2, 2)
>>> print(block_start_matrix)
[[[10 20]
[10 40]]
<BLANKLINE>
[[20 20]
[20 40]]]
This function works for negative rois, too.
>>> block_starts = getIntersectingBlocks( (10, 20), [(-10, -5),(5, 5)] )
>>> print(block_starts)
[[-10 -20]
[-10 0]
[ 0 -20]
[ 0 0]]
"""
assert len(blockshape) == len(roi[0]) == len(
roi[1]), "blockshape and roi are mismatched: {} vs {}".format(
blockshape, roi)
assert not numpy.any(numpy.isclose(
blockshape,
0)), f"blockshape ({blockshape}) should not contain zero elements"
roistart = TinyVector(roi[0])
roistop = TinyVector(roi[1])
blockshape = TinyVector(blockshape)
block_index_map_start = roistart // blockshape
block_index_map_stop = (
roistop + (blockshape - 1)
) // blockshape # Add (blockshape-1) first as a faster alternative to ceil()
block_index_map_shape = block_index_map_stop - block_index_map_start
num_axes = len(blockshape)
block_indices = numpy.indices(block_index_map_shape)
block_indices = numpy.rollaxis(block_indices, 0, num_axes + 1)
block_indices += block_index_map_start
# Multiply by blockshape to get the list of start coordinates
block_indices *= blockshape
if asarray:
return block_indices
else:
# Reshape into N*M matrix for easy iteration
num_indexes = bigintprod(block_indices.shape[0:-1])
axiscount = block_indices.shape[-1]
return numpy.reshape(block_indices, (num_indexes, axiscount))