def testInvalidMax(self):
try:
max_shape = (1,2,3,2,0)
determineBlockShape( max_shape, 1000 )
except AssertionError:
pass
except Exception as e:
assert False, "Wrong type of exception. Expected AssertionError, but got {}".format( e )
else:
assert False, "Expected assertion in determineBlockShape() due to invalid inputs"
def setupOutputs(self):
# We assume that channel the last axis
assert self.FeatureImage.meta.getAxisKeys()[-1] == 'c'
assert self.LabelImage.meta.getAxisKeys()[-1] == 'c'
assert self.LabelImage.meta.shape[-1] == 1
# For now, we assume that the two input images have the same shape (except channel)
# This constraint could be relaxed in the future if necessary
assert self.FeatureImage.meta.shape[:-1] == self.LabelImage.meta.shape[:-1],\
"FeatureImage and LabelImage shapes do not match: {} vs {}"\
"".format( self.FeatureImage.meta.shape, self.LabelImage.meta.shape )
self.LabelAndFeatureMatrix.meta.shape = (1,)
self.LabelAndFeatureMatrix.meta.dtype = object
num_feature_channels = self.FeatureImage.meta.shape[-1]
if num_feature_channels != self.LabelAndFeatureMatrix.meta.num_feature_channels:
self.LabelAndFeatureMatrix.meta.num_feature_channels = num_feature_channels
self.LabelAndFeatureMatrix.setDirty()
self.ProgressSignal.meta.shape = (1,)
self.ProgressSignal.meta.dtype = object
self.ProgressSignal.setValue( self.progressSignal )
# Auto-choose a blockshape
tagged_shape = self.LabelImage.meta.getTaggedShape()
if 't' in tagged_shape:
# A block should never span multiple time slices.
# For txy volumes, that could lead to lots of extra features being computed.
tagged_shape['t'] = 1
blockshape = determineBlockShape( tagged_shape.values(), OpFeatureMatrixCache.MAX_BLOCK_PIXELS )
# Don't span more than 256 px along any axis
blockshape = tuple(min(x, 256) for x in blockshape)
self._init_blocks(self.LabelImage.meta.shape, blockshape)
def _determine_blockshape(self, outputSlot):
"""
Choose a blockshape using the slot metadata (if available) or an arbitrary guess otherwise.
"""
input_shape = outputSlot.meta.shape
ideal_blockshape = outputSlot.meta.ideal_blockshape
ram_usage_per_requested_pixel = outputSlot.meta.ram_usage_per_requested_pixel
num_channels = 1
tagged_shape = outputSlot.meta.getTaggedShape()
# Generally, we don't want to split requests across channels.
if 'c' in tagged_shape.keys():
num_channels = tagged_shape['c']
channel_index = tagged_shape.keys().index('c')
input_shape = input_shape[:channel_index] + input_shape[channel_index+1:]
if ideal_blockshape:
# Never enlarge 'ideal' in the channel dimension.
num_channels = ideal_blockshape[channel_index]
ideal_blockshape = ideal_blockshape[:channel_index] + ideal_blockshape[channel_index+1:]
max_blockshape = input_shape
available_ram = Memory.getAvailableRamComputation()
if ram_usage_per_requested_pixel is None:
# Make a conservative guess: 2*(bytes for dtype) * (num channels) + (fudge factor=4)
ram_usage_per_requested_pixel = 2*outputSlot.meta.dtype().nbytes*num_channels + 4
warnings.warn( "Unknown per-pixel RAM requirement. Making a guess." )
# Safety factor (fudge factor): Double the estimated RAM usage per pixel
safety_factor = 2.0
logger.info("Estimated RAM usage per pixel is {} * safety factor ({})"
.format( Memory.format(ram_usage_per_requested_pixel), safety_factor ) )
ram_usage_per_requested_pixel *= safety_factor
if ideal_blockshape is None:
blockshape = determineBlockShape( input_shape, available_ram/(self._num_threads*ram_usage_per_requested_pixel) )
if 'c' in outputSlot.meta.getAxisKeys():
blockshape = blockshape[:channel_index] + (num_channels,) + blockshape[channel_index:]
warnings.warn( "Chose an arbitrary request blockshape {}".format( blockshape ) )
else:
logger.info("determining blockshape assuming available_ram is {}"
", split between {} threads"
.format(Memory.format(available_ram), self._num_threads))
# By convention, ram_usage_per_requested_pixel refers to the ram used when requesting ALL channels of a 'pixel'
# Therefore, we do not include the channel dimension in the blockshapes here.
blockshape = determine_optimal_request_blockshape( max_blockshape,
ideal_blockshape,
ram_usage_per_requested_pixel,
self._num_threads,
available_ram )
if 'c' in outputSlot.meta.getAxisKeys():
blockshape = blockshape[:channel_index] + (num_channels,) + blockshape[channel_index:]
logger.info( "Chose blockshape: {}".format( blockshape ) )
fmt = Memory.format(ram_usage_per_requested_pixel *
numpy.prod(blockshape[:-1]))
logger.info("Estimated RAM usage per block is {}".format(fmt))
return blockshape
def setupOutputs(self):
shape, dtype, axiskeys = self._default_accessor.shape, self._default_accessor.dtype, self._default_accessor.axiskeys
try:
no_extents_checking = bool(int(os.getenv("LAZYFLOW_NO_DVID_EXTENTS", 0)))
except ValueError:
raise RuntimeError("Didn't understand value for environment variable "\
"LAZYFLOW_NO_DVID_EXTENTS: '{}'. Please use either 0 or 1."
.format(os.getenv("LAZYFLOW_NO_DVID_EXTENTS")))
if no_extents_checking or (None in shape):
# In headless mode, we allow the users to request regions outside the currently valid regions of the image.
# For now, the easiest way to allow that is to simply hard-code DVID volumes to have a really large (1M cubed) shape.
logger.info("Using absurdly large DVID volume extents, to allow out-of-bounds requests.")
tagged_shape = collections.OrderedDict( zip(axiskeys, shape) )
for k,v in tagged_shape.items():
if k in 'xyz':
tagged_shape[k] = int(1e6)
shape = tuple(tagged_shape.values())
num_channels = shape[-1]
self.Output.meta.shape = shape
self.Output.meta.dtype = dtype.type
self.Output.meta.axistags = vigra.defaultAxistags( axiskeys ) # FIXME: Also copy resolution, etc.
# To avoid requesting extremely large blocks, limit each request to 500MB each.
# Note that this isn't a hard max: halos, etc. may increase this somewhat.
max_pixels = 2**29 / self.Output.meta.dtype().nbytes
max_blockshape = determineBlockShape( self.Output.meta.shape, max_pixels )
self.Output.meta.max_blockshape = max_blockshape
self.Output.meta.ideal_blockshape = max_blockshape
# For every request, we probably need room RAM for the array and for the http buffer
# (and hopefully nothing more)
self.Output.meta.ram_usage_per_requested_pixel = 2 * dtype.type().nbytes * num_channels
def setupOutputs(self):
self.outputs["WriteImage"].meta.shape = (1,)
self.outputs["WriteImage"].meta.dtype = object
self.f = self.inputs["h5N5File"].value
h5N5Path = self.inputs["h5N5Path"].value
# On windows, there may be backslashes.
h5N5Path = h5N5Path.replace("\\", "/")
h5N5GroupName, datasetName = os.path.split(h5N5Path)
if h5N5GroupName == "":
g = self.f
else:
if h5N5GroupName in self.f:
g = self.f[h5N5GroupName]
else:
g = self.f.create_group(h5N5GroupName)
dataShape = self.Image.meta.shape
self.logger.info(f"Data shape: {dataShape}")
dtype = self.Image.meta.dtype
if isinstance(dtype, numpy.dtype):
# Make sure we're dealing with a type (e.g. numpy.float64),
# not a numpy.dtype
dtype = dtype.type
# Set up our chunk shape: Aim for a cube that's roughly 512k in size
dtypeBytes = dtype().nbytes
tagged_maxshape = self.Image.meta.getTaggedShape()
if "t" in tagged_maxshape:
# Assume that chunks should not span multiple t-slices,
# and channels are often handled separately, too.
tagged_maxshape["t"] = 1
if "c" in tagged_maxshape:
tagged_maxshape["c"] = 1
self.chunkShape = determineBlockShape(list(tagged_maxshape.values()), 512_000.0 / dtypeBytes)
if datasetName in list(g.keys()):
del g[datasetName]
kwargs = {"shape": dataShape, "dtype": dtype, "chunks": self.chunkShape}
if self.CompressionEnabled.value:
kwargs["compression"] = "gzip" # <-- Would be nice to use lzf compression here, but that is h5py-specific.
if isinstance(self.f, h5py.File):
kwargs["compression_opts"] = 1 # <-- Optimize for speed, not disk space.
else: # z5py has uses different names here
kwargs["level"] = 1 # <-- Optimize for speed, not disk space.
else:
if isinstance(self.f, z5py.N5File): # n5 uses gzip level 5 as default compression.
kwargs["compression"] = "raw"
self.d = g.create_dataset(datasetName, **kwargs)
if self.Image.meta.drange is not None:
self.d.attrs["drange"] = self.Image.meta.drange
if self.Image.meta.display_mode is not None:
self.d.attrs["display_mode"] = self.Image.meta.display_mode
def setupOutputs(self):
tagged_shape = self.RawImage.meta.getTaggedShape()
tagged_shape['c'] = 1
# Aim for blocks that are roughly 1MB
block_shape = determineBlockShape(tagged_shape.values(), 1e6)
self.opLabelArray.blockShape.setValue(block_shape)
def setupOutputs(self):
self.outputs["WriteImage"].meta.shape = (1, )
self.outputs["WriteImage"].meta.dtype = object
self.f = self.inputs["hdf5File"].value
hdf5Path = self.inputs["hdf5Path"].value
# On windows, there may be backslashes.
hdf5Path = hdf5Path.replace('\\', '/')
hdf5GroupName, datasetName = os.path.split(hdf5Path)
if hdf5GroupName == "":
g = self.f
else:
if hdf5GroupName in self.f:
g = self.f[hdf5GroupName]
else:
g = self.f.create_group(hdf5GroupName)
dataShape = self.Image.meta.shape
self.logger.info("Data shape: {}".format(dataShape))
dtype = self.Image.meta.dtype
if type(dtype) is numpy.dtype:
# Make sure we're dealing with a type (e.g. numpy.float64),
# not a numpy.dtype
dtype = dtype.type
# Set up our chunk shape: Aim for a cube that's roughly 512k in size
dtypeBytes = dtype().nbytes
tagged_maxshape = self.Image.meta.getTaggedShape()
if 't' in tagged_maxshape:
# Assume that chunks should not span multiple t-slices,
# and channels are often handled separately, too.
tagged_maxshape['t'] = 1
if 'c' in tagged_maxshape:
tagged_maxshape['c'] = 1
self.chunkShape = determineBlockShape(tagged_maxshape.values(),
512000.0 / dtypeBytes)
if datasetName in g.keys():
del g[datasetName]
kwargs = {
'shape': dataShape,
'dtype': dtype,
'chunks': self.chunkShape
}
if self.CompressionEnabled.value:
kwargs[
'compression'] = 'gzip' # <-- Would be nice to use lzf compression here, but that is h5py-specific.
kwargs[
'compression_opts'] = 1 # <-- Optimize for speed, not disk space.
self.d = g.create_dataset(datasetName, **kwargs)
if self.Image.meta.drange is not None:
self.d.attrs['drange'] = self.Image.meta.drange
if self.Image.meta.display_mode is not None:
self.d.attrs['display_mode'] = self.Image.meta.display_mode
def setupOutputs(self):
# We assume that channel the last axis
assert self.FeatureImage.meta.getAxisKeys()[-1] == 'c'
assert self.LabelImage.meta.getAxisKeys()[-1] == 'c'
assert self.LabelImage.meta.shape[-1] == 1
# For now, we assume that the two input images have the same shape (except channel)
# This constraint could be relaxed in the future if necessary
assert self.FeatureImage.meta.shape[:-1] == self.LabelImage.meta.shape[:-1],\
"FeatureImage and LabelImage shapes do not match: {} vs {}"\
"".format( self.FeatureImage.meta.shape, self.LabelImage.meta.shape )
self.LabelAndFeatureMatrix.meta.shape = (1,)
self.LabelAndFeatureMatrix.meta.dtype = object
num_feature_channels = self.FeatureImage.meta.shape[-1]
if num_feature_channels != self.LabelAndFeatureMatrix.meta.num_feature_channels:
self.LabelAndFeatureMatrix.meta.num_feature_channels = num_feature_channels
self.LabelAndFeatureMatrix.setDirty()
self.ProgressSignal.meta.shape = (1,)
self.ProgressSignal.meta.dtype = object
self.ProgressSignal.setValue( self.progressSignal )
# Auto-choose a blockshape
blockshape = determineBlockShape( self.LabelImage.meta.shape,
OpFeatureMatrixCache.MAX_BLOCK_PIXELS )
self._init_blocks(self.LabelImage.meta.shape, blockshape)
def setupOutputs(self):
shape, dtype, axiskeys = self._default_accessor.shape, self._default_accessor.dtype, self._default_accessor.axiskeys
# FIXME: For now, we hard-code DVID volumes to have a large (1M cubed) shape.
tagged_shape = collections.OrderedDict( zip(axiskeys, shape) )
for k,v in tagged_shape.items():
if k in 'xyz':
tagged_shape[k] = int(1e6)
shape = tuple(tagged_shape.values())
num_channels = shape[0]
if self._transpose_axes:
shape = tuple(reversed(shape))
axiskeys = "".join(reversed(axiskeys))
self.Output.meta.shape = shape
self.Output.meta.dtype = dtype.type
self.Output.meta.axistags = vigra.defaultAxistags( axiskeys ) # FIXME: Also copy resolution, etc.
# To avoid requesting extremely large blocks, limit each request to 500MB each.
max_pixels = 2**29 / self.Output.meta.dtype().nbytes
self.Output.meta.ideal_blockshape = determineBlockShape( self.Output.meta.shape, max_pixels )
# For every request, we probably need room RAM for the array and for the http buffer
# (and hopefully nothing more)
self.Output.meta.ram_usage_per_requested_pixel = 2 * dtype.type().nbytes * num_channels
def setupOutputs(self):
tagged_shape = self.RawImage.meta.getTaggedShape()
tagged_shape['c'] = 1
# Aim for blocks that are roughly 1MB
block_shape = determineBlockShape( tagged_shape.values(), 1e6 )
self.opLabelArray.blockShape.setValue( block_shape )
def setupOutputs(self):
shape, dtype, axiskeys = self._default_accessor.shape, self._default_accessor.dtype, self._default_accessor.axiskeys
# FIXME: For now, we hard-code DVID volumes to have a large (1M cubed) shape.
tagged_shape = collections.OrderedDict(zip(axiskeys, shape))
for k, v in tagged_shape.items():
if k in 'xyz':
tagged_shape[k] = int(1e6)
shape = tuple(tagged_shape.values())
num_channels = shape[0]
if self._transpose_axes:
shape = tuple(reversed(shape))
axiskeys = "".join(reversed(axiskeys))
self.Output.meta.shape = shape
self.Output.meta.dtype = dtype.type
self.Output.meta.axistags = vigra.defaultAxistags(
axiskeys) # FIXME: Also copy resolution, etc.
# To avoid requesting extremely large blocks, limit each request to 500MB each.
max_pixels = 2**29 / self.Output.meta.dtype().nbytes
self.Output.meta.ideal_blockshape = determineBlockShape(
self.Output.meta.shape, max_pixels)
# For every request, we probably need room RAM for the array and for the http buffer
# (and hopefully nothing more)
self.Output.meta.ram_usage_per_requested_pixel = 2 * dtype.type(
).nbytes * num_channels
def setupOutputs(self):
tagged_shape = self.RawImage.meta.getTaggedShape()
# labels are created for one channel (i.e. the label) and only in the
# current time slice, so we can set both c and t to 1
tagged_shape['c'] = 1
tagged_shape['t'] = 1
# Aim for blocks that are roughly 1MB
block_shape = determineBlockShape( tagged_shape.values(), 1e6 )
self.opLabelArray.blockShape.setValue( block_shape )
def setupOutputs(self):
tagged_shape = self.RawImage.meta.getTaggedShape()
# labels are created for one channel (i.e. the label) and only in the
# current time slice, so we can set both c and t to 1
tagged_shape['c'] = 1
if 't' in tagged_shape:
tagged_shape['t'] = 1
# Aim for blocks that are roughly 1MB
block_shape = determineBlockShape(tagged_shape.values(), 20**3)
self.opLabelArray.blockShape.setValue(block_shape)
def setupOutputs(self):
self.outputs["WriteImage"].meta.shape = (1,)
self.outputs["WriteImage"].meta.dtype = object
self.f = self.inputs["hdf5File"].value
hdf5Path = self.inputs["hdf5Path"].value
# On windows, there may be backslashes.
hdf5Path = hdf5Path.replace('\\', '/')
hdf5GroupName, datasetName = os.path.split(hdf5Path)
if hdf5GroupName == "":
g = self.f
else:
if hdf5GroupName in self.f:
g = self.f[hdf5GroupName]
else:
g = self.f.create_group(hdf5GroupName)
dataShape=self.Image.meta.shape
self.logger.info( "Data shape: {}".format(dataShape))
dtype = self.Image.meta.dtype
if type(dtype) is numpy.dtype:
# Make sure we're dealing with a type (e.g. numpy.float64),
# not a numpy.dtype
dtype = dtype.type
# Set up our chunk shape: Aim for a cube that's roughly 512k in size
dtypeBytes = dtype().nbytes
tagged_maxshape = self.Image.meta.getTaggedShape()
if 't' in tagged_maxshape:
# Assume that chunks should not span multiple t-slices,
# and channels are often handled separately, too.
tagged_maxshape['t'] = 1
if 'c' in tagged_maxshape:
tagged_maxshape['c'] = 1
self.chunkShape = determineBlockShape( tagged_maxshape.values(), 512000.0 / dtypeBytes )
if datasetName in g.keys():
del g[datasetName]
kwargs = { 'shape' : dataShape, 'dtype' : dtype,
'chunks' : self.chunkShape }
if self.CompressionEnabled.value:
kwargs['compression'] = 'gzip' # <-- Would be nice to use lzf compression here, but that is h5py-specific.
kwargs['compression_opts'] = 1 # <-- Optimize for speed, not disk space.
self.d=g.create_dataset(datasetName, **kwargs)
if self.Image.meta.drange is not None:
self.d.attrs['drange'] = self.Image.meta.drange
if self.Image.meta.display_mode is not None:
self.d.attrs['display_mode'] = self.Image.meta.display_mode
def _determine_blockshape(self, outputSlot):
"""
Choose a blockshape using the slot metadata (if available) or an arbitrary guess otherwise.
"""
input_shape = outputSlot.meta.shape
max_blockshape = input_shape
ideal_blockshape = outputSlot.meta.ideal_blockshape
ram_usage_per_requested_pixel = outputSlot.meta.ram_usage_per_requested_pixel
num_threads = max(1, Request.global_thread_pool.num_workers)
if lazyflow.AVAILABLE_RAM_MB != 0:
available_ram = lazyflow.AVAILABLE_RAM_MB * 1e6
else:
available_ram = psutil.virtual_memory().available
if ram_usage_per_requested_pixel is None:
# Make a conservative guess: 2*(bytes for dtype) * (num channels) + (fudge factor=4)
ram_usage_per_requested_pixel = 2 * outputSlot.meta.dtype(
).nbytes * outputSlot.meta.shape[-1] + 4
logger.warn("Unknown per-pixel RAM requirement. Making a guess.")
# Safety factor (fudge factor): Double the estimated RAM usage per pixel
safety_factor = 2.0
logger.info(
"Estimated RAM usage per pixel is {} bytes * safety factor ({})".
format(ram_usage_per_requested_pixel, safety_factor))
ram_usage_per_requested_pixel *= safety_factor
if ideal_blockshape is None:
blockshape = determineBlockShape(
input_shape,
available_ram / (num_threads * ram_usage_per_requested_pixel))
logger.warn(
"Chose an arbitrary request blockshape {}".format(blockshape))
else:
logger.info(
"determining blockshape assuming available_ram is {} GB, split between {} threads"
.format(available_ram / 1e9, num_threads))
# By convention, ram_usage_per_requested_pixel refers to the ram used when requesting ALL channels of a 'pixel'
# Therefore, we do not include the channel dimension in the blockshapes here.
blockshape = determine_optimal_request_blockshape(
max_blockshape[:-1], ideal_blockshape[:-1],
ram_usage_per_requested_pixel, num_threads, available_ram)
blockshape += (outputSlot.meta.shape[-1], )
logger.info("Chose blockshape: {}".format(blockshape))
logger.info("Estimated RAM usage per block is {} GB".format(
ram_usage_per_requested_pixel * numpy.prod(blockshape[:-1]) /
1e9))
return blockshape
def _determine_blockshape(self, outputSlot):
"""
Choose a blockshape using the slot metadata (if available) or an arbitrary guess otherwise.
"""
input_shape = outputSlot.meta.shape
max_blockshape = input_shape
ideal_blockshape = outputSlot.meta.ideal_blockshape
ram_usage_per_requested_pixel = outputSlot.meta.ram_usage_per_requested_pixel
num_threads = Request.global_thread_pool.num_workers
if lazyflow.AVAILABLE_RAM_MB != 0:
available_ram = lazyflow.AVAILABLE_RAM_MB * 1e6
else:
available_ram = psutil.virtual_memory().available
if ram_usage_per_requested_pixel is None:
# Make a conservative guess: 2*(bytes for dtype) * (num channels) + (fudge factor=4)
ram_usage_per_requested_pixel = 2*outputSlot.meta.dtype().nbytes*outputSlot.meta.shape[-1] + 4
logger.warn( "Unknown per-pixel RAM requirement. Making a guess." )
# Safety factor (fudge factor): Double the estimated RAM usage per pixel
safety_factor = 2.0
logger.info( "Estimated RAM usage per pixel is {} bytes * safety factor ({})"
.format( ram_usage_per_requested_pixel, safety_factor ) )
ram_usage_per_requested_pixel *= safety_factor
if ideal_blockshape is None:
blockshape = determineBlockShape( input_shape, available_ram/(num_threads*ram_usage_per_requested_pixel) )
logger.warn( "Chose an arbitrary request blockshape {}".format( blockshape ) )
else:
logger.info( "determining blockshape assuming available_ram is {} GB, split between {} threads"
.format( available_ram/1e9, num_threads ) )
# By convention, ram_usage_per_requested_pixel refers to the ram used when requesting ALL channels of a 'pixel'
# Therefore, we do not include the channel dimension in the blockshapes here.
blockshape = determine_optimal_request_blockshape( max_blockshape[:-1],
ideal_blockshape[:-1],
ram_usage_per_requested_pixel,
num_threads,
available_ram )
blockshape += (outputSlot.meta.shape[-1],)
logger.info( "Chose blockshape: {}".format( blockshape ) )
logger.info( "Estimated RAM usage per block is {} GB"
.format( ram_usage_per_requested_pixel * numpy.prod( blockshape[:-1] ) / 1e9 ) )
return blockshape
def setupOutputs(self):
"""Copied that from opPixelClassification.py
opLabelArray is OpCompressedUserLabelArray -> needs to be configured:
- blockShape needs to be set!
"""
tagged_shape = self.RawImage.meta.getTaggedShape()
# labels are created for one channel (i.e. the label) and only in the
# current time slice, so we can set both c and t to 1
tagged_shape['c'] = 1
if 't' in tagged_shape:
tagged_shape['t'] = 1
# Aim for blocks with roughly the same size as in pixel classification,
# taking into account that counting will be 2d: 40 ** 3 = 256 ** 2
block_shape = determineBlockShape(list(tagged_shape.values()), 256 ** 2)
self.opLabelArray.blockShape.setValue(block_shape)
self.opBoxArray.blockShape.setValue(block_shape)
def setupOutputs(self):
shape, dtype, axiskeys = self._default_accessor.shape, self._default_accessor.dtype, self._default_accessor.axiskeys
num_channels = shape[0]
if self._transpose_axes:
shape = tuple(reversed(shape))
axiskeys = "".join(reversed(axiskeys))
self.Output.meta.shape = shape
self.Output.meta.dtype = dtype.type
self.Output.meta.axistags = vigra.defaultAxistags( axiskeys ) # FIXME: Also copy resolution, etc.
# To avoid requesting extremely large blocks, limit each request to 500MB each.
max_pixels = 2**29 / self.Output.meta.dtype().nbytes
self.Output.meta.ideal_blockshape = determineBlockShape( self.Output.meta.shape, max_pixels )
# For every request, we probably need room RAM for the array and for the http buffer
# (and hopefully nothing more)
self.Output.meta.ram_usage_per_requested_pixel = 2 * dtype.type().nbytes * num_channels
def setupOutputs(self):
"""Copied that from opPixelClassification.py
opLabelArray is OpCompressedUserLabelArray -> needs to be configured:
- blockShape needs to be set!
"""
tagged_shape = self.RawImage.meta.getTaggedShape()
# labels are created for one channel (i.e. the label) and only in the
# current time slice, so we can set both c and t to 1
tagged_shape["c"] = 1
if "t" in tagged_shape:
tagged_shape["t"] = 1
# Aim for blocks with roughly the same size as in pixel classification,
# taking into account that counting will be 2d: 40 ** 3 = 256 ** 2
block_shape = determineBlockShape(list(tagged_shape.values()), 256**2)
self.opLabelArray.blockShape.setValue(block_shape)
self.opBoxArray.blockShape.setValue(block_shape)
def _determine_blockshape(self, outputSlot):
"""
Choose a blockshape using the slot metadata (if available) or an arbitrary guess otherwise.
"""
input_shape = outputSlot.meta.shape
max_blockshape = input_shape
ideal_blockshape = outputSlot.meta.ideal_blockshape
ram_usage_per_requested_pixel = outputSlot.meta.ram_usage_per_requested_pixel
num_threads = Request.global_thread_pool.num_workers
available_ram = psutil.virtual_memory().available
# Fudge factor: Reduce RAM usage by a bit
available_ram *= 0.5
if ram_usage_per_requested_pixel is None:
# Make a conservative guess: (bytes for dtype) * (num channels) + (fudge factor=4)
ram_usage_per_requested_pixel = 4 + 2*outputSlot.meta.dtype().nbytes*outputSlot.meta.shape[-1]
logger.warn( "Unknown RAM usage. Making a guess." )
else:
logger.info( "Estimated RAM usage per pixel is {} bytes"
.format( ram_usage_per_requested_pixel ) )
if ideal_blockshape is None:
blockshape = determineBlockShape( input_shape, available_ram/(num_threads*ram_usage_per_requested_pixel) )
logger.warn( "Chose an arbitrary request blockshape {}".format( blockshape ) )
else:
logger.info( "determining blockshape assuming available_ram is {} GB, split between {} threads"
.format( available_ram/1e9, num_threads ) )
# By convention, ram_usage_per_requested_pixel refers to the ram used when requesting ALL channels of a 'pixel'
# Therefore, we do not include the channel dimension in the blockshapes here.
blockshape = determine_optimal_request_blockshape( max_blockshape[:-1],
ideal_blockshape[:-1],
ram_usage_per_requested_pixel,
num_threads,
available_ram )
blockshape += (outputSlot.meta.shape[-1],)
logger.info( "Chose blockshape: {}".format( blockshape ) )
logger.info( "Estimated RAM usage per block is {} GB"
.format( ram_usage_per_requested_pixel * numpy.prod( blockshape[:-1] ) / 1e9 ) )
return blockshape
def setupOutputs(self):
# We assume that channel the last axis
assert self.FeatureImage.meta.getAxisKeys()[-1] == "c"
assert self.LabelImage.meta.getAxisKeys()[-1] == "c"
assert self.LabelImage.meta.shape[-1] == 1
# For now, we assume that the two input images have the same shape (except channel)
# This constraint could be relaxed in the future if necessary
assert self.FeatureImage.meta.shape[:-1] == self.LabelImage.meta.shape[:-1], (
"FeatureImage and LabelImage shapes do not match: {} vs {}"
"".format(self.FeatureImage.meta.shape,
self.LabelImage.meta.shape))
self.LabelAndFeatureMatrix.meta.shape = (1, )
self.LabelAndFeatureMatrix.meta.dtype = object
self.LabelAndFeatureMatrix.meta.channel_names = self.FeatureImage.meta.channel_names
num_feature_channels = self.FeatureImage.meta.shape[-1]
if num_feature_channels != self.LabelAndFeatureMatrix.meta.num_feature_channels:
self.LabelAndFeatureMatrix.meta.num_feature_channels = num_feature_channels
self.LabelAndFeatureMatrix.setDirty()
self.ProgressSignal.meta.shape = (1, )
self.ProgressSignal.meta.dtype = object
self.ProgressSignal.setValue(self.progressSignal)
if self.LabelImage.meta.ideal_blockshape is not None and all(
self.LabelImage.meta.ideal_blockshape):
blockshape = self.LabelImage.meta.ideal_blockshape
else:
# Auto-choose a blockshape
tagged_shape = self.LabelImage.meta.getTaggedShape()
if "t" in tagged_shape:
# A block should never span multiple time slices.
# For txy volumes, that could lead to lots of extra features being computed.
tagged_shape["t"] = 1
blockshape = determineBlockShape(list(tagged_shape.values()),
40**3)
# Don't span more than 256 px along any axis
blockshape = tuple(min(x, 256) for x in blockshape)
self._init_blocks(self.LabelImage.meta.shape, blockshape)
def setupOutputs(self):
shape, dtype, axiskeys = self._default_accessor.shape, self._default_accessor.dtype, self._default_accessor.axiskeys
num_channels = shape[0]
if self._transpose_axes:
shape = tuple(reversed(shape))
axiskeys = "".join(reversed(axiskeys))
self.Output.meta.shape = shape
self.Output.meta.dtype = dtype.type
self.Output.meta.axistags = vigra.defaultAxistags(
axiskeys) # FIXME: Also copy resolution, etc.
# To avoid requesting extremely large blocks, limit each request to 500MB each.
max_pixels = 2**29 / self.Output.meta.dtype().nbytes
self.Output.meta.ideal_blockshape = determineBlockShape(
self.Output.meta.shape, max_pixels)
# For every request, we probably need room RAM for the array and for the http buffer
# (and hopefully nothing more)
self.Output.meta.ram_usage_per_requested_pixel = 2 * dtype.type(
).nbytes * num_channels
def _determine_blockshape(self, outputSlot):
"""
Choose a blockshape using the slot metadata (if available) or an arbitrary guess otherwise.
"""
input_shape = outputSlot.meta.shape
ideal_blockshape = outputSlot.meta.ideal_blockshape
ram_usage_per_requested_pixel = outputSlot.meta.ram_usage_per_requested_pixel
max_blockshape = outputSlot.meta.max_blockshape or input_shape
num_channels = 1
tagged_shape = outputSlot.meta.getTaggedShape()
available_ram = Memory.getAvailableRamComputation()
# Generally, we don't want to split requests across channels.
if "c" in list(tagged_shape.keys()):
num_channels = tagged_shape["c"]
channel_index = list(tagged_shape.keys()).index("c")
input_shape = input_shape[:channel_index] + input_shape[
channel_index + 1:]
max_blockshape = max_blockshape[:channel_index] + max_blockshape[
channel_index + 1:]
if ideal_blockshape:
# Never enlarge 'ideal' in the channel dimension.
num_channels = ideal_blockshape[channel_index]
ideal_blockshape = ideal_blockshape[:
channel_index] + ideal_blockshape[
channel_index + 1:]
del tagged_shape["c"]
# Generally, we don't want to join time slices
if "t" in tagged_shape.keys():
blockshape_time_steps = 1
time_index = list(tagged_shape.keys()).index("t")
input_shape = input_shape[:time_index] + input_shape[time_index +
1:]
max_blockshape = max_blockshape[:time_index] + max_blockshape[
time_index + 1:]
if ideal_blockshape:
# Never enlarge 'ideal' in the time dimension.
blockshape_time_steps = ideal_blockshape[time_index]
ideal_blockshape = ideal_blockshape[:
time_index] + ideal_blockshape[
time_index + 1:]
available_ram /= blockshape_time_steps
del tagged_shape["t"]
if ram_usage_per_requested_pixel is None:
# Make a conservative guess: 2*(bytes for dtype) * (num channels) + (fudge factor=4)
ram_usage_per_requested_pixel = 2 * outputSlot.meta.dtype(
).nbytes * num_channels + 4
warnings.warn(
"Unknown per-pixel RAM requirement. Making a guess.")
# Safety factor (fudge factor): Double the estimated RAM usage per pixel
safety_factor = 2.0
logger.info(
"Estimated RAM usage per pixel is {} * safety factor ({})".format(
Memory.format(ram_usage_per_requested_pixel), safety_factor))
ram_usage_per_requested_pixel *= safety_factor
if ideal_blockshape is None:
blockshape = determineBlockShape(
input_shape,
(available_ram //
(self._num_threads * ram_usage_per_requested_pixel)))
blockshape = tuple(numpy.minimum(max_blockshape, blockshape))
warnings.warn("Chose an arbitrary request blockshape")
else:
logger.info("determining blockshape assuming available_ram is {}"
", split between {} threads".format(
Memory.format(available_ram), self._num_threads))
# By convention, ram_usage_per_requested_pixel refers to the ram used when requesting ALL channels of a 'pixel'
# Therefore, we do not include the channel dimension in the blockshapes here.
#
# Also, it rarely makes sense to request more than one time slice, so we omit that, too. (See above.)
blockshape = determine_optimal_request_blockshape(
max_blockshape, ideal_blockshape,
ram_usage_per_requested_pixel, self._num_threads,
available_ram)
# compute the RAM size of the block before adding back t anc c dimensions
fmt = Memory.format(ram_usage_per_requested_pixel *
numpy.prod(blockshape))
# If we removed time and channel from consideration, add them back now before returning
if "t" in outputSlot.meta.getAxisKeys():
blockshape = blockshape[:time_index] + (
blockshape_time_steps, ) + blockshape[time_index:]
if "c" in outputSlot.meta.getAxisKeys():
blockshape = blockshape[:channel_index] + (
num_channels, ) + blockshape[channel_index:]
logger.info("Chose blockshape: {}".format(blockshape))
logger.info("Estimated RAM usage per block is {}".format(fmt))
return blockshape
def testSmallMax(self):
# In this case, the target size is too large for the given max_shape
# Therefore, block_shape == max_shape
max_shape = (1,2,3,2,1)
block_shape = determineBlockShape( max_shape, 1000 )
assert block_shape == max_shape
def testBasic2(self):
max_shape = (1, 100, 5, 200, 3)
block_shape = determineBlockShape(max_shape, 1000)
assert block_shape == (1, 8, 5, 8, 3)
def testBasic(self):
max_shape = (1000, 1000, 1000, 1)
block_shape = determineBlockShape(max_shape, 1e6)
assert block_shape == (
100, 100, 100,
1), "Got {}, expected (100,100,100,1)".format(block_shape)
def _determine_blockshape(self, outputSlot):
"""
Choose a blockshape using the slot metadata (if available) or an arbitrary guess otherwise.
"""
input_shape = outputSlot.meta.shape
ideal_blockshape = outputSlot.meta.ideal_blockshape
ram_usage_per_requested_pixel = outputSlot.meta.ram_usage_per_requested_pixel
num_channels = 1
tagged_shape = outputSlot.meta.getTaggedShape()
# Generally, we don't want to split requests across channels.
if 'c' in tagged_shape.keys():
num_channels = tagged_shape['c']
channel_index = tagged_shape.keys().index('c')
input_shape = input_shape[:channel_index] + input_shape[channel_index+1:]
if ideal_blockshape:
# Never enlarge 'ideal' in the channel dimension.
num_channels = ideal_blockshape[channel_index]
ideal_blockshape = ideal_blockshape[:channel_index] + ideal_blockshape[channel_index+1:]
max_blockshape = input_shape
num_threads = max(1, Request.global_thread_pool.num_workers)
available_ram = Memory.getAvailableRamComputation()
if ram_usage_per_requested_pixel is None:
# Make a conservative guess: 2*(bytes for dtype) * (num channels) + (fudge factor=4)
ram_usage_per_requested_pixel = 2*outputSlot.meta.dtype().nbytes*num_channels + 4
warnings.warn( "Unknown per-pixel RAM requirement. Making a guess." )
# Safety factor (fudge factor): Double the estimated RAM usage per pixel
safety_factor = 2.0
logger.info("Estimated RAM usage per pixel is {} * safety factor ({})"
.format( Memory.format(ram_usage_per_requested_pixel), safety_factor ) )
ram_usage_per_requested_pixel *= safety_factor
if ideal_blockshape is None:
blockshape = determineBlockShape( input_shape, available_ram/(num_threads*ram_usage_per_requested_pixel) )
if 'c' in outputSlot.meta.getAxisKeys():
blockshape = blockshape[:channel_index] + (num_channels,) + blockshape[channel_index:]
warnings.warn( "Chose an arbitrary request blockshape {}".format( blockshape ) )
else:
logger.info("determining blockshape assuming available_ram is {}"
", split between {} threads"
.format(Memory.format(available_ram), num_threads))
# By convention, ram_usage_per_requested_pixel refers to the ram used when requesting ALL channels of a 'pixel'
# Therefore, we do not include the channel dimension in the blockshapes here.
blockshape = determine_optimal_request_blockshape( max_blockshape,
ideal_blockshape,
ram_usage_per_requested_pixel,
num_threads,
available_ram )
if 'c' in outputSlot.meta.getAxisKeys():
blockshape = blockshape[:channel_index] + (num_channels,) + blockshape[channel_index:]
logger.info( "Chose blockshape: {}".format( blockshape ) )
fmt = Memory.format(ram_usage_per_requested_pixel *
numpy.prod(blockshape[:-1]))
logger.info("Estimated RAM usage per block is {}".format(fmt))
return blockshape
def testSmallMax(self):
# In this case, the target size is too large for the given max_shape
# Therefore, block_shape == max_shape
max_shape = (1, 2, 3, 2, 1)
block_shape = determineBlockShape(max_shape, 1000)
assert block_shape == max_shape
def testBasic2(self):
max_shape = (1,100,5,200,3)
block_shape = determineBlockShape( max_shape, 1000 )
assert block_shape == (1,8,5,8,3)
def testBasic(self):
max_shape = (1000,1000,1000,1)
block_shape = determineBlockShape( max_shape, 1e6 )
assert block_shape == (100,100,100,1), "Got {}, expected (100,100,100,1)".format(block_shape)
def _determine_blockshape(self, outputSlot):
"""
Choose a blockshape using the slot metadata (if available) or an arbitrary guess otherwise.
"""
input_shape = outputSlot.meta.shape
ideal_blockshape = outputSlot.meta.ideal_blockshape
ram_usage_per_requested_pixel = outputSlot.meta.ram_usage_per_requested_pixel
max_blockshape = outputSlot.meta.max_blockshape or input_shape
num_channels = 1
tagged_shape = outputSlot.meta.getTaggedShape()
available_ram = Memory.getAvailableRamComputation()
# Generally, we don't want to split requests across channels.
if "c" in list(tagged_shape.keys()):
num_channels = tagged_shape["c"]
channel_index = list(tagged_shape.keys()).index("c")
input_shape = input_shape[:channel_index] + input_shape[channel_index + 1 :]
max_blockshape = max_blockshape[:channel_index] + max_blockshape[channel_index + 1 :]
if ideal_blockshape:
# Never enlarge 'ideal' in the channel dimension.
num_channels = ideal_blockshape[channel_index]
ideal_blockshape = ideal_blockshape[:channel_index] + ideal_blockshape[channel_index + 1 :]
del tagged_shape["c"]
# Generally, we don't want to join time slices
if "t" in tagged_shape.keys():
blockshape_time_steps = 1
time_index = list(tagged_shape.keys()).index("t")
input_shape = input_shape[:time_index] + input_shape[time_index + 1 :]
max_blockshape = max_blockshape[:time_index] + max_blockshape[time_index + 1 :]
if ideal_blockshape:
# Never enlarge 'ideal' in the time dimension.
blockshape_time_steps = ideal_blockshape[time_index]
ideal_blockshape = ideal_blockshape[:time_index] + ideal_blockshape[time_index + 1 :]
available_ram /= blockshape_time_steps
del tagged_shape["t"]
if ram_usage_per_requested_pixel is None:
# Make a conservative guess: 2*(bytes for dtype) * (num channels) + (fudge factor=4)
ram_usage_per_requested_pixel = 2 * outputSlot.meta.dtype().nbytes * num_channels + 4
warnings.warn("Unknown per-pixel RAM requirement. Making a guess.")
# Safety factor (fudge factor): Double the estimated RAM usage per pixel
safety_factor = 2.0
logger.info(
"Estimated RAM usage per pixel is {} * safety factor ({})".format(
Memory.format(ram_usage_per_requested_pixel), safety_factor
)
)
ram_usage_per_requested_pixel *= safety_factor
if ideal_blockshape is None:
blockshape = determineBlockShape(
input_shape, (available_ram // (self._num_threads * ram_usage_per_requested_pixel))
)
blockshape = tuple(numpy.minimum(max_blockshape, blockshape))
warnings.warn("Chose an arbitrary request blockshape")
else:
logger.info(
"determining blockshape assuming available_ram is {}"
", split between {} threads".format(Memory.format(available_ram), self._num_threads)
)
# By convention, ram_usage_per_requested_pixel refers to the ram used when requesting ALL channels of a 'pixel'
# Therefore, we do not include the channel dimension in the blockshapes here.
#
# Also, it rarely makes sense to request more than one time slice, so we omit that, too. (See above.)
blockshape = determine_optimal_request_blockshape(
max_blockshape, ideal_blockshape, ram_usage_per_requested_pixel, self._num_threads, available_ram
)
# If we removed time and channel from consideration, add them back now before returning
if "t" in outputSlot.meta.getAxisKeys():
blockshape = blockshape[:time_index] + (blockshape_time_steps,) + blockshape[time_index:]
if "c" in outputSlot.meta.getAxisKeys():
blockshape = blockshape[:channel_index] + (num_channels,) + blockshape[channel_index:]
logger.info("Chose blockshape: {}".format(blockshape))
fmt = Memory.format(ram_usage_per_requested_pixel * numpy.prod(blockshape[:-1]))
logger.info("Estimated RAM usage per block is {}".format(fmt))
return blockshape