def execute(self, slot, subindex, roi, result):
assert slot == self.MST, "Invalid output slot: {}".format(slot.name)
#first thing, show the user that we are waiting for computations to finish
self.applet.progressSignal.emit(0)
volume_feat = self.Image(*roiFromShape(self.Image.meta.shape)).wait()
labelVolume = self.LabelImage(
*roiFromShape(self.LabelImage.meta.shape)).wait()
self.applet.progress = 0
def updateProgressBar(x):
#send signal iff progress is significant
if x - self.applet.progress > 1 or x == 100:
self.applet.progressSignal.emit(x)
self.applet.progress = x
#mst= MSTSegmentor(labelVolume[0,...,0],
# numpy.asarray(volume_feat[0,...,0], numpy.float32),
# edgeWeightFunctor = "minimum",
# progressCallback = updateProgressBar)
##mst.raw is not set here in order to avoid redundant data storage
#mst.raw = None
newMst = WatershedSegmentor(labelVolume[0, ..., 0],
numpy.asarray(volume_feat[0, ..., 0],
numpy.float32),
edgeWeightFunctor="minimum",
progressCallback=updateProgressBar)
#Output is of shape 1
#result[0] = mst
newMst.raw = None
result[0] = newMst
return result
def execute(self,slot,subindex,roi,result):
assert slot == self.MST, "Invalid output slot: {}".format(slot.name)
#first thing, show the user that we are waiting for computations to finish
self.applet.progressSignal.emit(0)
volume_feat = self.Image( *roiFromShape( self.Image.meta.shape ) ).wait()
labelVolume = self.LabelImage( *roiFromShape( self.LabelImage.meta.shape ) ).wait()
self.applet.progress = 0
def updateProgressBar(x):
#send signal iff progress is significant
if x-self.applet.progress>1 or x==100:
self.applet.progressSignal.emit(x)
self.applet.progress = x
mst= MSTSegmentor(labelVolume[0,...,0],
numpy.asarray(volume_feat[0,...,0], numpy.float32),
edgeWeightFunctor = "minimum",
progressCallback = updateProgressBar)
#mst.raw is not set here in order to avoid redundant data storage
mst.raw = None
#Output is of shape 1
result[0] = mst
return result
def write(self):
"""
Requests the entire input and saves it to the file.
This function executes synchronously.
"""
# TODO: Use a lazyflow.utility.BigRequestStreamer to split up
# this giant request into a series of streamed subrequests.
logger.warning(
"The current implementation of NPY-format data export computes the entire dataset at once, which requires lots of RAM."
)
path = self.Filepath.value
self.progressSignal(0)
final_data = numpy.zeros(self.Input.meta.shape, self.Input.meta.dtype)
def handle_block_result(roi, data):
slicing = roiToSlice(*roi)
final_data[slicing] = data
requester = BigRequestStreamer(self.Input,
roiFromShape(self.Input.meta.shape))
requester.resultSignal.subscribe(handle_block_result)
requester.progressSignal.subscribe(self.progressSignal)
requester.execute()
numpy.save(path, final_data)
self.progressSignal(100)
def _calculate_probabilities(self, roi):
classifier = self.Classifier.value
assert isinstance(
classifier, LazyflowPixelwiseClassifierABC
), f"Classifier {classifier} must be sublcass of {LazyflowPixelwiseClassifierABC}"
upstream_roi = (roi.start, roi.stop)
# Ask for the halo needed by the classifier
axiskeys = self.Image.meta.getAxisKeys()
halo_shape = classifier.get_halo_shape(axiskeys)
assert len(halo_shape) == len(upstream_roi[0])
assert halo_shape[-1] == 0, "Didn't expect a non-zero halo for channel dimension."
# Expand block by halo, then clip to image bounds
upstream_roi = numpy.array(upstream_roi)
upstream_roi[0] -= halo_shape
upstream_roi[1] += halo_shape
upstream_roi = getIntersection(upstream_roi, roiFromShape(self.Image.meta.shape))
upstream_roi = numpy.asarray(upstream_roi)
# Determine how to extract the data from the result (without the halo)
downstream_roi = numpy.array((roi.start, roi.stop))
predictions_roi = downstream_roi[:, :-1] - upstream_roi[0, :-1]
# Request all upstream channels
input_channels = self.Image.meta.shape[-1]
upstream_roi[:, -1] = [0, input_channels]
input_data = self.Image(*upstream_roi).wait()
axistags = self.Image.meta.axistags
probabilities = classifier.predict_probabilities_pixelwise(input_data, predictions_roi, axistags)
return probabilities
def run_export(self):
"""
Request the volume in slices (running in parallel), and write each slice to a separate image.
"""
# Make the directory first if necessary
export_dir = os.path.split(self.FilepathPattern.value)[0]
if not os.path.exists(export_dir):
os.makedirs(export_dir)
# Sliceshape is the same as the input shape, except for the sliced dimension
tagged_sliceshape = self.Input.meta.getTaggedShape()
tagged_sliceshape[self._volume_axes[0]] = 1
slice_shape = (tagged_sliceshape.values())
# Use a request streamer to automatically request a constant batch of 4 active requests.
streamer = BigRequestStreamer( self.Input,
roiFromShape( self.Input.meta.shape ),
slice_shape,
batchSize=4 )
# Write the slices as they come in (possibly out-of-order, but probably not.
streamer.resultSignal.subscribe( self._write_slice )
streamer.progressSignal.subscribe( self.progressSignal )
logger.debug("Starting Stack Export with slicing shape: {}".format( slice_shape ))
streamer.execute()
def test3(self):
# Generate a random dataset and see if it we get the right masking from the operator.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(
data,
mask=numpy.zeros(data.shape, dtype=bool),
shrink=False
)
# Provide input read all output.
self.operator_identity.Input.setValue(numpy.zeros_like(data))
assert(self.operator_identity.Input.meta.has_mask)
assert(self.operator_identity.Output.meta.has_mask)
output = self.operator_identity.Output[None].wait()
assert((output == 0).all())
assert(data.mask.shape == output.mask.shape)
assert((output.mask == False).all())
# Try setInSlot
data_shape_roi = roiFromShape(data.shape)
data_shape_slice = roiToSlice(*data_shape_roi)
self.operator_identity.Input[data_shape_slice] = data
output = self.operator_identity.Output[None].wait()
assert((data == output).all())
assert(data.mask.shape == output.mask.shape)
assert((data.mask == output.mask).all())
def get_new_roi(self) -> Tuple[Tuple, Tuple]:
# Prepare subregion operator
total_roi = roiFromShape(self.Input.meta.shape)
total_roi = list(map(tuple, total_roi))
# Default to full roi
new_start, new_stop = total_roi
if self.RegionStart.ready():
# RegionStart is permitted to contain 'None' values, which we replace with zeros
new_start = [x or 0 for x in self.RegionStart.value]
if self.RegionStop.ready():
# RegionStop is permitted to contain 'None' values,
# which we replace with the full extent of the corresponding axis
new_stop = [x_extent[0] or x_extent[1] for x_extent in zip(self.RegionStop.value, total_roi[1])]
clipped_start = numpy.maximum(0, new_start)
clipped_stop = numpy.minimum(total_roi[1], new_stop)
if (clipped_start != new_start).any() or (clipped_stop != new_stop).any():
warnings.warn(
"The ROI you are attempting to export exceeds the extents of your dataset. Clipping to dataset bounds."
)
new_start, new_stop = tuple(clipped_start), tuple(clipped_stop)
return new_start, new_stop
def _calculate_probabilities(self, roi):
classifier = self.Classifier.value
assert isinstance(
classifier, LazyflowPixelwiseClassifierABC
), f"Classifier {classifier} must be sublcass of {LazyflowPixelwiseClassifierABC}"
upstream_roi = (roi.start, roi.stop)
# Ask for the halo needed by the classifier
axiskeys = self.Image.meta.getAxisKeys()
halo_shape = classifier.get_halo_shape(axiskeys)
assert len(halo_shape) == len(upstream_roi[0])
assert halo_shape[-1] == 0, "Didn't expect a non-zero halo for channel dimension."
# Expand block by halo, then clip to image bounds
upstream_roi = numpy.array(upstream_roi)
upstream_roi[0] -= halo_shape
upstream_roi[1] += halo_shape
upstream_roi = getIntersection(upstream_roi, roiFromShape(self.Image.meta.shape))
upstream_roi = numpy.asarray(upstream_roi)
# Determine how to extract the data from the result (without the halo)
downstream_roi = numpy.array((roi.start, roi.stop))
predictions_roi = downstream_roi[:, :-1] - upstream_roi[0, :-1]
# Request all upstream channels
input_channels = self.Image.meta.shape[-1]
upstream_roi[:, -1] = [0, input_channels]
input_data = self.Image(*upstream_roi).wait()
axistags = self.Image.meta.axistags
probabilities = classifier.predict_probabilities_pixelwise(input_data, predictions_roi, axistags)
return probabilities
def _executeExportPath(self, result):
path_format = self.OutputFilenameFormat.value
file_extension = self._export_impls[self.OutputFormat.value][0]
# Remove existing extension (if present) and add the correct extension (if any)
if file_extension:
path_format = os.path.splitext(path_format)[0]
path_format += '.' + file_extension
# Provide the TOTAL path (including dataset name)
if self.OutputFormat.value == 'hdf5':
path_format += '/' + self.OutputInternalPath.value
roi = numpy.array(roiFromShape(self.Input.meta.shape))
# Intermediate state can cause coordinate offset and input shape to be mismatched.
# Just don't use the offset if it looks wrong.
# (The client will provide a valid offset later on.)
if self.CoordinateOffset.ready() and len(
self.CoordinateOffset.value) == len(roi[0]):
offset = self.CoordinateOffset.value
assert len(roi[0] == len(offset))
roi += offset
optional_replacements = {}
optional_replacements['roi'] = map(tuple, roi)
for key, (start, stop) in zip(self.Input.meta.getAxisKeys(),
roi.transpose()):
optional_replacements[key + '_start'] = start
optional_replacements[key + '_stop'] = stop
formatted_path = format_known_keys(path_format, optional_replacements)
result[0] = formatted_path
return result
def execute(self, slot, subindex, rroi, result):
self.progressSignal(0)
# Save the axistags as a dataset attribute
self.d.attrs['axistags'] = self.Image.meta.axistags.toJSON()
def handle_block_result(roi, data):
slicing = roiToSlice(*roi)
if data.flags.c_contiguous:
self.d.write_direct(data.view(numpy.ndarray), dest_sel=slicing)
else:
self.d[slicing] = data
batch_size = None
if self.BatchSize.ready():
batch_size = self.BatchSize.value
requester = BigRequestStreamer(self.Image,
roiFromShape(self.Image.meta.shape),
batchSize=batch_size)
requester.resultSignal.subscribe(handle_block_result)
requester.progressSignal.subscribe(self.progressSignal)
requester.execute()
# Be paranoid: Flush right now.
self.f.file.flush()
# We're finished.
result[0] = True
self.progressSignal(100)
def test_via_OpExportSlot(self):
data = 255 * numpy.random.random( (64, 128, 128, 1) )
data = data.astype( numpy.uint8 )
data = vigra.taggedView( data, vigra.defaultAxistags('zyxc') )
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue( data )
opExport = OpExportSlot(graph=graph)
opExport.Input.connect( opPiper.Output )
opExport.OutputFormat.setValue( 'dvid' )
url = 'http://{hostname}/api/node/{data_uuid}/{data_name}'.format( **self.__dict__ )
opExport.OutputFilenameFormat.setValue( url )
assert opExport.ExportPath.ready()
assert opExport.ExportPath.value == url
opExport.run_export()
opRead = OpInputDataReader( graph=graph )
try:
opRead.FilePath.setValue( opExport.ExportPath.value )
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output( *roiFromShape(data.shape) ).wait()
assert (read_data == expected_data).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def _executeExportPath(self, result):
path_format = self.OutputFilenameFormat.value
file_extension = self._export_impls[ self.OutputFormat.value ][0]
# Remove existing extension (if present) and add the correct extension
path_format = os.path.splitext(path_format)[0]
path_format += '.' + file_extension
# Provide the TOTAL path (including dataset name)
if self.OutputFormat.value == 'hdf5':
path_format += '/' + self.OutputInternalPath.value
roi = numpy.array( roiFromShape(self.Input.meta.shape) )
if self.CoordinateOffset.ready():
offset = self.CoordinateOffset.value
assert len(roi[0] == len(offset))
roi += offset
optional_replacements = {}
optional_replacements['roi'] = map(tuple, roi)
for key, (start, stop) in zip( self.Input.meta.getAxisKeys(), roi.transpose() ):
optional_replacements[key + '_start'] = start
optional_replacements[key + '_stop'] = stop
formatted_path = format_known_keys( path_format, optional_replacements )
result[0] = formatted_path
return result
def _executeExportPath(self, result):
path_format = self.OutputFilenameFormat.value
file_extension = self._export_impls[ self.OutputFormat.value ][0]
# Remove existing extension (if present) and add the correct extension (if any)
if file_extension:
path_format = os.path.splitext(path_format)[0]
path_format += '.' + file_extension
# Provide the TOTAL path (including dataset name)
if self.OutputFormat.value == 'hdf5':
path_format += '/' + self.OutputInternalPath.value
roi = numpy.array( roiFromShape(self.Input.meta.shape) )
# Intermediate state can cause coordinate offset and input shape to be mismatched.
# Just don't use the offset if it looks wrong.
# (The client will provide a valid offset later on.)
if self.CoordinateOffset.ready() and len(self.CoordinateOffset.value) == len(roi[0]):
offset = self.CoordinateOffset.value
assert len(roi[0] == len(offset))
roi += offset
optional_replacements = {}
optional_replacements['roi'] = map(tuple, roi)
for key, (start, stop) in zip( self.Input.meta.getAxisKeys(), roi.transpose() ):
optional_replacements[key + '_start'] = start
optional_replacements[key + '_stop'] = stop
formatted_path = format_known_keys( path_format, optional_replacements )
result[0] = formatted_path
return result
def __init__(self, slot, start = None, stop = None, pslice = None):
super(SubRegion,self).__init__(slot)
shape = None
if slot is not None:
shape = slot.meta.shape
if pslice != None or start is not None and stop is None and pslice is None:
if pslice is None:
pslice = start
if shape is None:
# Okay to use a shapeless slot if the key is bounded
# AND if the key has the correct length
assert slicingtools.is_bounded(pslice)
# Supply a dummy shape
shape = [0] * len(pslice)
self.start, self.stop = sliceToRoi(pslice,shape)
elif start is None and pslice is None:
assert shape is not None, "Can't create a default subregion without a slot and a shape."
self.start, self.stop = roiFromShape(shape)
else:
self.start = TinyVector(start)
self.stop = TinyVector(stop)
self.dim = len(self.start)
for start, stop in zip(self.start, self.stop):
assert isinstance(start, (int, long, numpy.integer)), "Roi contains non-integers: {}".format( self )
assert isinstance(start, (int, long, numpy.integer)), "Roi contains non-integers: {}".format( self )
def test_via_OpExportSlot(self):
data = 255 * numpy.random.random((64, 128, 128, 1))
data = data.astype(numpy.uint8)
data = vigra.taggedView(data, vigra.defaultAxistags("zyxc"))
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue(data)
opExport = OpExportSlot(graph=graph)
opExport.Input.connect(opPiper.Output)
opExport.OutputFormat.setValue("dvid")
url = "http://{hostname}/api/node/{data_uuid}/{data_name}".format(
**self.__dict__)
opExport.OutputFilenameFormat.setValue(url)
assert opExport.ExportPath.ready()
assert opExport.ExportPath.value == url
opExport.run_export()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(opExport.ExportPath.value)
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output(*roiFromShape(data.shape)).wait()
assert (read_data == expected_data
).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def ingestData(self, slot):
"""
Read the data from the given slot and copy it into this cache.
The rules about special pixel meanings apply here, just like setInSlot
Returns: the max label found in the slot.
"""
assert self._blockshape is not None
assert self.Input.meta.shape == slot.meta.shape
max_label = 0
# Get logical blocking.
block_starts = getIntersectingBlocks( self._blockshape, roiFromShape(self.Input.meta.shape) )
block_starts = map( tuple, block_starts )
# Write each block
for block_start in block_starts:
block_roi = getBlockBounds( self.Input.meta.shape, self._blockshape, block_start )
# Request the block data
block_data = slot(*block_roi).wait()
# Write into the array
subregion_roi = SubRegion(self.Input, *block_roi)
cleaned_block_data = self._setInSlotInput( self.Input, (), subregion_roi, block_data )
max_label = max( max_label, cleaned_block_data.max() )
return max_label
def test3(self):
# Generate a random dataset and see if it we get the right masking from the operator.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
data = numpy.ma.masked_array(data,
mask=numpy.zeros(data.shape, dtype=bool),
shrink=False)
# Provide input read all output.
self.operator_identity.Input.setValue(numpy.zeros_like(data))
assert self.operator_identity.Input.meta.has_mask
assert self.operator_identity.Output.meta.has_mask
output = self.operator_identity.Output[None].wait()
assert (output == 0).all()
assert data.mask.shape == output.mask.shape
assert (output.mask == False).all()
# Try setInSlot
data_shape_roi = roiFromShape(data.shape)
data_shape_slice = roiToSlice(*data_shape_roi)
self.operator_identity.Input[data_shape_slice] = data
output = self.operator_identity.Output[None].wait()
assert (data == output).all()
assert data.mask.shape == output.mask.shape
assert (data.mask == output.mask).all()
def execute(self, slot, subindex, rroi, result):
self.progressSignal(0)
# Save the axistags as a dataset attribute
self.d.attrs['axistags'] = self.Image.meta.axistags.toJSON()
def handle_block_result(roi, data):
slicing = roiToSlice(*roi)
if data.flags.c_contiguous:
self.d.write_direct(data.view(numpy.ndarray), dest_sel=slicing)
else:
self.d[slicing] = data
batch_size = None
if self.BatchSize.ready():
batch_size = self.BatchSize.value
requester = BigRequestStreamer( self.Image, roiFromShape( self.Image.meta.shape ), batchSize=batch_size )
requester.resultSignal.subscribe( handle_block_result )
requester.progressSignal.subscribe( self.progressSignal )
requester.execute()
# Be paranoid: Flush right now.
self.f.file.flush()
# We're finished.
result[0] = True
self.progressSignal(100)
def write(self):
"""
Requests the entire input and saves it to the file.
This function executes synchronously.
"""
# TODO: Use a lazyflow.utility.BigRequestStreamer to split up
# this giant request into a series of streamed subrequests.
logger.warn("The current implementation of NPY-format data export computes the entire dataset at once, which requires lots of RAM.")
path = self.Filepath.value
self.progressSignal(0)
final_data = numpy.zeros( self.Input.meta.shape, self.Input.meta.dtype )
def handle_block_result(roi, data):
slicing = roiToSlice(*roi)
final_data[slicing] = data
requester = BigRequestStreamer( self.Input, roiFromShape( self.Input.meta.shape ) )
requester.resultSignal.subscribe( handle_block_result )
requester.progressSignal.subscribe( self.progressSignal )
requester.execute()
numpy.save(path, final_data)
self.progressSignal(100)
def run_export(self):
"""
Request the volume in slices (running in parallel), and write each slice to a separate image.
"""
# Make the directory first if necessary
export_dir = os.path.split(self.FilepathPattern.value)[0]
if not os.path.exists(export_dir):
os.makedirs(export_dir)
# Sliceshape is the same as the input shape, except for the sliced dimension
tagged_sliceshape = self.Input.meta.getTaggedShape()
tagged_sliceshape[self._volume_axes[0]] = 1
slice_shape = (tagged_sliceshape.values())
# Use a request streamer to automatically request a constant batch of 4 active requests.
streamer = BigRequestStreamer( self.Input,
roiFromShape( self.Input.meta.shape ),
slice_shape,
batchSize=4 )
# Write the slices as they come in (possibly out-of-order, but probably not.
streamer.resultSignal.subscribe( self._write_slice )
streamer.progressSignal.subscribe( self.progressSignal )
logger.debug("Starting Stack Export with slicing shape: {}".format( slice_shape ))
streamer.execute()
def run_export(self):
"""
Request the volume in slices (running in parallel), and write each slice to the correct page.
Note: We can't use BigRequestStreamer here, because the data for each slice wouldn't be
guaranteed to arrive in the correct order.
"""
# Make the directory first if necessary
export_dir = os.path.split(self.FilepathPattern.value)[0]
if not os.path.exists(export_dir):
os.makedirs(export_dir)
# Blockshape is the same as the input shape, except for the sliced dimension
step_axis = self._volume_axes[0]
tagged_blockshape = self.Input.meta.getTaggedShape()
tagged_blockshape[step_axis] = 1
block_shape = (tagged_blockshape.values())
logger.debug("Starting Multipage Sequence Export with block shape: {}".format( block_shape ))
# Block step is all zeros except step axis, e.g. (0, 1, 0, 0, 0)
block_step = numpy.array( self.Input.meta.getAxisKeys() ) == step_axis
block_step = block_step.astype(int)
filepattern = self.FilepathPattern.value
# If the user didn't provide custom formatting for the slice field,
# auto-format to include zero-padding
if '{slice_index}' in filepattern:
filepattern = filepattern.format( slice_index='{' + 'slice_index:0{}'.format(self._max_slice_digits) + '}' )
self.progressSignal(0)
# Nothing fancy here: Just loop over the blocks in order.
tagged_shape = self.Input.meta.getTaggedShape()
for block_index in xrange( tagged_shape[step_axis] ):
roi = numpy.array(roiFromShape(block_shape))
roi += block_index*block_step
roi = map(tuple, roi)
try:
opSubregion = OpSubRegion( parent=self )
opSubregion.Roi.setValue( roi )
opSubregion.Input.connect( self.Input )
formatted_path = filepattern.format( slice_index=(block_index + self.SliceIndexOffset.value) )
opExportBlock = OpExportMultipageTiff( parent=self )
opExportBlock.Input.connect( opSubregion.Output )
opExportBlock.Filepath.setValue( formatted_path )
block_start_progress = 100*block_index // tagged_shape[step_axis]
def _handleBlockProgress(block_progress):
self.progressSignal( block_start_progress + block_progress//tagged_shape[step_axis] )
opExportBlock.progressSignal.subscribe( _handleBlockProgress )
# Run the export for this block
opExportBlock.run_export()
finally:
opExportBlock.cleanUp()
opSubregion.cleanUp()
self.progressSignal(100)
def run_export(self):
self.progressSignal(0)
url = self.NodeDataUrl.value
url_path = url.split('://')[1]
hostname, api, node, uuid, dataname = url_path.split('/')
assert api == 'api'
assert node == 'node'
axiskeys = self.Input.meta.getAxisKeys()
shape = self.Input.meta.shape
if self._transpose_axes:
axiskeys = reversed(axiskeys)
shape = tuple(reversed(shape))
axiskeys = "".join( axiskeys )
if self.OffsetCoord.ready():
offset_start = self.OffsetCoord.value
else:
offset_start = (0,) * len( self.Input.meta.shape )
self.progressSignal(5)
# Get the dataset details
try:
metadata = VoxelsAccessor.get_metadata(hostname, uuid, dataname)
except VoxelsAccessor.BadRequestError as ex:
# Dataset doesn't exist yet. Let's create it.
metadata = VoxelsMetadata.create_default_metadata( shape,
self.Input.meta.dtype,
axiskeys,
0.0,
"" )
VoxelsAccessor.create_new(hostname, uuid, dataname, metadata)
# Since this class is generally used to push large blocks of data,
# we'll be nice and set throttle=True
client = VoxelsAccessor( hostname, uuid, dataname, throttle=True )
def handle_block_result(roi, data):
# Send it to dvid
roi = numpy.asarray(roi)
roi += offset_start
start, stop = roi
if self._transpose_axes:
data = data.transpose()
start = tuple(reversed(start))
stop = tuple(reversed(stop))
client.post_ndarray( start, stop, data )
requester = BigRequestStreamer( self.Input, roiFromShape( self.Input.meta.shape ) )
requester.resultSignal.subscribe( handle_block_result )
requester.progressSignal.subscribe( self.progressSignal )
requester.execute()
self.progressSignal(100)
def setInSlot(self, slot, subindex, roi, value):
# Forward to the output
self.Output[roiToSlice(roi.start, roi.stop)] = value
entire_roi = roiFromShape(self.Input.meta.shape)
if (numpy.array((roi.start, roi.stop)) == entire_roi).all():
# Nothing is dirty any more.
self._init_fixed_dirty_roi()
def setInSlot(self, slot, subindex, roi, value):
# Forward to the output
self.Output[roiToSlice(roi.start, roi.stop)] = value
entire_roi = roiFromShape(self.Input.meta.shape)
if (numpy.array((roi.start, roi.stop)) == entire_roi).all():
# Nothing is dirty any more.
self._init_fixed_dirty_roi()
def autoSeedBackground(cls, laneView, foreground_label):
# Seed the entire image with background labels, except for the individual label in question
# To save memory, we'll do this in blocks instead of all at once
volume_shape = laneView.RavelerLabels.meta.shape
volume_roi = roiFromShape(volume_shape)
block_shape = (OpSplitBodyCarving.BLOCK_SIZE, ) * len(volume_shape)
block_shape = numpy.minimum(block_shape, volume_shape)
block_starts = getIntersectingBlocks(block_shape, volume_roi)
logger.debug("Auto-seeding {} blocks for label".format(
len(block_starts), foreground_label))
for block_index, block_start in enumerate(block_starts):
block_roi = getBlockBounds(volume_shape, block_shape, block_start)
label_block = laneView.RavelerLabels(*block_roi).wait()
background_block = numpy.where(label_block == foreground_label, 0,
1)
background_block = numpy.asarray(
background_block,
numpy.float32) # Distance transform requires float
if (background_block == 0.0).any():
# We need to leave a small border between the background seeds and the object membranes
background_block_view = background_block.view(vigra.VigraArray)
background_block_view.axistags = copy.copy(
laneView.RavelerLabels.meta.axistags)
background_block_view_4d = background_block_view.bindAxis(
't', 0)
background_block_view_3d = background_block_view_4d.bindAxis(
'c', 0)
distance_transformed_block = vigra.filters.distanceTransform(
background_block_view_3d, background=False)
distance_transformed_block = distance_transformed_block.astype(
numpy.uint8)
# Create a 'hull' surrounding the foreground, but leave some space.
background_seed_block = (distance_transformed_block ==
OpSplitBodyCarving.SEED_MARGIN)
background_seed_block = background_seed_block.astype(
numpy.uint8) * 1 # (In carving, background is label 1)
# # Make the hull VERY sparse to avoid over-biasing graph cut toward the background class
# # FIXME: Don't regenerate this random block on every loop iteration
# rand_bytes = numpy.random.randint(0, 1000, background_seed_block.shape)
# background_seed_block = numpy.where( rand_bytes < 1, background_seed_block, 0 )
# background_seed_block = background_seed_block.view(vigra.VigraArray)
# background_seed_block.axistags = background_block_view_3d.axistags
axisorder = laneView.RavelerLabels.meta.getTaggedShape().keys()
logger.debug("Writing backgound seeds: {}/{}".format(
block_index, len(block_starts)))
laneView.WriteSeeds[roiToSlice(
*block_roi)] = background_seed_block.withAxes(*axisorder)
else:
logger.debug("Skipping all-background block: {}/{}".format(
block_index, len(block_starts)))
def run_export(self):
self.progressSignal(0)
url = self.NodeDataUrl.value
url_path = url.split('://')[1]
hostname, api, node, uuid, dataname = url_path.split('/')
assert api == 'api'
assert node == 'node'
axiskeys = self.Input.meta.getAxisKeys()
shape = self.Input.meta.shape
if self._transpose_axes:
axiskeys = reversed(axiskeys)
shape = tuple(reversed(shape))
axiskeys = "".join(axiskeys)
if self.OffsetCoord.ready():
offset_start = self.OffsetCoord.value
else:
offset_start = (0, ) * len(self.Input.meta.shape)
self.progressSignal(5)
# Get the dataset details
try:
metadata = VoxelsAccessor.get_metadata(hostname, uuid, dataname)
except DVIDException as ex:
if ex.status != 404:
raise
# Dataset doesn't exist yet. Let's create it.
metadata = VoxelsMetadata.create_default_metadata(
shape, self.Input.meta.dtype, axiskeys, 0.0, "")
VoxelsAccessor.create_new(hostname, uuid, dataname, metadata)
# Since this class is generally used to push large blocks of data,
# we'll be nice and set throttle=True
client = VoxelsAccessor(hostname, uuid, dataname, throttle=True)
def handle_block_result(roi, data):
# Send it to dvid
roi = numpy.asarray(roi)
roi += offset_start
start, stop = roi
if self._transpose_axes:
data = data.transpose()
start = tuple(reversed(start))
stop = tuple(reversed(stop))
client.post_ndarray(start, stop, data)
requester = BigRequestStreamer(self.Input,
roiFromShape(self.Input.meta.shape))
requester.resultSignal.subscribe(handle_block_result)
requester.progressSignal.subscribe(self.progressSignal)
requester.execute()
self.progressSignal(100)
def _purge_label(self,
label_to_purge,
decrement_remaining,
replacement_value=0):
"""
Scan through all labeled pixels.
(1) Reassign all pixels of the given value (set to replacement_value)
(2) If decrement_remaining=True, decrement all labels above that
value so the set of stored labels remains consecutive.
Note that the decrement is performed AFTER replacement.
"""
changed_block_rois = []
# stored_block_rois = self.CleanBlocks.value
stored_block_roi_destination = [None]
self.execute(self.CleanBlocks, (), SubRegion(self.Output, (0, ),
(1, )),
stored_block_roi_destination)
stored_block_rois = stored_block_roi_destination[0]
for block_roi in stored_block_rois:
# Get data
block_shape = numpy.subtract(block_roi[1], block_roi[0])
block = self.Output.stype.allocateDestination(
SubRegion(self.Output, *roiFromShape(block_shape)))
self.execute(self.Output, (), SubRegion(self.Output, *block_roi),
block)
# Locate pixels to change
matching_label_coords = numpy.nonzero(block == label_to_purge)
# Change the data
block[matching_label_coords] = replacement_value
coords_to_decrement = block > label_to_purge
if decrement_remaining:
block[coords_to_decrement] -= numpy.uint8(1)
# Update cache with the new data (only if something really changed)
if len(matching_label_coords[0]) > 0 or (
decrement_remaining and coords_to_decrement.sum() > 0):
super(OpCompressedUserLabelArray,
self)._setInSlotInput(self.Input, (),
SubRegion(self.Output, *block_roi),
block,
store_zero_blocks=False)
changed_block_rois.append(block_roi)
for block_roi in changed_block_rois:
# FIXME: Shouldn't this dirty notification be handled in OpUnmanagedCompressedCache?
self.Output.setDirty(*block_roi)
def testBasic_Dvid(self):
if _skip_dvid:
raise nose.SkipTest
# Spin up a mock dvid server to test with.
dvid_dataset, data_uuid, data_name = "datasetA", "abcde", "indices_data"
mockserver_data_file = self._tmpdir + '/mockserver_data.h5'
with H5MockServerDataFile(mockserver_data_file) as test_h5file:
test_h5file.add_node(dvid_dataset, data_uuid)
server_proc, shutdown_event = H5MockServer.create_and_start(
mockserver_data_file,
"localhost",
8000,
same_process=False,
disable_server_logging=True)
try:
data = 255 * numpy.random.random((100, 100, 4))
data = data.astype(numpy.uint8)
data = vigra.taggedView(data, vigra.defaultAxistags('xyc'))
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue(data)
opExport = OpExportSlot(graph=graph)
opExport.Input.connect(opPiper.Output)
opExport.OutputFormat.setValue('dvid')
url = 'http://localhost:8000/api/node/{data_uuid}/{data_name}'.format(
**locals())
opExport.OutputFilenameFormat.setValue(url)
assert opExport.ExportPath.ready()
assert opExport.ExportPath.value == url
opExport.run_export()
try:
opRead = OpInputDataReader(graph=graph)
opRead.FilePath.setValue(opExport.ExportPath.value)
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output(*roiFromShape(data.shape)).wait()
assert (read_data == expected_data
).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
finally:
shutdown_event.set()
server_proc.join()
def run_export(self):
"""
Request the volume in slices (running in parallel), and write each slice to a separate image.
"""
# Make the directory first if necessary
export_dir = os.path.split(self.FilepathPattern.value)[0]
if not os.path.exists(export_dir):
os.makedirs(export_dir)
# Sliceshape is the same as the input shape, except for the sliced dimension
tagged_sliceshape = self.Input.meta.getTaggedShape()
tagged_sliceshape[self._volume_axes[0]] = 1
slice_shape = (list(tagged_sliceshape.values()))
parallel_requests = 4
# If ram usage info is available, make a better guess about how many requests we can launch in parallel
ram_usage_per_requested_pixel = self.Input.meta.ram_usage_per_requested_pixel
if ram_usage_per_requested_pixel is not None:
pixels_per_slice = numpy.prod(slice_shape)
if 'c' in tagged_sliceshape:
pixels_per_slice //= tagged_sliceshape['c']
ram_usage_per_slice = pixels_per_slice * ram_usage_per_requested_pixel
# Fudge factor: Reduce RAM usage by a bit
available_ram = psutil.virtual_memory().available
available_ram *= 0.5
parallel_requests = int(available_ram // ram_usage_per_slice)
if parallel_requests < 1:
raise MemoryError(
'Not enough RAM to export to the selected format. '
'Consider exporting to hdf5 (h5).'
)
streamer = BigRequestStreamer( self.Input,
roiFromShape( self.Input.meta.shape ),
slice_shape,
parallel_requests )
# Write the slices as they come in (possibly out-of-order, but probably not)
streamer.resultSignal.subscribe( self._write_slice )
streamer.progressSignal.subscribe( self.progressSignal )
logger.debug("Starting Stack Export with slicing shape: {}".format( slice_shape ))
streamer.execute()
def run_export_to_array(self):
# Allocate result
final_result = numpy.ndarray( dtype=self.Input.meta.dtype, shape=self.Input.meta.shape )
# Prepare streamer
streamer = BigRequestStreamer( self.Input,
roiFromShape(self.Input.meta.shape),
allowParallelResults=True )
def handle_block_result(roi, block_result):
final_result[roiToSlice(*roi)] = block_result
streamer.resultSignal.subscribe( handle_block_result )
streamer.progressSignal.subscribe( self.progressSignal )
# Perform export
streamer.execute()
return final_result
def run_export_to_array(self):
# Allocate result
final_result = numpy.ndarray(dtype=self.Input.meta.dtype, shape=self.Input.meta.shape)
# Prepare streamer
streamer = BigRequestStreamer(self.Input, roiFromShape(self.Input.meta.shape), allowParallelResults=True)
def handle_block_result(roi, block_result):
final_result[roiToSlice(*roi)] = block_result
streamer.resultSignal.subscribe(handle_block_result)
streamer.progressSignal.subscribe(self.progressSignal)
# Perform export
streamer.execute()
return final_result
def execute(self, slot, subindex, roi, result):
classifier_factory = self.ClassifierFactory.value
assert issubclass(type(classifier_factory), LazyflowPixelwiseClassifierFactoryABC), \
"Factory is of type {}, which does not satisfy the LazyflowPixelwiseClassifierFactoryABC interface."\
"".format( type(classifier_factory) )
# Accumulate all non-zero blocks of each image into lists
label_data_blocks = []
image_data_blocks = []
for image_slot, label_slot, nonzero_block_slot in zip(self.Images, self.Labels, self.nonzeroLabelBlocks):
block_slicings = nonzero_block_slot.value
for block_slicing in block_slicings:
block_label_roi = sliceToRoi( block_slicing, image_slot.meta.shape )
# Ask for the halo needed by the classifier
axiskeys = image_slot.meta.getAxisKeys()
halo_shape = classifier_factory.get_halo_shape(axiskeys)
assert len(halo_shape) == len( block_label_roi[0] )
assert halo_shape[-1] == 0, "Didn't expect a non-zero halo for channel dimension."
# Expand block by halo, then clip to image bounds
block_label_roi = numpy.array( block_label_roi )
block_label_roi[0] -= halo_shape
block_label_roi[1] += halo_shape
block_label_roi = getIntersection( block_label_roi, roiFromShape(image_slot.meta.shape) )
block_image_roi = numpy.array( block_label_roi )
assert (block_image_roi[:, -1] == [0,1]).all()
num_channels = image_slot.meta.shape[-1]
block_image_roi[:, -1] = [0, num_channels]
# Ensure the results are plain ndarray, not VigraArray,
# which some classifiers might have trouble with.
block_label_data = numpy.asarray( label_slot(*block_label_roi).wait() )
block_image_data = numpy.asarray( image_slot(*block_image_roi).wait() )
label_data_blocks.append( block_label_data )
image_data_blocks.append( block_image_data )
logger.debug("Training new classifier: {}".format( classifier_factory.description ))
classifier = classifier_factory.create_and_train_pixelwise( image_data_blocks, label_data_blocks )
assert issubclass(type(classifier), LazyflowPixelwiseClassifierABC), \
"Classifier is of type {}, which does not satisfy the LazyflowPixelwiseClassifierABC interface."\
"".format( type(classifier) )
result[0] = classifier
return result
def test3(self):
# Generate a random dataset and see if it we get the right masking from the operator.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
# Provide input read all output.
self.operator_identity.Input.setValue(numpy.zeros_like(data))
output = self.operator_identity.Output[None].wait()
assert (output == 0).all()
# Try setInSlot
data_shape_roi = roiFromShape(data.shape)
data_shape_slice = roiToSlice(*data_shape_roi)
self.operator_identity.Input[data_shape_slice] = data
output = self.operator_identity.Output[None].wait()
assert (data == output).all()
def propagateDirty(self, slot, subindex, roi):
dirty_roi = self._standardize_roi( roi.start, roi.stop )
maximum_roi = roiFromShape(self.Input.meta.shape)
maximum_roi = self._standardize_roi( *maximum_roi )
if dirty_roi == maximum_roi:
# Optimize the common case:
# Everything is dirty, so no need to loop
self._resetBlocks()
else:
# FIXME: This is O(N) for now.
# We should speed this up by maintaining a bookkeeping data structure in execute().
for block_roi in self._block_data.keys():
if getIntersection(block_roi, dirty_roi, assertIntersect=False):
self.freeBlock(block_roi)
self.Output.setDirty( roi.start, roi.stop )
def test3(self):
# Generate a random dataset and see if it we get the right masking from the operator.
data = numpy.random.random((4, 5, 6, 7, 3)).astype(numpy.float32)
# Provide input read all output.
self.operator_identity.Input.setValue(numpy.zeros_like(data))
output = self.operator_identity.Output[None].wait()
assert (output == 0).all()
# Try setInSlot
data_shape_roi = roiFromShape(data.shape)
data_shape_slice = roiToSlice(*data_shape_roi)
self.operator_identity.Input[data_shape_slice] = data
output = self.operator_identity.Output[None].wait()
assert (data == output).all()
def propagateDirty(self, slot, subindex, roi):
dirty_roi = self._standardize_roi(roi.start, roi.stop)
maximum_roi = roiFromShape(self.Input.meta.shape)
maximum_roi = self._standardize_roi(*maximum_roi)
if dirty_roi == maximum_roi:
# Optimize the common case:
# Everything is dirty, so no need to loop
self._resetBlocks()
else:
# FIXME: This is O(N) for now.
# We should speed this up by maintaining a bookkeeping data structure in execute().
for block_roi in self._block_data.keys():
if getIntersection(block_roi, dirty_roi,
assertIntersect=False):
self.freeBlock(block_roi)
self.Output.setDirty(roi.start, roi.stop)
def _setInSlotInput(self, slot, subindex, roi, new_pixels):
"""
Since this is a label array, inserting pixels has a special meaning:
We only overwrite the new non-zero pixels. In the new data, zeros mean "don't change".
So, here's what each pixel we're adding means:
0: don't change
1: change to 1
2: change to 2
...
N: change to N
magic_eraser_value: change to 0
"""
if isinstance(new_pixels, vigra.VigraArray):
new_pixels = new_pixels.view(numpy.ndarray)
# Extract the data to modify
original_data = self.Output.stype.allocateDestination(
SubRegion(self.Output, *roiFromShape(new_pixels.shape)))
self.execute(self.Output, (), roi, original_data)
# Reset the pixels we need to change (so we can use |= below)
original_data[new_pixels.nonzero()] = 0
# Update
original_data |= new_pixels
# Replace 'eraser' values with zeros.
cleaned_data = original_data.copy()
cleaned_data[original_data == self._eraser_magic_value] = 0
# Set in the cache (our superclass).
super(OpCompressedUserLabelArray,
self)._setInSlotInput(slot,
subindex,
roi,
cleaned_data,
store_zero_blocks=False)
# FIXME: Shouldn't this notification be triggered from within OpUnmanagedCompressedCache?
self.Output.setDirty(roi.start, roi.stop)
return cleaned_data # Internal use: Return the cleaned_data
def autoSeedBackground(cls, laneView, foreground_label):
# Seed the entire image with background labels, except for the individual label in question
# To save memory, we'll do this in blocks instead of all at once
volume_shape = laneView.RavelerLabels.meta.shape
volume_roi = roiFromShape( volume_shape )
block_shape = (OpSplitBodyCarving.BLOCK_SIZE,) * len( volume_shape )
block_shape = numpy.minimum( block_shape, volume_shape )
block_starts = getIntersectingBlocks( block_shape, volume_roi )
logger.debug("Auto-seeding {} blocks for label".format( len(block_starts), foreground_label ))
for block_index, block_start in enumerate(block_starts):
block_roi = getBlockBounds( volume_shape, block_shape, block_start )
label_block = laneView.RavelerLabels(*block_roi).wait()
background_block = numpy.where( label_block == foreground_label, 0, 1 )
background_block = numpy.asarray( background_block, numpy.float32 ) # Distance transform requires float
if (background_block == 0.0).any():
# We need to leave a small border between the background seeds and the object membranes
background_block_view = background_block.view( vigra.VigraArray )
background_block_view.axistags = copy.copy( laneView.RavelerLabels.meta.axistags )
background_block_view_4d = background_block_view.bindAxis('t', 0)
background_block_view_3d = background_block_view_4d.bindAxis('c', 0)
distance_transformed_block = vigra.filters.distanceTransform3D(background_block_view_3d, background=False)
distance_transformed_block = distance_transformed_block.astype( numpy.uint8 )
# Create a 'hull' surrounding the foreground, but leave some space.
background_seed_block = (distance_transformed_block == OpSplitBodyCarving.SEED_MARGIN)
background_seed_block = background_seed_block.astype(numpy.uint8) * 1 # (In carving, background is label 1)
# # Make the hull VERY sparse to avoid over-biasing graph cut toward the background class
# # FIXME: Don't regenerate this random block on every loop iteration
# rand_bytes = numpy.random.randint(0, 1000, background_seed_block.shape)
# background_seed_block = numpy.where( rand_bytes < 1, background_seed_block, 0 )
# background_seed_block = background_seed_block.view(vigra.VigraArray)
# background_seed_block.axistags = background_block_view_3d.axistags
axisorder = laneView.RavelerLabels.meta.getTaggedShape().keys()
logger.debug("Writing backgound seeds: {}/{}".format( block_index, len(block_starts) ))
laneView.WriteSeeds[ roiToSlice( *block_roi ) ] = background_seed_block.withAxes(*axisorder)
else:
logger.debug("Skipping all-background block: {}/{}".format( block_index, len(block_starts) ))
def run_export(self):
"""
Request the volume in slices (running in parallel), and write each slice to a separate image.
"""
# Make the directory first if necessary
export_dir = os.path.split(self.FilepathPattern.value)[0]
if not os.path.exists(export_dir):
os.makedirs(export_dir)
# Sliceshape is the same as the input shape, except for the sliced dimension
tagged_sliceshape = self.Input.meta.getTaggedShape()
tagged_sliceshape[self._volume_axes[0]] = 1
slice_shape = list(tagged_sliceshape.values())
parallel_requests = 4
# If ram usage info is available, make a better guess about how many requests we can launch in parallel
ram_usage_per_requested_pixel = self.Input.meta.ram_usage_per_requested_pixel
if ram_usage_per_requested_pixel is not None:
pixels_per_slice = numpy.prod(slice_shape)
if "c" in tagged_sliceshape:
pixels_per_slice //= tagged_sliceshape["c"]
ram_usage_per_slice = pixels_per_slice * ram_usage_per_requested_pixel
# Fudge factor: Reduce RAM usage by a bit
available_ram = psutil.virtual_memory().available
available_ram *= 0.5
parallel_requests = int(available_ram // ram_usage_per_slice)
if parallel_requests < 1:
raise MemoryError(
"Not enough RAM to export to the selected format. " "Consider exporting to hdf5 (h5)."
)
streamer = BigRequestStreamer(self.Input, roiFromShape(self.Input.meta.shape), slice_shape, parallel_requests)
# Write the slices as they come in (possibly out-of-order, but probably not)
streamer.resultSignal.subscribe(self._write_slice)
streamer.progressSignal.subscribe(self.progressSignal)
logger.debug(f"Starting Stack Export with slicing shape: {slice_shape}")
streamer.execute()
def export_to_tiles(volume, tile_size, output_dir, print_progress=True):
"""
volume: The volume to export (either hdf5 dataset or numpy array). Must be 3D.
tile_size: The width of the tiles to generate
output_dir: The directory to dump the tiles to.
"""
assert len(volume.shape) == 3
tile_blockshape = (1, tile_size, tile_size)
tile_starts = getIntersectingBlocks(tile_blockshape,
roiFromShape(volume.shape))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
logger.info("Writing {} tiles ...".format(len(tile_starts)))
for tile_start in tile_starts:
tile_roi = getBlockBounds(volume.shape, tile_blockshape, tile_start)
if print_progress:
sys.stdout.write("Tile: {} ".format(tile_roi))
sys.stdout.flush()
tile_data = volume[roiToSlice(*tile_roi)]
tile_data = vigra.taggedView(tile_data, 'zyx')
if print_progress:
sys.stdout.write('reading... ')
sys.stdout.flush()
tile_name = 'tile_z{:05}_y{:05}_x{:05}.png'.format(*tile_start)
output_path = os.path.join(output_dir, tile_name)
if print_progress:
sys.stdout.write('writing... ')
sys.stdout.flush()
vigra.impex.writeImage(tile_data[0], output_path, dtype='NATIVE')
if print_progress:
sys.stdout.write('done.\n')
sys.stdout.flush()
logger.info("TILES COMPLETE.")
def testBasic_Dvid(self):
if _skip_dvid:
raise nose.SkipTest
# Spin up a mock dvid server to test with.
dvid_dataset, data_uuid, data_name = "datasetA", "abcde", "indices_data"
mockserver_data_file = self._tmpdir + '/mockserver_data.h5'
with H5MockServerDataFile( mockserver_data_file ) as test_h5file:
test_h5file.add_node( dvid_dataset, data_uuid )
server_proc, shutdown_event = H5MockServer.create_and_start( mockserver_data_file, "localhost", 8000,
same_process=False, disable_server_logging=True )
try:
data = 255 * numpy.random.random( (100,100, 4) )
data = data.astype( numpy.uint8 )
data = vigra.taggedView( data, vigra.defaultAxistags('xyc') )
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue( data )
opExport = OpExportSlot(graph=graph)
opExport.Input.connect( opPiper.Output )
opExport.OutputFormat.setValue( 'dvid' )
url = 'http://localhost:8000/api/node/{data_uuid}/{data_name}'.format( **locals() )
opExport.OutputFilenameFormat.setValue( url )
assert opExport.ExportPath.ready()
assert opExport.ExportPath.value == url
opExport.run_export()
try:
opRead = OpInputDataReader( graph=graph )
opRead.FilePath.setValue( opExport.ExportPath.value )
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output( *roiFromShape(data.shape) ).wait()
assert (read_data == expected_data).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
finally:
shutdown_event.set()
server_proc.join()
def export_to_tiles(volume, tile_size, output_dir, print_progress=True):
"""
volume: The volume to export (either hdf5 dataset or numpy array). Must be 3D.
tile_size: The width of the tiles to generate
output_dir: The directory to dump the tiles to.
"""
assert len(volume.shape) == 3
tile_blockshape = (1, tile_size, tile_size)
tile_starts = getIntersectingBlocks( tile_blockshape, roiFromShape( volume.shape ) )
if not os.path.exists(output_dir):
os.makedirs(output_dir)
logger.info("Writing {} tiles ...".format( len(tile_starts) ) )
for tile_start in tile_starts:
tile_roi = getBlockBounds( volume.shape, tile_blockshape, tile_start )
if print_progress:
sys.stdout.write("Tile: {} ".format( tile_roi ))
sys.stdout.flush()
tile_data = volume[ roiToSlice(*tile_roi) ]
tile_data = vigra.taggedView(tile_data, 'zyx')
if print_progress:
sys.stdout.write('reading... ')
sys.stdout.flush()
tile_name = 'tile_z{:05}_y{:05}_x{:05}.png'.format( *tile_start )
output_path = os.path.join( output_dir, tile_name )
if print_progress:
sys.stdout.write('writing... ')
sys.stdout.flush()
vigra.impex.writeImage( tile_data[0], output_path, dtype='NATIVE' )
if print_progress:
sys.stdout.write('done.\n')
sys.stdout.flush()
logger.info( "TILES COMPLETE." )
def _purge_label(self, label_to_purge, decrement_remaining, replacement_value=0):
"""
Scan through all labeled pixels.
(1) Reassign all pixels of the given value (set to replacement_value)
(2) If decrement_remaining=True, decrement all labels above that
value so the set of stored labels remains consecutive.
Note that the decrement is performed AFTER replacement.
"""
changed_block_rois = []
# stored_block_rois = self.CleanBlocks.value
stored_block_roi_destination = [None]
self.execute(self.CleanBlocks, (), SubRegion(self.Output, (0,), (1,)), stored_block_roi_destination)
stored_block_rois = stored_block_roi_destination[0]
for block_roi in stored_block_rois:
# Get data
block_shape = numpy.subtract(block_roi[1], block_roi[0])
block = self.Output.stype.allocateDestination(SubRegion(self.Output, *roiFromShape(block_shape)))
self.execute(self.Output, (), SubRegion(self.Output, *block_roi), block)
# Locate pixels to change
matching_label_coords = numpy.nonzero(block == label_to_purge)
# Change the data
block[matching_label_coords] = replacement_value
coords_to_decrement = block > label_to_purge
if decrement_remaining:
block[coords_to_decrement] -= numpy.uint8(1)
# Update cache with the new data (only if something really changed)
if len(matching_label_coords[0]) > 0 or (decrement_remaining and coords_to_decrement.sum() > 0):
super(OpCompressedUserLabelArray, self)._setInSlotInput(
self.Input, (), SubRegion(self.Output, *block_roi), block, store_zero_blocks=False
)
changed_block_rois.append(block_roi)
for block_roi in changed_block_rois:
# FIXME: Shouldn't this dirty notification be handled in OpUnmanagedCompressedCache?
self.Output.setDirty(*block_roi)
def run_export(self):
self.progressSignal(0)
url = self.NodeDataUrl.value
url_path = url.split('://')[1]
hostname, api, node, uuid, dataname = url_path.split('/')
assert api == 'api'
assert node == 'node'
# Request the data
axiskeys = self.Input.meta.getAxisKeys()
data = self.Input[:].wait()
if self._transpose_axes:
data = data.transpose()
axiskeys = reversed(axiskeys)
axiskeys = "".join( axiskeys )
# FIXME: We assume the dataset needs to be created first.
# If it already existed, this (presumably) causes an error on the DVID side.
metadata = pydvid.voxels.VoxelsMetadata.create_default_metadata( data.shape, data.dtype.type, axiskeys, 0.0, "" )
connection = httplib.HTTPConnection( hostname )
with contextlib.closing( connection ):
self.progressSignal(5)
pydvid.voxels.create_new(connection, uuid, dataname, metadata)
client = pydvid.voxels.VoxelsAccessor( connection, uuid, dataname )
# For now, we send the whole darn thing at once.
# TODO: Stream it over in blocks...
# Send it to dvid
start, stop = roiFromShape(data.shape)
client.post_ndarray( start, stop, data )
self.progressSignal(100)
def execute(self, slot, subindex, rroi, result):
self.progressSignal(0)
# Save the axistags as a dataset attribute
self.d.attrs["axistags"] = self.Image.meta.axistags.toJSON()
if isinstance(self.d, h5py.Dataset):
for index, tag in enumerate(self.Image.meta.axistags):
self.d.dims[index].label = tag.key
else: # if n5 dataset, apply neuroglancer's axes tags convention
self.d.attrs["axes"] = "".join(
tag.key for tag in self.Image.meta.axistags)[::-1]
drange = self.Image.meta.get("drange")
if drange:
self.d.attrs["drange"] = drange
def handle_block_result(roi, data):
slicing = roiToSlice(*roi)
if data.flags.c_contiguous:
self.d.write_direct(data.view(numpy.ndarray), dest_sel=slicing)
else:
self.d[slicing] = data
batch_size = None
if self.BatchSize.ready():
batch_size = self.BatchSize.value
requester = BigRequestStreamer(self.Image,
roiFromShape(self.Image.meta.shape),
batchSize=batch_size)
requester.resultSignal.subscribe(handle_block_result)
requester.progressSignal.subscribe(self.progressSignal)
requester.execute()
# Be paranoid: Flush right now.
if isinstance(self.f, h5py.File):
self.f.file.flush() # not available in z5py
# We're finished.
result[0] = True
self.progressSignal(100)
def run_export(self):
self.progressSignal(0)
url = self.NodeDataUrl.value
url_path = url.split('://')[1]
hostname, api, node, uuid, dataname = url_path.split('/')
assert api == 'api'
assert node == 'node'
# Request the data
axiskeys = self.Input.meta.getAxisKeys()
data = self.Input[:].wait()
if self._transpose_axes:
data = data.transpose()
axiskeys = reversed(axiskeys)
axiskeys = "".join(axiskeys)
# FIXME: We assume the dataset needs to be created first.
# If it already existed, this (presumably) causes an error on the DVID side.
metadata = pydvid.voxels.VoxelsMetadata.create_default_metadata(
data.shape, data.dtype.type, axiskeys, 0.0, "")
connection = httplib.HTTPConnection(hostname)
with contextlib.closing(connection):
self.progressSignal(5)
pydvid.voxels.create_new(connection, uuid, dataname, metadata)
client = pydvid.voxels.VoxelsAccessor(connection, uuid, dataname)
# For now, we send the whole darn thing at once.
# TODO: Stream it over in blocks...
# Send it to dvid
start, stop = roiFromShape(data.shape)
client.post_ndarray(start, stop, data)
self.progressSignal(100)
graph = Graph()
opReader = OpInputDataReader(graph=graph)
opReader.WorkingDirectory.setValue( cwd )
opReader.FilePath.setValue( label_data_path )
print "Reading label volume: {}".format( label_data_path )
label_volume = opReader.Output[:].wait()
axiskeys = opPixelClassification
raw_shape = opPixelClassification.InputImages[lane].meta.shape
if label_volume.ndim != len(raw_shape):
# Append a singleton channel axis
assert label_volume.ndim == len(raw_shape)-1
label_volume = label_volume[...,None]
print "Applying label volume to lane #{}".format(lane)
entire_volume_slicing = roiToSlice(*roiFromShape(label_volume.shape))
opPixelClassification.LabelInputs[lane][entire_volume_slicing] = label_volume
##
## REQUEST TRAINED CLASSIFIER
##
opPixelClassification.FreezePredictions.setValue(False)
_ = opPixelClassification.Classifier.value
##
## SAVE PROJECT
##
# save project file (saves new classifier).
shell.projectManager.saveProject(force_all_save=False)
def _init_fixed_dirty_roi(self):
# Intentionally flipped: nothing is dirty at first.
entire_roi = roiFromShape(self.Input.meta.shape)
self._fixed_dirty_roi = (entire_roi[1], entire_roi[0])
def _deserializeFromHdf5(self, topGroup, groupVersion, hdf5File, projectFilePath):
obj = topGroup["objects"]
for imageIndex, opCarving in enumerate(self._o.innerOperators):
mst = opCarving._mst
for i, name in enumerate(obj):
print " loading object with name='%s'" % name
try:
g = obj[name]
fg_voxels = g["fg_voxels"]
bg_voxels = g["bg_voxels"]
fg_voxels = [fg_voxels[:, k] for k in range(3)]
bg_voxels = [bg_voxels[:, k] for k in range(3)]
sv = g["sv"].value
mst.object_names[name] = i + 1
mst.object_seeds_fg_voxels[name] = fg_voxels
mst.object_seeds_bg_voxels[name] = bg_voxels
mst.object_lut[name] = sv
mst.bg_priority[name] = g["bg_prio"].value
mst.no_bias_below[name] = g["no_bias_below"].value
print "[CarvingSerializer] de-serializing %s, with opCarving=%d, mst=%d" % (
name,
id(opCarving),
id(mst),
)
print " %d voxels labeled with green seed" % fg_voxels[0].shape[0]
print " %d voxels labeled with red seed" % bg_voxels[0].shape[0]
print " object is made up of %d supervoxels" % sv.size
print " bg priority = %f" % mst.bg_priority[name]
print " no bias below = %d" % mst.no_bias_below[name]
except Exception as e:
print "object %s could not be loaded due to exception: %s" % (name, e)
shape = opCarving.opLabelArray.Output.meta.shape
dtype = opCarving.opLabelArray.Output.meta.dtype
fg_voxels = None
if "fg_voxels" in topGroup.keys():
fg_voxels = topGroup["fg_voxels"]
fg_voxels = [fg_voxels[:, k] for k in range(3)]
bg_voxels = None
if "bg_voxels" in topGroup.keys():
bg_voxels = topGroup["bg_voxels"]
bg_voxels = [bg_voxels[:, k] for k in range(3)]
# Start with inverse roi
total_roi = roiFromShape(opCarving.opLabelArray.Output.meta.shape)
bounding_box_roi = numpy.array([total_roi[1][1:4], total_roi[0][1:4]])
if fg_voxels is not None and len(fg_voxels[0]) > 0:
fg_bounding_box_start = numpy.array(map(numpy.min, fg_voxels))
fg_bounding_box_stop = 1 + numpy.array(map(numpy.max, fg_voxels))
bounding_box_roi[0] = numpy.minimum(bounding_box_roi[0], fg_bounding_box_start)
bounding_box_roi[1] = numpy.maximum(bounding_box_roi[1], fg_bounding_box_stop)
if bg_voxels is not None and len(bg_voxels[0]) > 0:
bg_bounding_box_start = numpy.array(map(numpy.min, bg_voxels))
bg_bounding_box_stop = 1 + numpy.array(map(numpy.max, bg_voxels))
bounding_box_roi[0] = numpy.minimum(bounding_box_roi[0], bg_bounding_box_start)
bounding_box_roi[1] = numpy.maximum(bounding_box_roi[1], bg_bounding_box_stop)
if (bounding_box_roi[1] > bounding_box_roi[0]).all():
z = numpy.zeros(bounding_box_roi[1] - bounding_box_roi[0], dtype=dtype)
if fg_voxels is not None:
fg_voxels = fg_voxels - numpy.array([bounding_box_roi[0]]).transpose()
z[fg_voxels] = 2
if bg_voxels is not None:
bg_voxels = bg_voxels - numpy.array([bounding_box_roi[0]]).transpose()
z[bg_voxels] = 1
bounding_box_slicing = roiToSlice(bounding_box_roi[0], bounding_box_roi[1])
opCarving.WriteSeeds[(slice(0, 1),) + bounding_box_slicing + (slice(0, 1),)] = z[
numpy.newaxis, :, :, :, numpy.newaxis
]
print "restored seeds"
opCarving._buildDone()
def _init_fixed_dirty_roi(self):
# Intentionally flipped: nothing is dirty at first.
entire_roi = roiFromShape(self.Input.meta.shape)
self._fixed_dirty_roi = (entire_roi[1], entire_roi[0])
def generate_trained_project_file(new_project_path,
raw_data_paths,
label_data_paths,
feature_selections,
classifier_factory=None):
"""
Create a new project file from scratch, add the given raw data files,
inject the corresponding labels, configure the given feature selections,
and (if provided) override the classifier type ('factory').
Finally, request the classifier object from the pipeline (which forces training),
and save the project.
new_project_path: Where to save the new project file
raw_data_paths: A list of paths to the raw data images to train with
label_data_paths: A list of paths to the label image data to train with
feature_selections: A matrix of bool, representing the selected features
classifier_factory: Override the classifier type. Must be a subclass of either:
- lazyflow.classifiers.LazyflowVectorwiseClassifierFactoryABC
- lazyflow.classifiers.LazyflowPixelwiseClassifierFactoryABC
"""
assert len(raw_data_paths) == len(
label_data_paths
), "Number of label images must match number of raw images."
import ilastik_main
from ilastik.workflows.pixelClassification import PixelClassificationWorkflow
from lazyflow.graph import Graph
from lazyflow.operators.ioOperators import OpInputDataReader
from lazyflow.roi import roiToSlice, roiFromShape
##
## CREATE PROJECT
##
# Manually configure the arguments to ilastik, as if they were parsed from the command line.
# (Start with empty args and fill in below.)
ilastik_args = ilastik_main.parse_args([])
ilastik_args.new_project = new_project_path
ilastik_args.headless = True
ilastik_args.workflow = "Pixel Classification"
shell = ilastik_main.main(ilastik_args)
assert isinstance(shell.workflow, PixelClassificationWorkflow)
##
## CONFIGURE GRAYSCALE INPUT
##
data_selection_applet = shell.workflow.dataSelectionApplet
# To configure data selection, start with empty cmdline args and manually fill them in
data_selection_args, _ = data_selection_applet.parse_known_cmdline_args(
[], PixelClassificationWorkflow.ROLE_NAMES)
data_selection_args.raw_data = raw_data_paths
data_selection_args.preconvert_stacks = True
# Simplest thing here is to configure using cmd-line interface
data_selection_applet.configure_operator_with_parsed_args(
data_selection_args)
##
## APPLY FEATURE MATRIX (from matrix above)
##
opFeatures = shell.workflow.featureSelectionApplet.topLevelOperator
opFeatures.Scales.setValue(ScalesList)
opFeatures.FeatureIds.setValue(FeatureIds)
opFeatures.SelectionMatrix.setValue(feature_selections)
##
## CUSTOMIZE CLASSIFIER TYPE
##
opPixelClassification = shell.workflow.pcApplet.topLevelOperator
if classifier_factory is not None:
opPixelClassification.ClassifierFactory.setValue(classifier_factory)
##
## READ/APPLY LABEL VOLUMES
##
# Read each label volume and inject the label data into the appropriate training slot
cwd = os.getcwd()
max_label_class = 0
for lane, label_data_path in enumerate(label_data_paths):
graph = Graph()
opReader = OpInputDataReader(graph=graph)
try:
opReader.WorkingDirectory.setValue(cwd)
opReader.FilePath.setValue(label_data_path)
print("Reading label volume: {}".format(label_data_path))
label_volume = opReader.Output[:].wait()
finally:
opReader.cleanUp()
raw_shape = opPixelClassification.InputImages[lane].meta.shape
if label_volume.ndim != len(raw_shape):
# Append a singleton channel axis
assert label_volume.ndim == len(raw_shape) - 1
label_volume = label_volume[..., None]
# Auto-calculate the max label value
max_label_class = max(max_label_class, label_volume.max())
print("Applying label volume to lane #{}".format(lane))
entire_volume_slicing = roiToSlice(*roiFromShape(label_volume.shape))
opPixelClassification.LabelInputs[lane][
entire_volume_slicing] = label_volume
assert max_label_class > 1, "Not enough label classes were found in your label data."
label_names = list(map(str, list(range(max_label_class))))
opPixelClassification.LabelNames.setValue(label_names)
##
## TRAIN CLASSIFIER
##
# Make sure the caches in the pipeline are not 'frozen'.
# (This is the equivalent of 'live update' mode in the GUI.)
opPixelClassification.FreezePredictions.setValue(False)
# Request the classifier object from the pipeline.
# This forces the pipeline to produce (train) the classifier.
_ = opPixelClassification.Classifier.value
##
## SAVE PROJECT
##
# save project file (includes the new classifier).
shell.projectManager.saveProject(force_all_save=False)
def generate_trained_project_file(
new_project_path, raw_data_paths, label_data_paths, feature_selections, classifier_factory=None
):
"""
Create a new project file from scratch, add the given raw data files,
inject the corresponding labels, configure the given feature selections,
and (if provided) override the classifier type ('factory').
Finally, request the classifier object from the pipeline (which forces training),
and save the project.
new_project_path: Where to save the new project file
raw_data_paths: A list of paths to the raw data images to train with
label_data_paths: A list of paths to the label image data to train with
feature_selections: A matrix of bool, representing the selected features
classifier_factory: Override the classifier type. Must be a subclass of either:
- lazyflow.classifiers.LazyflowVectorwiseClassifierFactoryABC
- lazyflow.classifiers.LazyflowPixelwiseClassifierFactoryABC
"""
assert len(raw_data_paths) == len(label_data_paths), "Number of label images must match number of raw images."
import ilastik_main
from ilastik.workflows.pixelClassification import PixelClassificationWorkflow
from lazyflow.graph import Graph
from lazyflow.operators.ioOperators import OpInputDataReader
from lazyflow.roi import roiToSlice, roiFromShape
##
## CREATE PROJECT
##
# Manually configure the arguments to ilastik, as if they were parsed from the command line.
# (Start with empty args and fill in below.)
ilastik_args = ilastik_main.parser.parse_args([])
ilastik_args.new_project = new_project_path
ilastik_args.headless = True
ilastik_args.workflow = "Pixel Classification"
shell = ilastik_main.main(ilastik_args)
assert isinstance(shell.workflow, PixelClassificationWorkflow)
##
## CONFIGURE GRAYSCALE INPUT
##
data_selection_applet = shell.workflow.dataSelectionApplet
# To configure data selection, start with empty cmdline args and manually fill them in
data_selection_args, _ = data_selection_applet.parse_known_cmdline_args([], PixelClassificationWorkflow.ROLE_NAMES)
data_selection_args.raw_data = raw_data_paths
data_selection_args.preconvert_stacks = True
# Simplest thing here is to configure using cmd-line interface
data_selection_applet.configure_operator_with_parsed_args(data_selection_args)
##
## APPLY FEATURE MATRIX (from matrix above)
##
opFeatures = shell.workflow.featureSelectionApplet.topLevelOperator
opFeatures.Scales.setValue(ScalesList)
opFeatures.FeatureIds.setValue(FeatureIds)
opFeatures.SelectionMatrix.setValue(feature_selections)
##
## CUSTOMIZE CLASSIFIER TYPE
##
opPixelClassification = shell.workflow.pcApplet.topLevelOperator
if classifier_factory is not None:
opPixelClassification.ClassifierFactory.setValue(classifier_factory)
##
## READ/APPLY LABEL VOLUMES
##
# Read each label volume and inject the label data into the appropriate training slot
cwd = os.getcwd()
max_label_class = 0
for lane, label_data_path in enumerate(label_data_paths):
graph = Graph()
opReader = OpInputDataReader(graph=graph)
try:
opReader.WorkingDirectory.setValue(cwd)
opReader.FilePath.setValue(label_data_path)
print "Reading label volume: {}".format(label_data_path)
label_volume = opReader.Output[:].wait()
finally:
opReader.cleanUp()
raw_shape = opPixelClassification.InputImages[lane].meta.shape
if label_volume.ndim != len(raw_shape):
# Append a singleton channel axis
assert label_volume.ndim == len(raw_shape) - 1
label_volume = label_volume[..., None]
# Auto-calculate the max label value
max_label_class = max(max_label_class, label_volume.max())
print "Applying label volume to lane #{}".format(lane)
entire_volume_slicing = roiToSlice(*roiFromShape(label_volume.shape))
opPixelClassification.LabelInputs[lane][entire_volume_slicing] = label_volume
assert max_label_class > 1, "Not enough label classes were found in your label data."
label_names = map(str, range(max_label_class))
opPixelClassification.LabelNames.setValue(label_names)
##
## TRAIN CLASSIFIER
##
# Make sure the caches in the pipeline are not 'frozen'.
# (This is the equivalent of 'live update' mode in the GUI.)
opPixelClassification.FreezePredictions.setValue(False)
# Request the classifier object from the pipeline.
# This forces the pipeline to produce (train) the classifier.
_ = opPixelClassification.Classifier.value
##
## SAVE PROJECT
##
# save project file (includes the new classifier).
shell.projectManager.saveProject(force_all_save=False)
def import_labeling_layer(labelLayer, labelingSlots, parent_widget=None):
"""
Prompt the user for layer import settings, and perform the layer import.
:param labelLayer: The top label layer source
:param labelingSlots: An instance of LabelingGui.LabelingSlots
:param parent_widget: The Qt GUI parent object
"""
writeSeeds = labelingSlots.labelInput
assert isinstance(writeSeeds, lazyflow.graph.Slot), "slot is of type %r" % (type(writeSeeds))
opLabels = writeSeeds.getRealOperator()
assert isinstance(opLabels, lazyflow.graph.Operator), "slot's operator is of type %r" % (type(opLabels))
recentlyImported = PreferencesManager().get('labeling', 'recently imported')
mostRecentProjectPath = PreferencesManager().get('shell', 'recently opened')
mostRecentImageFile = PreferencesManager().get( 'DataSelection', 'recent image' )
if recentlyImported:
defaultDirectory = os.path.split(recentlyImported)[0]
elif mostRecentProjectPath:
defaultDirectory = os.path.split(mostRecentProjectPath)[0]
elif mostRecentImageFile:
defaultDirectory = os.path.split(mostRecentImageFile)[0]
else:
defaultDirectory = os.path.expanduser('~')
fileNames = DataSelectionGui.getImageFileNamesToOpen(parent_widget, defaultDirectory)
fileNames = map(str, fileNames)
if not fileNames:
return
PreferencesManager().set('labeling', 'recently imported', fileNames[0])
try:
# Initialize operators
opImport = OpInputDataReader( parent=opLabels.parent )
opCache = OpArrayCache( parent=opLabels.parent )
opMetadataInjector = OpMetadataInjector( parent=opLabels.parent )
opReorderAxes = OpReorderAxes( parent=opLabels.parent )
# Set up the pipeline as follows:
#
# opImport --> opCache --> opMetadataInjector --------> opReorderAxes --(inject via setInSlot)--> labelInput
# / /
# User-specified axisorder labelInput.meta.axistags
opImport.WorkingDirectory.setValue(defaultDirectory)
opImport.FilePath.setValue(fileNames[0] if len(fileNames) == 1 else
os.path.pathsep.join(fileNames))
assert opImport.Output.ready()
opCache.blockShape.setValue( opImport.Output.meta.shape )
opCache.Input.connect( opImport.Output )
assert opCache.Output.ready()
opMetadataInjector.Input.connect( opCache.Output )
metadata = opCache.Output.meta.copy()
opMetadataInjector.Metadata.setValue( metadata )
opReorderAxes.Input.connect( opMetadataInjector.Output )
# Transpose the axes for assignment to the labeling operator.
opReorderAxes.AxisOrder.setValue( writeSeeds.meta.getAxisKeys() )
# We'll show a little window with a busy indicator while the data is loading
busy_dlg = QProgressDialog(parent=parent_widget)
busy_dlg.setLabelText("Importing Label Data...")
busy_dlg.setCancelButton(None)
busy_dlg.setMinimum(100)
busy_dlg.setMaximum(100)
def close_busy_dlg(*args):
QApplication.postEvent(busy_dlg, QCloseEvent())
# Load the data from file into our cache
# When it's done loading, close the progress dialog.
req = opCache.Output[:]
req.notify_finished( close_busy_dlg )
req.notify_failed( close_busy_dlg )
req.submit()
busy_dlg.exec_()
readData = req.result
maxLabels = len(labelingSlots.labelNames.value)
# Can't use return_counts feature because that requires numpy >= 1.9
#unique_read_labels, readLabelCounts = numpy.unique(readData, return_counts=True)
# This does the same as the above, albeit slower, and probably with more ram.
unique_read_labels = numpy.unique(readData)
readLabelCounts = vigra_bincount(readData)[unique_read_labels]
labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
del readData
# Ask the user how to interpret the data.
settingsDlg = LabelImportOptionsDlg( parent_widget,
fileNames, opMetadataInjector.Output,
labelingSlots.labelInput, labelInfo )
def handle_updated_axes():
# The user is specifying a new interpretation of the file's axes
updated_axisorder = str(settingsDlg.axesEdit.text())
metadata = opMetadataInjector.Metadata.value.copy()
metadata.axistags = vigra.defaultAxistags(updated_axisorder)
opMetadataInjector.Metadata.setValue( metadata )
if opReorderAxes._invalid_axes:
settingsDlg.buttonBox.button(QDialogButtonBox.Ok).setEnabled(False)
# Red background
settingsDlg.axesEdit.setStyleSheet("QLineEdit { background: rgb(255, 128, 128);"
"selection-background-color: rgb(128, 128, 255); }")
settingsDlg.axesEdit.editingFinished.connect( handle_updated_axes )
# Initialize
handle_updated_axes()
dlg_result = settingsDlg.exec_()
if dlg_result != LabelImportOptionsDlg.Accepted:
return
# Get user's chosen label mapping from dlg
labelMapping = settingsDlg.labelMapping
# Get user's chosen offsets.
# Offsets in dlg only include the file axes, not the 5D axes expected by the label input,
# so expand them to full 5D
axes_5d = opReorderAxes.Output.meta.getAxisKeys()
tagged_offsets = collections.OrderedDict( zip( axes_5d, [0]*len(axes_5d) ) )
tagged_offsets.update( dict( zip( opMetadataInjector.Output.meta.getAxisKeys(), settingsDlg.imageOffsets ) ) )
imageOffsets = tagged_offsets.values()
# Optimization if mapping is identity
if labelMapping.keys() == labelMapping.values():
labelMapping = None
# This will be fast (it's already cached)
label_data = opReorderAxes.Output[:].wait()
# Map input labels to output labels
if labelMapping:
# There are other ways to do a relabeling (e.g skimage.segmentation.relabel_sequential)
# But this supports potentially huge values of unique_read_labels (in the billions),
# without needing GB of RAM.
mapping_indexes = numpy.searchsorted(unique_read_labels, label_data)
new_labels = numpy.array([labelMapping[x] for x in unique_read_labels])
label_data[:] = new_labels[mapping_indexes]
label_roi = numpy.array( roiFromShape(opReorderAxes.Output.meta.shape) )
label_roi += imageOffsets
label_slice = roiToSlice(*label_roi)
writeSeeds[label_slice] = label_data
finally:
opReorderAxes.cleanUp()
opMetadataInjector.cleanUp()
opCache.cleanUp()
opImport.cleanUp()
def execute(self, slot, subindex, roi, result):
classifier = self.Classifier.value
# Training operator may return 'None' if there was no data to train with
skip_prediction = (classifier is None)
# Shortcut: If the mask is totally zero, skip this request entirely
if not skip_prediction and self.PredictionMask.ready():
mask_roi = numpy.array((roi.start, roi.stop))
mask_roi[:, -1:] = [[0], [1]]
start, stop = map(tuple, mask_roi)
mask = self.PredictionMask(start, stop).wait()
skip_prediction = not numpy.any(mask)
if skip_prediction:
result[:] = 0.0
return result
assert issubclass(type(classifier), LazyflowPixelwiseClassifierABC), \
"Classifier is of type {}, which does not satisfy the LazyflowPixelwiseClassifierABC interface."\
"".format( type(classifier) )
upstream_roi = (roi.start, roi.stop)
# Ask for the halo needed by the classifier
axiskeys = self.Image.meta.getAxisKeys()
halo_shape = classifier.get_halo_shape(axiskeys)
assert len(halo_shape) == len(upstream_roi[0])
assert halo_shape[
-1] == 0, "Didn't expect a non-zero halo for channel dimension."
# Expand block by halo, then clip to image bounds
upstream_roi = numpy.array(upstream_roi)
upstream_roi[0] -= halo_shape
upstream_roi[1] += halo_shape
upstream_roi = getIntersection(upstream_roi,
roiFromShape(self.Image.meta.shape))
upstream_roi = numpy.asarray(upstream_roi)
# Determine how to extract the data from the result (without the halo)
downstream_roi = numpy.array((roi.start, roi.stop))
downstream_channels = self.PMaps.meta.shape[-1]
roi_within_result = downstream_roi - upstream_roi[0]
roi_within_result[:, -1] = [0, downstream_channels]
# Request all upstream channels
input_channels = self.Image.meta.shape[-1]
upstream_roi[:, -1] = [0, input_channels]
# Request the data
input_data = self.Image(*upstream_roi).wait()
axistags = self.Image.meta.axistags
probabilities = classifier.predict_probabilities_pixelwise(
input_data, axistags)
# We're expecting a channel for each label class.
# If we didn't provide at least one sample for each label,
# we may get back fewer channels.
if probabilities.shape[-1] != self.PMaps.meta.shape[-1]:
# Copy to an array of the correct shape
# This is slow, but it's an unusual case
assert probabilities.shape[-1] == len(classifier.known_classes)
full_probabilities = numpy.zeros(probabilities.shape[:-1] +
(self.PMaps.meta.shape[-1], ),
dtype=numpy.float32)
for i, label in enumerate(classifier.known_classes):
full_probabilities[..., label - 1] = probabilities[..., i]
probabilities = full_probabilities
# Extract requested region (discard halo)
probabilities = probabilities[roiToSlice(*roi_within_result)]
# Copy only the prediction channels the client requested.
result[...] = probabilities[..., roi.start[-1]:roi.stop[-1]]
return result
def create_slice_req( index ):
roi = roiFromShape(slice_shape)
roi += index*slice_step
return self.Input(*roi)
def import_labeling_layer(labelLayer, labelingSlots, parent_widget=None):
"""
Prompt the user for layer import settings, and perform the layer import.
:param labelLayer: The top label layer source
:param labelingSlots: An instance of LabelingGui.LabelingSlots
:param parent_widget: The Qt GUI parent object
"""
writeSeeds = labelingSlots.labelInput
assert isinstance(
writeSeeds,
lazyflow.graph.Slot), "slot is of type %r" % (type(writeSeeds))
opLabels = writeSeeds.getRealOperator()
assert isinstance(opLabels, lazyflow.graph.Operator
), "slot's operator is of type %r" % (type(opLabels))
recentlyImported = PreferencesManager().get('labeling',
'recently imported')
mostRecentProjectPath = PreferencesManager().get('shell',
'recently opened')
mostRecentImageFile = PreferencesManager().get('DataSelection',
'recent image')
if recentlyImported:
defaultDirectory = os.path.split(recentlyImported)[0]
elif mostRecentProjectPath:
defaultDirectory = os.path.split(mostRecentProjectPath)[0]
elif mostRecentImageFile:
defaultDirectory = os.path.split(mostRecentImageFile)[0]
else:
defaultDirectory = os.path.expanduser('~')
fileNames = DataSelectionGui.getImageFileNamesToOpen(
parent_widget, defaultDirectory)
fileNames = map(str, fileNames)
if not fileNames:
return
PreferencesManager().set('labeling', 'recently imported', fileNames[0])
try:
# Initialize operators
opImport = OpInputDataReader(parent=opLabels.parent)
opCache = OpArrayCache(parent=opLabels.parent)
opMetadataInjector = OpMetadataInjector(parent=opLabels.parent)
opReorderAxes = OpReorderAxes(parent=opLabels.parent)
# Set up the pipeline as follows:
#
# opImport --> opCache --> opMetadataInjector --------> opReorderAxes --(inject via setInSlot)--> labelInput
# / /
# User-specified axisorder labelInput.meta.axistags
opImport.WorkingDirectory.setValue(defaultDirectory)
opImport.FilePath.setValue(fileNames[0] if len(fileNames) ==
1 else os.path.pathsep.join(fileNames))
assert opImport.Output.ready()
opCache.blockShape.setValue(opImport.Output.meta.shape)
opCache.Input.connect(opImport.Output)
assert opCache.Output.ready()
opMetadataInjector.Input.connect(opCache.Output)
metadata = opCache.Output.meta.copy()
opMetadataInjector.Metadata.setValue(metadata)
opReorderAxes.Input.connect(opMetadataInjector.Output)
# Transpose the axes for assignment to the labeling operator.
opReorderAxes.AxisOrder.setValue(writeSeeds.meta.getAxisKeys())
# We'll show a little window with a busy indicator while the data is loading
busy_dlg = QProgressDialog(parent=parent_widget)
busy_dlg.setLabelText("Importing Label Data...")
busy_dlg.setCancelButton(None)
busy_dlg.setMinimum(100)
busy_dlg.setMaximum(100)
def close_busy_dlg(*args):
QApplication.postEvent(busy_dlg, QCloseEvent())
# Load the data from file into our cache
# When it's done loading, close the progress dialog.
req = opCache.Output[:]
req.notify_finished(close_busy_dlg)
req.notify_failed(close_busy_dlg)
req.submit()
busy_dlg.exec_()
readData = req.result
maxLabels = len(labelingSlots.labelNames.value)
# Can't use return_counts feature because that requires numpy >= 1.9
#unique_read_labels, readLabelCounts = numpy.unique(readData, return_counts=True)
# This does the same as the above, albeit slower, and probably with more ram.
unique_read_labels = numpy.unique(readData)
readLabelCounts = numpy.bincount(readData.flat)[unique_read_labels]
labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
del readData
# Ask the user how to interpret the data.
settingsDlg = LabelImportOptionsDlg(parent_widget, fileNames,
opMetadataInjector.Output,
labelingSlots.labelInput,
labelInfo)
def handle_updated_axes():
# The user is specifying a new interpretation of the file's axes
updated_axisorder = str(settingsDlg.axesEdit.text())
metadata = opMetadataInjector.Metadata.value.copy()
metadata.axistags = vigra.defaultAxistags(updated_axisorder)
opMetadataInjector.Metadata.setValue(metadata)
settingsDlg.axesEdit.editingFinished.connect(handle_updated_axes)
dlg_result = settingsDlg.exec_()
if dlg_result != LabelImportOptionsDlg.Accepted:
return
# Get user's chosen label mapping from dlg
labelMapping = settingsDlg.labelMapping
# Get user's chosen offsets.
# Offsets in dlg only include the file axes, not the 5D axes expected by the label input,
# so expand them to full 5D
axes_5d = opReorderAxes.Output.meta.getAxisKeys()
tagged_offsets = collections.OrderedDict(
zip(axes_5d, [0] * len(axes_5d)))
tagged_offsets.update(
dict(
zip(opMetadataInjector.Output.meta.getAxisKeys(),
settingsDlg.imageOffsets)))
imageOffsets = tagged_offsets.values()
# Optimization if mapping is identity
if labelMapping.keys() == labelMapping.values():
labelMapping = None
# This will be fast (it's already cached)
label_data = opReorderAxes.Output[:].wait()
# Map input labels to output labels
if labelMapping:
# There are other ways to do a relabeling (e.g skimage.segmentation.relabel_sequential)
# But this supports potentially huge values of unique_read_labels (in the billions),
# without needing GB of RAM.
mapping_indexes = numpy.searchsorted(unique_read_labels,
label_data)
new_labels = numpy.array(
[labelMapping[x] for x in unique_read_labels])
label_data[:] = new_labels[mapping_indexes]
label_roi = numpy.array(roiFromShape(opReorderAxes.Output.meta.shape))
label_roi += imageOffsets
label_slice = roiToSlice(*label_roi)
writeSeeds[label_slice] = label_data
finally:
opReorderAxes.cleanUp()
opMetadataInjector.cleanUp()
opCache.cleanUp()
opImport.cleanUp()
